From 4925ffc10ce8e8287980eaec38b587512568a302 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Wed, 17 Jan 2018 15:26:03 +0300 Subject: [PATCH 01/65] IGNITE-7453 Use GridUnsafe.cleanDirectBuffer in PageSnapshot --- .../ignite/internal/pagemem/wal/record/PageSnapshot.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java index 674022176d152..126b84510ad9a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java @@ -25,7 +25,6 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.tostring.GridToStringExclude; -import sun.nio.ch.DirectBuffer; /** * @@ -98,7 +97,7 @@ public FullPageId fullPageId() { ", pageData = " + Arrays.toString(pageData) + ", super=" + super.toString() + "]"; } finally { - ((DirectBuffer)buf).cleaner().clean(); + GridUnsafe.cleanDirectBuffer(buf); } } } From bcd68a058683b2f17b7ac60471b6e7aab3e4f6de Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 17 Jan 2018 15:38:20 +0300 Subject: [PATCH 02/65] IGNITE-7301 .NET: Baseline topology This closes #3352 --- .../platform/PlatformContextImpl.java | 20 +-- .../platform/PlatformProcessorImpl.java | 46 +++++++ .../cluster/PlatformClusterGroup.java | 13 +- .../platform/utils/PlatformUtils.java | 41 ++++++ .../Common/TestUtils.DotNetCore.cs | 3 +- .../Apache.Ignite.Core.Tests.csproj | 1 + .../ApiParity/ClusterNodeParityTest.cs | 45 +++++++ .../ApiParity/ClusterParityTest.cs | 6 +- .../ApiParity/ParityTest.cs | 26 +++- .../Cache/DataStorageMetricsTest.cs | 6 +- .../Cache/PersistenceTest.cs | 126 +++++++++++++++--- .../Compute/ComputeApiTest.cs | 10 +- .../IgniteConfigurationTest.cs | 1 + .../TestUtils.Common.cs | 40 ++++++ .../TestUtils.Windows.cs | 3 +- .../Apache.Ignite.Core.csproj | 3 + .../Cluster/IBaselineNode.cs | 37 +++++ .../Apache.Ignite.Core/Cluster/ICluster.cs | 34 ++++- .../Cluster/IClusterNode.cs | 3 +- .../dotnet/Apache.Ignite.Core/IIgnite.cs | 2 + .../Apache.Ignite.Core/IgniteConfiguration.cs | 4 +- .../Impl/Binary/BinarySystemHandlers.cs | 16 +++ .../Impl/Binary/BinaryTypeId.cs | 3 + .../Impl/Binary/OptimizedMarshallerObject.cs | 54 ++++++++ .../Impl/Cluster/BaselineNode.cs | 89 +++++++++++++ .../Impl/Cluster/ClusterNodeImpl.cs | 38 +++++- .../dotnet/Apache.Ignite.Core/Impl/Ignite.cs | 39 +++++- .../Impl/Unmanaged/Jni/Env.cs | 2 - 28 files changed, 643 insertions(+), 68 deletions(-) create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterNodeParityTest.cs create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Cluster/IBaselineNode.cs create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/OptimizedMarshallerObject.cs create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/BaselineNode.cs diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java index b141313b4f946..9e22f38fb53d6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java @@ -72,10 +72,7 @@ import java.sql.Timestamp; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -205,27 +202,14 @@ public PlatformContextImpl(GridKernalContext ctx, PlatformCallbackGateway gate, BinaryRawWriterEx w = writer(out); w.writeUuid(node.id()); - - Map attrs = new HashMap<>(node.attributes()); - - Iterator> attrIter = attrs.entrySet().iterator(); - - while (attrIter.hasNext()) { - Map.Entry entry = attrIter.next(); - - Object val = entry.getValue(); - - if (val != null && !val.getClass().getName().startsWith("java.lang")) - attrIter.remove(); - } - - w.writeMap(attrs); + PlatformUtils.writeNodeAttributes(w, node.attributes()); w.writeCollection(node.addresses()); w.writeCollection(node.hostNames()); w.writeLong(node.order()); w.writeBoolean(node.isLocal()); w.writeBoolean(node.isDaemon()); w.writeBoolean(node.isClient()); + w.writeObjectDetached(node.consistentId()); writeClusterMetrics(w, node.metrics()); out.synchronize(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java index 0d88fbbf72bd8..de51b3d6e32d6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java @@ -23,6 +23,7 @@ import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cluster.BaselineNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.configuration.PlatformConfiguration; @@ -30,6 +31,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.binary.BinaryRawReaderEx; import org.apache.ignite.internal.binary.BinaryRawWriterEx; +import org.apache.ignite.internal.cluster.DetachedClusterNode; import org.apache.ignite.internal.logger.platform.PlatformLogger; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; @@ -57,6 +59,7 @@ import org.apache.ignite.lang.IgniteFuture; import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -140,6 +143,15 @@ public class PlatformProcessorImpl extends GridProcessorAdapter implements Platf /** */ private static final int OP_ADD_CACHE_CONFIGURATION = 23; + /** */ + private static final int OP_SET_BASELINE_TOPOLOGY_VER = 24; + + /** */ + private static final int OP_SET_BASELINE_TOPOLOGY_NODES = 25; + + /** */ + private static final int OP_GET_BASELINE_TOPOLOGY = 26; + /** Start latch. */ private final CountDownLatch startLatch = new CountDownLatch(1); @@ -408,6 +420,12 @@ private void loggerLog(int level, String message, String category, String errorI return 0; } + + case OP_SET_BASELINE_TOPOLOGY_VER: { + ctx.grid().cluster().setBaselineTopology(val); + + return 0; + } } return PlatformAbstractTarget.throwUnsupported(type); @@ -428,6 +446,22 @@ private void loggerLog(int level, String message, String category, String errorI return 0; } + case OP_SET_BASELINE_TOPOLOGY_NODES: { + int cnt = reader.readInt(); + Collection nodes = new ArrayList<>(cnt); + + for (int i = 0; i < cnt; i++) { + Object consId = reader.readObjectDetached(); + Map attrs = PlatformUtils.readNodeAttributes(reader); + + nodes.add(new DetachedClusterNode(consId, attrs)); + } + + ctx.grid().cluster().setBaselineTopology(nodes); + + return 0; + } + case OP_ADD_CACHE_CONFIGURATION: CacheConfiguration cfg = PlatformConfigurationUtils.readCacheConfiguration(reader); @@ -614,6 +648,18 @@ private void loggerLog(int level, String message, String category, String errorI return; } + + case OP_GET_BASELINE_TOPOLOGY: { + Collection blt = ignite().cluster().currentBaselineTopology(); + writer.writeInt(blt.size()); + + for (BaselineNode n : blt) { + writer.writeObjectDetached(n.consistentId()); + PlatformUtils.writeNodeAttributes(writer, n.attributes()); + } + + return; + } } PlatformAbstractTarget.throwUnsupported(type); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java index ef382d61e0abf..e0fff66d9457e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java @@ -17,10 +17,6 @@ package org.apache.ignite.internal.processors.platform.cluster; -import java.util.ArrayList; -import java.util.Collection; -import java.util.UUID; - import org.apache.ignite.DataRegionMetrics; import org.apache.ignite.DataStorageMetrics; import org.apache.ignite.IgniteCache; @@ -31,9 +27,9 @@ import org.apache.ignite.binary.BinaryRawWriter; import org.apache.ignite.cluster.ClusterMetrics; import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.internal.cluster.ClusterGroupEx; import org.apache.ignite.internal.binary.BinaryRawReaderEx; import org.apache.ignite.internal.binary.BinaryRawWriterEx; +import org.apache.ignite.internal.cluster.ClusterGroupEx; import org.apache.ignite.internal.processors.platform.PlatformAbstractTarget; import org.apache.ignite.internal.processors.platform.PlatformContext; import org.apache.ignite.internal.processors.platform.PlatformTarget; @@ -46,6 +42,10 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.UUID; + /** * Interop projection. */ @@ -344,7 +344,7 @@ public PlatformClusterGroup(PlatformContext platformCtx, ClusterGroupEx prj) { case OP_PING_NODE: return pingNode(reader.readUuid()) ? TRUE : FALSE; - case OP_RESET_LOST_PARTITIONS: + case OP_RESET_LOST_PARTITIONS: { int cnt = reader.readInt(); Collection cacheNames = new ArrayList<>(cnt); @@ -356,6 +356,7 @@ public PlatformClusterGroup(PlatformContext platformCtx, ClusterGroupEx prj) { platformCtx.kernalContext().grid().resetLostPartitions(cacheNames); return TRUE; + } default: return super.processInStreamOutLong(type, reader); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java index aa11dfa595046..b65ca040fb527 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java @@ -1224,6 +1224,47 @@ public IgniteBiTuple read(BinaryRawReaderEx reader) return new BinaryMetadata(typeId, typeName, fields, affKey, schemas, isEnum, enumMap); } + /** + * Writes node attributes. + * + * @param writer Writer. + * @param attrs Attributes. + */ + public static void writeNodeAttributes(BinaryRawWriterEx writer, Map attrs) { + assert writer != null; + assert attrs != null; + + if (attrs != null) { + writer.writeInt(attrs.size()); + + for (Map.Entry e : attrs.entrySet()) { + writer.writeString(e.getKey()); + writer.writeObjectDetached(e.getValue()); + } + } else { + writer.writeInt(0); + } + } + + /** + * Reads node attributes. + * + * @param reader Reader. + * @return Attributes. + */ + public static Map readNodeAttributes(BinaryRawReaderEx reader) { + assert reader != null; + + int attrCnt = reader.readInt(); + Map attrs = new HashMap<>(attrCnt); + + for (int j = 0; j < attrCnt; j++) { + attrs.put(reader.readString(), reader.readObjectDetached()); + } + + return attrs; + } + /** * Private constructor. */ diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.DotNetCore/Common/TestUtils.DotNetCore.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.DotNetCore/Common/TestUtils.DotNetCore.cs index 99f732d784706..08cae9671bd25 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests.DotNetCore/Common/TestUtils.DotNetCore.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests.DotNetCore/Common/TestUtils.DotNetCore.cs @@ -41,7 +41,8 @@ public static IgniteConfiguration GetTestConfiguration(string name = null) Localhost = "127.0.0.1", JvmOptions = TestJavaOptions(), IgniteInstanceName = name, - Logger = TestLogger.Instance + Logger = TestLogger.Instance, + WorkDirectory = WorkDir }; } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj index 22f4ad93ea381..ab16bf1eaf5ec 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj @@ -75,6 +75,7 @@ + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterNodeParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterNodeParityTest.cs new file mode 100644 index 0000000000000..fcf7f89398744 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterNodeParityTest.cs @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + namespace Apache.Ignite.Core.Tests.ApiParity +{ + using Apache.Ignite.Core.Cluster; + using NUnit.Framework; + + /// + /// Tests that has all APIs from Java Ignite interface. + /// + public class ClusterNodeParityTest + { + /** Members that are missing on .NET side and should be added in future. */ + private static readonly string[] MissingMembers = + { + "version" // IGNITE-7101 + }; + + /// + /// Tests the API parity. + /// + [Test] + public void TestClusterNode() + { + ParityTest.CheckInterfaceParity( + @"modules\core\src\main\java\org\apache\ignite\cluster\ClusterNode.java", + typeof(IClusterNode), MissingMembers); + } + } +} \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs index 4b4279fe0ab75..b940d09bc23b7 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs @@ -38,11 +38,7 @@ public class ClusterParityTest /** Members that are missing on .NET side and should be added in future. */ private static readonly string[] MissingMembers = { - "enableStatistics", // IGNITE-7276 - - // IGNITE-7301 - "active", - "setBaselineTopology" + "enableStatistics" // IGNITE-7276 }; /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ParityTest.cs index d30d6d17e49df..c9116af16640e 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ParityTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ParityTest.cs @@ -80,7 +80,7 @@ public static class ParityTest { var path = GetFullPath(javaFilePath); - var dotNetMembers = type.GetMembers() + var dotNetMembers = GetMembers(type) .GroupBy(x => x.Name) .ToDictionary(x => x.Key, x => x.First(), StringComparer.OrdinalIgnoreCase); @@ -90,6 +90,30 @@ public static class ParityTest CheckParity(type, knownMissingMembers, knownMappings, javaMethods, dotNetMembers); } + /// + /// Gets the members. + /// + private static IEnumerable GetMembers(Type type) + { + var types = new Stack(); + types.Push(type); + + while (types.Count > 0) + { + var t = types.Pop(); + + foreach (var m in t.GetMembers()) + { + yield return m; + } + + foreach (var i in t.GetInterfaces()) + { + types.Push(i); + } + } + } + /// /// Gets the full path. /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs index 1bc0218baca2e..d98254d720d28 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs @@ -56,7 +56,7 @@ public void TestDataStorageMetrics() using (var ignite = Ignition.Start(cfg)) { - ignite.SetActive(true); + ignite.GetCluster().SetActive(true); var cache = ignite.CreateCache("c"); @@ -79,8 +79,8 @@ public void TestDataStorageMetrics() Assert.AreEqual(0, metrics.WalArchiveSegments); Assert.AreEqual(0, metrics.WalFsyncTimeAverage); - Assert.AreEqual(89, metrics.LastCheckpointTotalPagesNumber); - Assert.AreEqual(10, metrics.LastCheckpointDataPagesNumber); + Assert.Greater(metrics.LastCheckpointTotalPagesNumber, 26); + Assert.AreEqual(0, metrics.LastCheckpointDataPagesNumber); Assert.AreEqual(0, metrics.LastCheckpointCopiedOnWritePagesNumber); Assert.AreEqual(TimeSpan.Zero, metrics.LastCheckpointLockWaitDuration); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistenceTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistenceTest.cs index 4a60f7b1601da..e3c0acfb75ef9 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistenceTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistenceTest.cs @@ -19,6 +19,7 @@ namespace Apache.Ignite.Core.Tests.Cache { using System; using System.IO; + using System.Linq; using Apache.Ignite.Core.Cache.Configuration; using Apache.Ignite.Core.Common; using Apache.Ignite.Core.Configuration; @@ -33,6 +34,15 @@ public class PersistenceTest /** Temp dir for WAL. */ private readonly string _tempDir = TestUtils.GetTempDirectoryName(); + /// + /// Sets up the test. + /// + [SetUp] + public void SetUp() + { + TestUtils.ClearWorkDir(); + } + /// /// Tears down the test. /// @@ -45,6 +55,8 @@ public void TearDown() { Directory.Delete(_tempDir, true); } + + TestUtils.ClearWorkDir(); } /// @@ -84,7 +96,7 @@ public void TestCacheDataSurvivesNodeRestart() // Start Ignite, put data, stop. using (var ignite = Ignition.Start(cfg)) { - ignite.SetActive(true); + ignite.GetCluster().SetActive(true); // Create cache with default region (persistence enabled), add data. var cache = ignite.CreateCache(cacheName); @@ -110,7 +122,7 @@ public void TestCacheDataSurvivesNodeRestart() // Start Ignite, verify data survival. using (var ignite = Ignition.Start(cfg)) { - ignite.SetActive(true); + ignite.GetCluster().SetActive(true); // Persistent cache already exists and contains data. var cache = ignite.GetCache(cacheName); @@ -127,7 +139,7 @@ public void TestCacheDataSurvivesNodeRestart() // Start Ignite, verify data loss. using (var ignite = Ignition.Start(cfg)) { - ignite.SetActive(true); + ignite.GetCluster().SetActive(true); Assert.IsFalse(ignite.GetCacheNames().Contains(cacheName)); } @@ -149,27 +161,17 @@ private static void CheckDataStorageMetrics(IIgnite ignite) [Test] public void TestGridActivationWithPersistence() { - var cfg = new IgniteConfiguration(TestUtils.GetTestConfiguration()) - { - DataStorageConfiguration = new DataStorageConfiguration - { - DefaultDataRegionConfiguration = new DataRegionConfiguration - { - PersistenceEnabled = true, - Name = "foo" - } - } - }; + var cfg = GetPersistentConfiguration(); // Default config, inactive by default (IsActiveOnStart is ignored when persistence is enabled). using (var ignite = Ignition.Start(cfg)) { CheckIsActive(ignite, false); - ignite.SetActive(true); + ignite.GetCluster().SetActive(true); CheckIsActive(ignite, true); - ignite.SetActive(false); + ignite.GetCluster().SetActive(false); CheckIsActive(ignite, false); } } @@ -187,10 +189,10 @@ public void TestGridActivationNoPersistence() { CheckIsActive(ignite, true); - ignite.SetActive(false); + ignite.GetCluster().SetActive(false); CheckIsActive(ignite, false); - ignite.SetActive(true); + ignite.GetCluster().SetActive(true); CheckIsActive(ignite, true); } @@ -200,20 +202,84 @@ public void TestGridActivationNoPersistence() { CheckIsActive(ignite, false); - ignite.SetActive(true); + ignite.GetCluster().SetActive(true); CheckIsActive(ignite, true); - ignite.SetActive(false); + ignite.GetCluster().SetActive(false); CheckIsActive(ignite, false); } } + /// + /// Tests the baseline topology. + /// + [Test] + public void TestBaselineTopology() + { + var cfg1 = new IgniteConfiguration(GetPersistentConfiguration()) + { + ConsistentId = "node1" + }; + var cfg2 = new IgniteConfiguration(GetPersistentConfiguration()) + { + ConsistentId = "node2", + IgniteInstanceName = "2" + }; + + using (var ignite = Ignition.Start(cfg1)) + { + // Start and stop to bump topology version. + Ignition.Start(cfg2); + Ignition.Stop(cfg2.IgniteInstanceName, true); + + var cluster = ignite.GetCluster(); + Assert.AreEqual(3, cluster.TopologyVersion); + + // Can not set baseline while inactive. + var ex = Assert.Throws(() => cluster.SetBaselineTopology(2)); + Assert.AreEqual("Changing BaselineTopology on inactive cluster is not allowed.", ex.Message); + + // Set with version. + cluster.SetActive(true); + cluster.SetBaselineTopology(2); + + var res = cluster.GetBaselineTopology(); + CollectionAssert.AreEquivalent(new[] {"node1", "node2"}, res.Select(x => x.ConsistentId)); + + cluster.SetBaselineTopology(1); + Assert.AreEqual("node1", cluster.GetBaselineTopology().Single().ConsistentId); + + // Set with nodes. + cluster.SetBaselineTopology(res); + + res = cluster.GetBaselineTopology(); + CollectionAssert.AreEquivalent(new[] { "node1", "node2" }, res.Select(x => x.ConsistentId)); + + cluster.SetBaselineTopology(cluster.GetTopology(1)); + Assert.AreEqual("node1", cluster.GetBaselineTopology().Single().ConsistentId); + + // Set to two nodes. + cluster.SetBaselineTopology(cluster.GetTopology(2)); + } + + // Check auto activation on cluster restart. + using (var ignite = Ignition.Start(cfg1)) + using (Ignition.Start(cfg2)) + { + var cluster = ignite.GetCluster(); + Assert.IsTrue(cluster.IsActive()); + + var res = cluster.GetBaselineTopology(); + CollectionAssert.AreEquivalent(new[] { "node1", "node2" }, res.Select(x => x.ConsistentId)); + } + } + /// /// Checks active state. /// private static void CheckIsActive(IIgnite ignite, bool isActive) { - Assert.AreEqual(isActive, ignite.IsActive()); + Assert.AreEqual(isActive, ignite.GetCluster().IsActive()); if (isActive) { @@ -228,5 +294,23 @@ private static void CheckIsActive(IIgnite ignite, bool isActive) ex.Message.Substring(0, 62)); } } + + /// + /// Gets the persistent configuration. + /// + private static IgniteConfiguration GetPersistentConfiguration() + { + return new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + DataStorageConfiguration = new DataStorageConfiguration + { + DefaultDataRegionConfiguration = new DataRegionConfiguration + { + PersistenceEnabled = true, + Name = "foo" + } + } + }; + } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs index cf0ad40384d5a..d0c576d591149 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Compute/ComputeApiTest.cs @@ -142,9 +142,13 @@ public void TestNodeContent() Assert.IsTrue(node.Addresses.Count > 0); Assert.Throws(() => node.Addresses.Add("addr")); - Assert.NotNull(node.GetAttributes()); - Assert.IsTrue(node.GetAttributes().Count > 0); - Assert.Throws(() => node.GetAttributes().Add("key", "val")); + Assert.NotNull(node.Attributes); + Assert.IsTrue(node.Attributes.Count > 0); + Assert.Throws(() => node.Attributes.Add("key", "val")); + +#pragma warning disable 618 + Assert.AreSame(node.Attributes, node.GetAttributes()); +#pragma warning restore 618 Assert.NotNull(node.HostNames); Assert.Throws(() => node.HostNames.Add("h")); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs index 0bc4e0fdda5a8..734b0cf7d7d51 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs @@ -426,6 +426,7 @@ public void TestConsistentId() using (var ignite = Ignition.Start(cfg)) { Assert.AreEqual(id, ignite.GetConfiguration().ConsistentId); + Assert.AreEqual(id ?? "127.0.0.1:47500", ignite.GetCluster().GetLocalNode().ConsistentId); } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.Common.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.Common.cs index c00ca492fcf07..bf2849e4a29d1 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.Common.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.Common.cs @@ -20,7 +20,9 @@ namespace Apache.Ignite.Core.Tests using System; using System.Collections.Concurrent; using System.Collections.Generic; + using System.IO; using System.Linq; + using System.Reflection; using System.Threading; using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Cluster; @@ -44,6 +46,11 @@ public static partial class TestUtils /** */ private const int DfltBusywaitSleepInterval = 200; + /** Work dir. */ + private static readonly string WorkDir = + // ReSharper disable once AssignNullToNotNullAttribute + Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "ignite_work"); + /** */ private static readonly IList TestJvmOpts = Environment.Is64BitProcess ? new List @@ -349,5 +356,38 @@ public static T SerializeDeserialize(T obj, bool raw = false) return marsh.Unmarshal(marsh.Marshal(obj)); } + + /// + /// Clears the work dir. + /// + public static void ClearWorkDir() + { + if (!Directory.Exists(WorkDir)) + { + return; + } + + // Delete everything we can. Some files may be locked. + foreach (var e in Directory.GetFileSystemEntries(WorkDir, "*", SearchOption.AllDirectories)) + { + try + { + File.Delete(e); + } + catch (Exception) + { + // Ignore + } + + try + { + Directory.Delete(e, true); + } + catch (Exception) + { + // Ignore + } + } + } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.Windows.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.Windows.cs index 2169630c70d20..f8ff874aa2895 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.Windows.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/TestUtils.Windows.cs @@ -78,7 +78,8 @@ public static IgniteConfiguration GetTestConfiguration(bool? jvmDebug = null, st ? DataRegionConfiguration.DefaultMaxSize : 256 * 1024 * 1024 } - } + }, + WorkDirectory = WorkDir }; } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj index f9b22fc0c3f03..adae2b10c249a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj @@ -56,6 +56,7 @@ + @@ -79,6 +80,7 @@ + @@ -90,6 +92,7 @@ + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/IBaselineNode.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/IBaselineNode.cs new file mode 100644 index 0000000000000..d0c5f81c35e56 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/IBaselineNode.cs @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Cluster +{ + using System.Collections.Generic; + + /// + /// Baseline topology node (see ). + /// + public interface IBaselineNode + { + /// + /// Gets the consistent ID from . + /// + object ConsistentId { get; } + + /// + /// Gets all node attributes. Attributes are assigned to nodes at startup. + /// + IDictionary Attributes { get; } + } +} \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/ICluster.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/ICluster.cs index 812a644276767..f9bbdf796fa64 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/ICluster.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/ICluster.cs @@ -88,6 +88,38 @@ public interface ICluster : IClusterGroup /// /// The reconnect task. /// - Task ClientReconnectTask { get; } + Task ClientReconnectTask { get; } + + /// + /// Changes Ignite grid state to active or inactive. + /// + void SetActive(bool isActive); + + /// + /// Determines whether this grid is in active state. + /// + /// + /// true if the grid is active; otherwise, false. + /// + bool IsActive(); + + /// + /// Sets the baseline topology from the cluster topology of the given version. + /// This method requires active cluster (). + /// + /// The topology version. + void SetBaselineTopology(long topologyVersion); + + /// + /// Sets the baseline topology nodes. + /// + /// The nodes. + void SetBaselineTopology(IEnumerable nodes); + + /// + /// Gets the baseline topology. + /// Returns null if has not been called. + /// + ICollection GetBaselineTopology(); } } \ No newline at end of file diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/IClusterNode.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/IClusterNode.cs index 5f14116c93f9a..ec22d1880d08f 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/IClusterNode.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/IClusterNode.cs @@ -33,7 +33,7 @@ namespace Apache.Ignite.Core.Cluster /// /// All members are thread-safe and may be used concurrently from multiple threads. /// - public interface IClusterNode + public interface IClusterNode : IBaselineNode { /// /// Globally unique node ID. A new ID is generated every time a node restarts. @@ -66,6 +66,7 @@ public interface IClusterNode /// /// All node attributes. [SuppressMessage("Microsoft.Design", "CA1024:UsePropertiesWhereAppropriate", Justification = "Semantics.")] + [Obsolete("Use Attributes property.")] IDictionary GetAttributes(); /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IIgnite.cs b/modules/platforms/dotnet/Apache.Ignite.Core/IIgnite.cs index 035d4b580c8d9..52323c8efe497 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IIgnite.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IIgnite.cs @@ -357,6 +357,7 @@ public interface IIgnite : IDisposable /// /// Changes Ignite grid state to active or inactive. /// + [Obsolete("Use GetCluster().SetActive instead.")] void SetActive(bool isActive); /// @@ -365,6 +366,7 @@ public interface IIgnite : IDisposable /// /// true if the grid is active; otherwise, false. /// + [Obsolete("Use GetCluster().IsActive instead.")] bool IsActive(); /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs index f63d959cd1e48..0ddf9a8d287b4 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs @@ -1095,7 +1095,7 @@ public bool IsDaemon /// /// Gets or sets the user attributes for this node. /// - /// These attributes can be retrieved later via . + /// These attributes can be retrieved later via . /// Environment variables are added to node attributes automatically. /// NOTE: attribute names starting with "org.apache.ignite" are reserved for internal use. /// @@ -1377,7 +1377,7 @@ public TimeSpan LongQueryWarningTimeout /// /// Gets or sets a value indicating whether grid should be active on start. - /// See also and . + /// See also and . /// /// This property is ignored when is present: /// cluster is always inactive on start when Ignite Persistence is enabled. diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinarySystemHandlers.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinarySystemHandlers.cs index 430b426c25ff0..cae0f5fcf9a50 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinarySystemHandlers.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinarySystemHandlers.cs @@ -113,6 +113,9 @@ static BinarySystemHandlers() // 14. Enum. Should be read as Array, see WriteEnumArray implementation. ReadHandlers[BinaryTypeId.ArrayEnum] = new BinarySystemReader(ReadArray); + + // 15. Optimized marshaller objects. + ReadHandlers[BinaryTypeId.OptimizedMarshaller] = new BinarySystemReader(ReadOptimizedMarshallerObject); } /// @@ -229,6 +232,11 @@ private static IBinarySystemWriteHandler FindWriteHandler(Type type) return new BinarySystemWriteHandler(WriteArray, true); } + if (type == typeof(OptimizedMarshallerObject)) + { + return new BinarySystemWriteHandler((w, o) => o.Write(w.Stream), false); + } + return null; } @@ -533,6 +541,14 @@ private static void WriteIgnite(BinaryWriter ctx, object obj) ctx.Stream.WriteByte(BinaryUtils.HdrNull); } + /// + /// Reads the optimized marshaller object. + /// + private static object ReadOptimizedMarshallerObject(BinaryReader ctx, Type type) + { + return new OptimizedMarshallerObject(ctx.Stream); + } + /** * Read delegate. * Read context. diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs index 1d3d9c48497a9..8067a39126fd7 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryTypeId.cs @@ -164,6 +164,9 @@ internal static class BinaryTypeId /** Type: platform object proxy. */ public const byte PlatformJavaObjectFactoryProxy = 99; + /** Type: object written with Java OptimizedMarshaller. */ + public const byte OptimizedMarshaller = 254; + /** Type: platform object proxy. */ public const int IgniteUuid = 2018070327; diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/OptimizedMarshallerObject.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/OptimizedMarshallerObject.cs new file mode 100644 index 0000000000000..5d2ec0591311a --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/OptimizedMarshallerObject.cs @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl.Binary +{ + using System.Diagnostics; + using Apache.Ignite.Core.Impl.Binary.IO; + + /// + /// Object written with Java OptimizedMarshaller. + /// We just hold it as a byte array. + /// + internal class OptimizedMarshallerObject + { + /** */ + private readonly byte[] _data; + + /// + /// Initializes a new instance of the class. + /// + public OptimizedMarshallerObject(IBinaryStream stream) + { + Debug.Assert(stream != null); + + _data = stream.ReadByteArray(stream.ReadInt()); + } + + /// + /// Writes to the specified writer. + /// + public void Write(IBinaryStream stream) + { + Debug.Assert(stream != null); + + stream.WriteByte(BinaryTypeId.OptimizedMarshaller); + stream.WriteInt(_data.Length); + stream.WriteByteArray(_data); + } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/BaselineNode.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/BaselineNode.cs new file mode 100644 index 0000000000000..23ec7242912c5 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/BaselineNode.cs @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl.Cluster +{ + using System.Collections.Generic; + using System.Diagnostics; + using Apache.Ignite.Core.Binary; + using Apache.Ignite.Core.Cluster; + using Apache.Ignite.Core.Impl.Binary; + + /// + /// Baseline node. + /// + internal class BaselineNode : IBaselineNode + { + /** Attributes. */ + private readonly IDictionary _attributes; + + /** Consistent ID. */ + private readonly object _consistentId; + + /// + /// Initializes a new instance of the class. + /// + /// The reader. + public BaselineNode(IBinaryRawReader reader) + { + Debug.Assert(reader != null); + + _consistentId = reader.ReadObject(); + _attributes = ClusterNodeImpl.ReadAttributes(reader); + } + + /** */ + public object ConsistentId + { + get { return _consistentId; } + } + + /** */ + public IDictionary Attributes + { + get { return _attributes; } + } + + /// + /// Writes this instance to specified writer. + /// + public static void Write(BinaryWriter writer, IBaselineNode node) + { + Debug.Assert(writer != null); + Debug.Assert(node != null); + + writer.WriteObjectDetached(node.ConsistentId); + + var attrs = node.Attributes; + + if (attrs != null) + { + writer.WriteInt(attrs.Count); + + foreach (var attr in attrs) + { + writer.WriteString(attr.Key); + writer.WriteObjectDetached(attr.Value); + } + } + else + { + writer.WriteInt(0); + } + } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterNodeImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterNodeImpl.cs index b6583532e84b7..fbf47642c7a20 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterNodeImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterNodeImpl.cs @@ -55,6 +55,9 @@ internal class ClusterNodeImpl : IClusterNode /** Client flag. */ private readonly bool _isClient; + /** Consistent id. */ + private readonly object _consistentId; + /** Metrics. */ private volatile ClusterMetricsImpl _metrics; @@ -73,13 +76,14 @@ public ClusterNodeImpl(IBinaryRawReader reader) _id = id.Value; - _attrs = reader.ReadDictionaryAsGeneric().AsReadOnly(); + _attrs = ReadAttributes(reader); _addrs = reader.ReadCollectionAsList().AsReadOnly(); _hosts = reader.ReadCollectionAsList().AsReadOnly(); _order = reader.ReadLong(); _isLocal = reader.ReadBoolean(); _isDaemon = reader.ReadBoolean(); _isClient = reader.ReadBoolean(); + _consistentId = reader.ReadObject(); _metrics = reader.ReadBoolean() ? new ClusterMetricsImpl(reader) : null; } @@ -119,7 +123,7 @@ public bool TryGetAttribute(string name, out T attr) /** */ public IDictionary GetAttributes() { - return _attrs.AsReadOnly(); + return _attrs; } /** */ @@ -180,6 +184,18 @@ public IClusterMetrics GetMetrics() return oldMetrics; } + /** */ + public object ConsistentId + { + get { return _consistentId; } + } + + /** */ + public IDictionary Attributes + { + get { return _attrs; } + } + /** */ public bool IsClient { @@ -218,5 +234,23 @@ internal void Init(Ignite grid) { _igniteRef = new WeakReference(grid); } + + /// + /// Reads the attributes. + /// + internal static IDictionary ReadAttributes(IBinaryRawReader reader) + { + Debug.Assert(reader != null); + + var count = reader.ReadInt(); + var res = new Dictionary(count); + + for (var i = 0; i < count; i++) + { + res[reader.ReadString()] = reader.ReadObject(); + } + + return res.AsReadOnly(); + } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs index eb7f6275bd248..10b083ba1aba9 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs @@ -84,7 +84,10 @@ private enum Op LoggerLog = 20, GetBinaryProcessor = 21, ReleaseStart = 22, - AddCacheConfiguration = 23 + AddCacheConfiguration = 23, + SetBaselineTopologyVersion = 24, + SetBaselineTopologyNodes = 25, + GetBaselineTopology = 26 } /** */ @@ -780,6 +783,40 @@ public bool IsActive() return _prj.IsActive(); } + /** */ + public void SetBaselineTopology(long topologyVersion) + { + DoOutInOp((int) Op.SetBaselineTopologyVersion, topologyVersion); + } + + /** */ + public void SetBaselineTopology(IEnumerable nodes) + { + IgniteArgumentCheck.NotNull(nodes, "nodes"); + + DoOutOp((int) Op.SetBaselineTopologyNodes, w => + { + var pos = w.Stream.Position; + w.WriteInt(0); + var cnt = 0; + + foreach (var node in nodes) + { + cnt++; + BaselineNode.Write(w, node); + } + + w.Stream.WriteInt(pos, cnt); + }); + } + + /** */ + public ICollection GetBaselineTopology() + { + return DoInOp((int) Op.GetBaselineTopology, + s => Marshaller.StartUnmarshal(s).ReadCollectionRaw(r => (IBaselineNode) new BaselineNode(r))); + } + /** */ #pragma warning disable 618 public IPersistentStoreMetrics GetPersistentStoreMetrics() diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Env.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Env.cs index f3632ce0dd55f..2f0a2719bb7cb 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Env.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Env.cs @@ -19,11 +19,9 @@ namespace Apache.Ignite.Core.Impl.Unmanaged.Jni { using System; using System.Diagnostics; - using System.Reflection; using System.Runtime.InteropServices; using System.Security; using Apache.Ignite.Core.Common; - using Apache.Ignite.Core.Impl.Common; /// /// JNIEnv. From 268481c1cf7fe57df24be130eb67c3e3a13afe01 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Wed, 17 Jan 2018 16:50:34 +0300 Subject: [PATCH 03/65] IGNITE-7453 Use GridUnsafe.cleanDirectBuffer in WalStat --- .../main/java/org/apache/ignite/development/utils/WalStat.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java index 36933d8aa89df..a09c9a5b4f986 100644 --- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java +++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java @@ -249,8 +249,9 @@ private static String getPageType(PageSnapshot record) { catch (IgniteCheckedException ignored) { } finally { - ((DirectBuffer)buf).cleaner().clean(); + GridUnsafe.cleanDirectBuffer(buf); } + return ""; } From db0cd105719c8ae713b13b34d9dca0a8cd45d377 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 17 Jan 2018 17:05:25 +0300 Subject: [PATCH 04/65] IGNITE-6776 .NET: Thin client: Add SQL & LINQ example This closes #3390 --- .../Examples/ExamplesTest.cs | 4 +- .../Apache.Ignite.Examples.csproj | 1 + .../ThinClient/ThinClientSqlExample.cs | 222 ++++++++++++++++++ 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 modules/platforms/dotnet/examples/Apache.Ignite.Examples/ThinClient/ThinClientSqlExample.cs diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Examples/ExamplesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Examples/ExamplesTest.cs index fcae595f4a243..8483f3e596cc1 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Examples/ExamplesTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Examples/ExamplesTest.cs @@ -51,7 +51,7 @@ public class ExamplesTest private static readonly Type[] RemoteOnlyExamples = { typeof(PeerAssemblyLoadingExample), typeof(MessagingExample), typeof(NearCacheExample), - typeof(ThinClientPutGetExample), typeof(ThinClientQueryExample) + typeof(ThinClientPutGetExample), typeof(ThinClientQueryExample), typeof(ThinClientSqlExample) }; /** */ @@ -59,7 +59,7 @@ public class ExamplesTest { typeof(BinaryModeExample), typeof(NearCacheExample), typeof(PeerAssemblyLoadingExample), typeof(ThinClientPutGetExample), typeof(SqlExample), typeof(LinqExample), typeof(SqlDmlExample), - typeof(SqlDdlExample) + typeof(SqlDdlExample), typeof(ThinClientSqlExample) }; /** Config file path. */ diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Apache.Ignite.Examples.csproj b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Apache.Ignite.Examples.csproj index 51ac5ad87b156..08390d76e9fb9 100644 --- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Apache.Ignite.Examples.csproj +++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Apache.Ignite.Examples.csproj @@ -83,6 +83,7 @@ + diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/ThinClient/ThinClientSqlExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/ThinClient/ThinClientSqlExample.cs new file mode 100644 index 0000000000000..b29ccd0658ef8 --- /dev/null +++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/ThinClient/ThinClientSqlExample.cs @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Examples.ThinClient +{ + using System; + using System.Linq; + using Apache.Ignite.Core; + using Apache.Ignite.Core.Cache; + using Apache.Ignite.Core.Cache.Configuration; + using Apache.Ignite.Core.Cache.Query; + using Apache.Ignite.Core.Client; + using Apache.Ignite.Core.Client.Cache; + using Apache.Ignite.ExamplesDll.Binary; + using Apache.Ignite.Linq; + + /// + /// Demonstrates Ignite.NET "thin" client SQL queries. + /// + /// 1) Set this class as startup object (Apache.Ignite.Examples project -> right-click -> Properties -> + /// Application -> Startup object); + /// 2) Start example (F5 or Ctrl+F5). + /// + /// This example must be run with standalone Apache Ignite node: + /// 1) Run %IGNITE_HOME%/platforms/dotnet/bin/Apache.Ignite.exe: + /// Apache.Ignite.exe -configFileName=platforms\dotnet\examples\apache.ignite.examples\app.config + /// 2) Start example. + /// + /// This example can also work via pure Java node started with ignite.bat/ignite.sh. + /// + public class ThinClientSqlExample + { + /// Cache name. + private const string CacheName = "thinClientSqlCache"; + + [STAThread] + public static void Main() + { + var cfg = new IgniteClientConfiguration + { + Host = "127.0.0.1" + }; + + using (IIgniteClient igniteClient = Ignition.StartClient(cfg)) + { + Console.WriteLine(); + Console.WriteLine(">>> Cache query client example started."); + + // Configure query entities to enable SQL. + var cacheCfg = new CacheClientConfiguration + { + Name = CacheName, + QueryEntities = new[] + { + new QueryEntity(typeof(int), typeof(Employee)), + } + }; + + ICacheClient cache = igniteClient.GetOrCreateCache(cacheCfg); + + // Populate cache with sample data entries. + PopulateCache(cache); + + // Run SQL example. + SqlQueryExample(cache); + LinqExample(cache); + + // Run SQL fields query example. + SqlFieldsQueryExample(cache); + LinqFieldsExample(cache); + } + + Console.WriteLine(); + Console.WriteLine(">>> Example finished, press any key to exit ..."); + Console.ReadKey(); + } + + /// + /// Queries employees that have provided ZIP code in address. + /// + /// Cache. + private static void SqlQueryExample(ICacheClient cache) + { + const int zip = 94109; + + var qry = cache.Query(new SqlQuery(typeof(Employee), "zip = ?", zip)); + + Console.WriteLine(); + Console.WriteLine(">>> Employees with zipcode {0} (SQL):", zip); + + foreach (var entry in qry) + Console.WriteLine(">>> " + entry.Value); + } + + /// + /// Queries employees that have provided ZIP code in address. + /// + /// Cache. + private static void LinqExample(ICacheClient cache) + { + const int zip = 94109; + + IQueryable> qry = + cache.AsCacheQueryable().Where(emp => emp.Value.Address.Zip == zip); + + Console.WriteLine(); + Console.WriteLine(">>> Employees with zipcode " + zip + " (LINQ):"); + + foreach (ICacheEntry entry in qry) + Console.WriteLine(">>> " + entry.Value); + + Console.WriteLine(); + Console.WriteLine(">>> Generated SQL:"); + Console.WriteLine(">>> " + qry.ToCacheQueryable().GetFieldsQuery().Sql); + } + + + /// + /// Queries names and salaries for all employees. + /// + /// Cache. + private static void SqlFieldsQueryExample(ICacheClient cache) + { + var qry = cache.Query(new SqlFieldsQuery("select name, salary from Employee")); + + Console.WriteLine(); + Console.WriteLine(">>> Employee names and their salaries (SQL):"); + + foreach (var row in qry) + Console.WriteLine(">>> [Name=" + row[0] + ", salary=" + row[1] + ']'); + } + + /// + /// Queries names and salaries for all employees. + /// + /// Cache. + private static void LinqFieldsExample(ICacheClient cache) + { + var qry = cache.AsCacheQueryable().Select(entry => new {entry.Value.Name, entry.Value.Salary}); + + Console.WriteLine(); + Console.WriteLine(">>> Employee names and their salaries (LINQ):"); + + foreach (var row in qry) + Console.WriteLine(">>> [Name=" + row.Name + ", salary=" + row.Salary + ']'); + + Console.WriteLine(); + Console.WriteLine(">>> Generated SQL:"); + Console.WriteLine(">>> " + qry.ToCacheQueryable().GetFieldsQuery().Sql); + } + + /// + /// Populate cache with data for this example. + /// + /// Cache. + private static void PopulateCache(ICacheClient cache) + { + cache.Put(1, new Employee( + "James Wilson", + 12500, + new Address("1096 Eddy Street, San Francisco, CA", 94109), + new[] { "Human Resources", "Customer Service" }, + 1)); + + cache.Put(2, new Employee( + "Daniel Adams", + 11000, + new Address("184 Fidler Drive, San Antonio, TX", 78130), + new[] { "Development", "QA" }, + 1)); + + cache.Put(3, new Employee( + "Cristian Moss", + 12500, + new Address("667 Jerry Dove Drive, Florence, SC", 29501), + new[] { "Logistics" }, + 1)); + + cache.Put(4, new Employee( + "Allison Mathis", + 25300, + new Address("2702 Freedom Lane, San Francisco, CA", 94109), + new[] { "Development" }, + 2)); + + cache.Put(5, new Employee( + "Breana Robbin", + 6500, + new Address("3960 Sundown Lane, Austin, TX", 78130), + new[] { "Sales" }, + 2)); + + cache.Put(6, new Employee( + "Philip Horsley", + 19800, + new Address("2803 Elsie Drive, Sioux Falls, SD", 57104), + new[] { "Sales" }, + 2)); + + cache.Put(7, new Employee( + "Brian Peters", + 10600, + new Address("1407 Pearlman Avenue, Boston, MA", 12110), + new[] { "Development", "QA" }, + 2)); + } + } +} \ No newline at end of file From c214db879101aa5660e2a50b11cd20964c0bc114 Mon Sep 17 00:00:00 2001 From: Andrey Gura Date: Wed, 17 Jan 2018 15:42:41 +0300 Subject: [PATCH 05/65] ignite-7450 FileWriteAheadLogManager always uses RandomAccessFileIOFactory now --- .../configuration/DataStorageConfiguration.java | 2 +- .../wal/FileWriteAheadLogManager.java | 10 ++++++++-- .../db/wal/IgniteWalFlushFailoverTest.java | 8 ++++---- ...lFlushMultiNodeFailoverAbstractSelfTest.java | 17 ++++++++++++----- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java index f1439d426b26f..30507fec5e025 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java @@ -226,7 +226,7 @@ public class DataStorageConfiguration implements Serializable { /** Factory to provide I/O interface for files */ private FileIOFactory fileIOFactory = - IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_ASYNC_FILE_IO_FACTORY, false) ? + IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_ASYNC_FILE_IO_FACTORY, true) ? new AsyncFileIOFactory() : new RandomAccessFileIOFactory(); /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index d29774cc4140f..5ae0226b9b451 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -85,6 +85,7 @@ import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; @@ -267,7 +268,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl private volatile long lastTruncatedArchiveIdx = -1L; /** Factory to provide I/O interfaces for read/write operations with files */ - private final FileIOFactory ioFactory; + private FileIOFactory ioFactory; /** Next segment archived monitor. */ private final Object nextSegmentArchivedMonitor = new Object(); @@ -338,11 +339,16 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) { flushFreq = dsCfg.getWalFlushFrequency(); fsyncDelay = dsCfg.getWalFsyncDelayNanos(); alwaysWriteFullPages = dsCfg.isAlwaysWriteFullPages(); - ioFactory = dsCfg.getFileIOFactory(); + ioFactory = new RandomAccessFileIOFactory(); walAutoArchiveAfterInactivity = dsCfg.getWalAutoArchiveAfterInactivity(); evt = ctx.event(); } + /** For test purposes only. */ + public void setFileIOFactory(FileIOFactory ioFactory) { + this.ioFactory = ioFactory; + } + /** {@inheritDoc} */ @Override public void start0() throws IgniteCheckedException { if (!cctx.kernalContext().clientNode()) { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java index 2a3fc1fd0078e..386b83ca44cf2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java @@ -33,11 +33,11 @@ import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.GridKernalState; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; @@ -93,7 +93,6 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest { DataStorageConfiguration memCfg = new DataStorageConfiguration() .setDefaultDataRegionConfiguration( new DataRegionConfiguration().setMaxSize(2048L * 1024 * 1024).setPersistenceEnabled(true)) - .setFileIOFactory(new FailingFileIOFactory(canFail)) .setWalMode(WALMode.BACKGROUND) .setWalBufferSize(128 * 1024)// Setting WAL Segment size to high values forces flushing by timeout. .setWalSegmentSize(flushByTimeout ? 500_000 : 50_000); @@ -129,13 +128,15 @@ public void testErrorOnDirectFlush() throws Exception { private void flushingErrorTest() throws Exception { final IgniteEx grid = startGrid(0); - IgniteWriteAheadLogManager wal = grid.context().cache().context().wal(); + FileWriteAheadLogManager wal = (FileWriteAheadLogManager)grid.context().cache().context().wal(); boolean mmap = GridTestUtils.getFieldValue(wal, "mmap"); if (mmap) return; + wal.setFileIOFactory(new FailingFileIOFactory(canFail)); + try { grid.active(true); @@ -217,7 +218,6 @@ private static class FailingFileIOFactory implements FileIOFactory { @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException { return delegate.map(maxWalSegmentSize); } - }; } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java index 76c7851c0f3a2..9b110df0f62d0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java @@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; @@ -107,10 +108,8 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr .setDefaultDataRegionConfiguration( new DataRegionConfiguration().setMaxSize(2048L * 1024 * 1024).setPersistenceEnabled(true)) .setWalMode(this.walMode()) - .setWalSegmentSize(50_000); - - if (gridName.endsWith(String.valueOf(gridCount()))) - memCfg.setFileIOFactory(new FailingFileIOFactory(canFail)); + .setWalSegmentSize(50_000) + .setWalBufferSize(50_000); cfg.setDataStorageConfiguration(memCfg); @@ -174,6 +173,10 @@ public void failWhilePut(boolean failWhileStart) throws Exception { startGrid(gridCount()); + FileWriteAheadLogManager wal0 = (FileWriteAheadLogManager)grid(gridCount()).context().cache().context().wal(); + + wal0.setFileIOFactory(new FailingFileIOFactory(canFail)); + grid.cluster().setBaselineTopology(grid.cluster().topologyVersion()); waitForRebalancing(); @@ -200,6 +203,10 @@ public void failWhilePut(boolean failWhileStart) throws Exception { Ignite grid0 = startGrids(gridCount() + 1); + FileWriteAheadLogManager wal0 = (FileWriteAheadLogManager)grid(gridCount()).context().cache().context().wal(); + + wal0.setFileIOFactory(new FailingFileIOFactory(canFail)); + grid0.active(true); cache = grid0.cache(TEST_CACHE); @@ -247,7 +254,7 @@ private static class FailingFileIOFactory implements FileIOFactory { int writeAttempts = 2; @Override public int write(ByteBuffer srcBuf) throws IOException { - if (--writeAttempts <= 0 && fail!= null && fail.get()) + if (--writeAttempts <= 0 && fail != null && fail.get()) throw new IOException("No space left on device"); return super.write(srcBuf); From 75c27d5e49d7458e46eb46e6f87a445c3f1320ea Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 18 Jan 2018 09:25:19 +0700 Subject: [PATCH 06/65] IGNITE-7274 Web Console: Support multiple statements on Queries screen. (cherry picked from commit 1926783) --- .../cache/GatewayProtectedCacheProxy.java | 14 +++ .../processors/cache/IgniteCacheProxy.java | 41 ++++++--- .../cache/IgniteCacheProxyImpl.java | 86 ++++++++++++------- .../processors/query/GridQueryProcessor.java | 40 ++++++--- .../internal/visor/query/VisorQueryTask.java | 14 ++- .../processors/query/h2/IgniteH2Indexing.java | 3 + .../query/h2/sql/GridSqlQueryParser.java | 21 ++++- .../GridCacheCrossCacheQuerySelfTest.java | 78 ++++++++++++++--- .../cache/index/H2DynamicTableSelfTest.java | 6 +- 9 files changed, 226 insertions(+), 77 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java index 564b18ab2b571..27fc39563e36a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java @@ -386,6 +386,20 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) { } } + /** {@inheritDoc} */ + @Override public List>> queryMultipleStatements(SqlFieldsQuery qry) { + GridCacheGateway gate = gate(); + + CacheOperationContext prev = onEnter(gate, opCtx); + + try { + return delegate.queryMultipleStatements(qry); + } + finally { + onLeave(gate, prev); + } + } + /** {@inheritDoc} */ @Override public QueryCursor query(Query qry, IgniteClosure transformer) { GridCacheGateway gate = gate(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index 361764e2e8a2e..c33312539d4e5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -19,11 +19,16 @@ import java.io.Externalizable; import java.util.Date; +import java.util.List; import java.util.UUID; import org.apache.ignite.IgniteCache; -import org.apache.ignite.lang.IgniteAsyncSupport; +import org.apache.ignite.cache.query.FieldsQueryCursor; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.lang.IgniteFuture; +/** + * Cache proxy. + */ public interface IgniteCacheProxy extends IgniteCache, Externalizable { /** * @return Context. @@ -39,15 +44,22 @@ public interface IgniteCacheProxy extends IgniteCache, Externalizabl public IgniteCacheProxy cacheNoGate(); /** - * Creates projection that will operate with binary objects.

Projection returned by this method will force - * cache not to deserialize binary objects, so keys and values will be returned from cache API methods without - * changes. Therefore, signature of the projection can contain only following types:

  • {@code BinaryObject} - * for binary classes
  • All primitives (byte, int, ...) and there boxed versions (Byte, Integer, ...)
    • - *
  • Arrays of primitives (byte[], int[], ...)
  • {@link String} and array of {@link String}s
    • - *
  • {@link UUID} and array of {@link UUID}s
  • {@link Date} and array of {@link Date}s
  • {@link - * java.sql.Timestamp} and array of {@link java.sql.Timestamp}s
  • Enums and array of enums
  • Maps, - * collections and array of objects (but objects inside them will still be converted if they are binary)
    • - *

For example, if you use {@link Integer} as a key and {@code Value} class as a value (which will be + * Creates projection that will operate with binary objects. + *

Projection returned by this method will force cache not to deserialize binary objects, + * so keys and values will be returned from cache API methods without changes. + * Therefore, signature of the projection can contain only following types: + *

    + *
  • {@code BinaryObject} for binary classes
    • + *
  • All primitives (byte, int, ...) and there boxed versions (Byte, Integer, ...)
    • + *
  • Arrays of primitives (byte[], int[], ...)
    • + *
  • {@link String} and array of {@link String}s
    • + *
  • {@link UUID} and array of {@link UUID}s
    • + *
  • {@link Date} and array of {@link Date}s
    • + *
  • {@link java.sql.Timestamp} and array of {@link java.sql.Timestamp}s
    • + *
  • Enums and array of enums
    • + *
  • Maps, collections and array of objects (but objects inside them will still be converted if they are binary)
    • + *
+ *

For example, if you use {@link Integer} as a key and {@code Value} class as a value (which will be * stored in binary format), you should acquire following projection to avoid deserialization: * 

      * IgniteInternalCache prj = cache.keepBinary();
@@ -100,4 +112,13 @@ public interface IgniteCacheProxy extends IgniteCache, Externalizabl
      * @return Future that contains cache close operation.
      */
     public IgniteFuture closeAsync();
+
+    /**
+     * Queries cache with multiple statements. Accepts {@link SqlFieldsQuery} class.
+     *
+     * @param qry SqlFieldsQuery.
+     * @return List of cursors.
+     * @see SqlFieldsQuery
+     */
+    public List>> queryMultipleStatements(SqlFieldsQuery qry);
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index 4c638a9b22d81..1cbd2b0909343 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -171,8 +171,7 @@ private IgniteCacheProxyImpl(
     /**
      * @return Context.
      */
-    @Override
-    public GridCacheContext context() {
+    @Override public GridCacheContext context() {
         return ctx;
     }
 
@@ -279,8 +278,8 @@ public IgniteCacheProxy gatewayWrapper() {
         try {
             if (ctx.cache().isLocal())
                 return (IgniteFuture)createFuture(ctx.cache().localLoadCacheAsync(p, args));
-            else
-                return (IgniteFuture)createFuture(ctx.cache().globalLoadCacheAsync(p, args));
+
+            return (IgniteFuture)createFuture(ctx.cache().globalLoadCacheAsync(p, args));
         }
         catch (IgniteCheckedException | IgniteException e) {
             throw cacheException(e);
@@ -356,15 +355,13 @@ private  QueryCursor query(
         @Nullable ClusterGroup grp)
         throws IgniteCheckedException {
 
-        final CacheQuery qry;
-
         CacheOperationContext opCtxCall = ctx.operationContextPerCall();
 
         boolean isKeepBinary = opCtxCall != null && opCtxCall.isKeepBinary();
 
         IgniteBiPredicate p = scanQry.getFilter();
 
-        qry = ctx.queries().createScanQuery(p, transformer, scanQry.getPartition(), isKeepBinary);
+        final CacheQuery qry = ctx.queries().createScanQuery(p, transformer, scanQry.getPartition(), isKeepBinary);
 
         if (scanQry.getPageSize() > 0)
             qry.pageSize(scanQry.getPageSize());
@@ -560,6 +557,30 @@ private QueryCursor> queryContinuous(ContinuousQuery qry, bool
         return (FieldsQueryCursor>)query((Query)qry);
     }
 
+    /** {@inheritDoc} */
+    @Override public List>> queryMultipleStatements(SqlFieldsQuery qry) {
+        A.notNull(qry, "qry");
+        try {
+            ctx.checkSecurity(SecurityPermission.CACHE_READ);
+
+            validate(qry);
+
+            convertToBinary(qry);
+
+            CacheOperationContext opCtxCall = ctx.operationContextPerCall();
+
+            boolean keepBinary = opCtxCall != null && opCtxCall.isKeepBinary();
+
+            return ctx.kernalContext().query().querySqlFields(ctx, qry, keepBinary, false);
+        }
+        catch (Exception e) {
+            if (e instanceof CacheException)
+                throw (CacheException)e;
+
+            throw new CacheException(e);
+        }
+    }
+
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public  QueryCursor query(Query qry) {
@@ -1582,15 +1603,22 @@ else if (clazz.isAssignableFrom(IgniteEx.class))
     }
 
     /**
-     * Creates projection that will operate with binary objects. 
 Projection returned by this method will force
-     * cache not to deserialize binary objects, so keys and values will be returned from cache API methods without
-     * changes. Therefore, signature of the projection can contain only following types: 
     
  • {@code BinaryObject}
-     * for binary classes
    •  
  • All primitives (byte, int, ...) and there boxed versions (Byte, Integer, ...)
    • 
-     * 
  • Arrays of primitives (byte[], int[], ...)
    •  
  • {@link String} and array of {@link String}s
    • 
-     * 
  • {@link UUID} and array of {@link UUID}s
    •  
  • {@link Date} and array of {@link Date}s
    •  
  • {@link
-     * java.sql.Timestamp} and array of {@link java.sql.Timestamp}s
    •  
  • Enums and array of enums
    •  
  •  Maps,
-     * collections and array of objects (but objects inside them will still be converted if they are binary) 
    • 
-     * 
 
 For example, if you use {@link Integer} as a key and {@code Value} class as a value (which will be
+     * Creates projection that will operate with binary objects.
+     * 
 Projection returned by this method will force cache not to deserialize binary objects,
+     * so keys and values will be returned from cache API methods without changes.
+     * Therefore, signature of the projection can contain only following types:
+     * 
    
+     *     
  • {@code BinaryObject} for binary classes
    • 
+     *     
  • All primitives (byte, int, ...) and there boxed versions (Byte, Integer, ...)
    • 
+     *     
  • Arrays of primitives (byte[], int[], ...)
    • 
+     *     
  • {@link String} and array of {@link String}s
    • 
+     *     
  • {@link UUID} and array of {@link UUID}s
    • 
+     *     
  • {@link Date} and array of {@link Date}s
    • 
+     *     
  • {@link java.sql.Timestamp} and array of {@link java.sql.Timestamp}s
    • 
+     *     
  • Enums and array of enums
    • 
+     *     
  •  Maps, collections and array of objects (but objects inside them will still be converted if they are binary) 
    • 
+     * 

+     * 
 For example, if you use {@link Integer} as a key and {@code Value} class as a value (which will be
      * stored in binary format), you should acquire following projection to avoid deserialization:
      * 
      * IgniteInternalCache prj = cache.keepBinary();
@@ -1604,9 +1632,8 @@ else if (clazz.isAssignableFrom(IgniteEx.class))
      *
      * @return Projection for binary objects.
      */
-    @Override
     @SuppressWarnings("unchecked")
-    public  IgniteCache keepBinary() {
+    @Override public  IgniteCache keepBinary() {
         throw new UnsupportedOperationException();
     }
 
@@ -1614,17 +1641,15 @@ public  IgniteCache keepBinary() {
      * @param dataCenterId Data center ID.
      * @return Projection for data center id.
      */
-    @Override
     @SuppressWarnings("unchecked")
-    public IgniteCache withDataCenterId(byte dataCenterId) {
+    @Override public IgniteCache withDataCenterId(byte dataCenterId) {
         throw new UnsupportedOperationException();
     }
 
     /**
      * @return Cache with skip store enabled.
      */
-    @Override
-    public IgniteCache skipStore() {
+    @Override public IgniteCache skipStore() {
         throw new UnsupportedOperationException();
     }
 
@@ -1671,8 +1696,7 @@ private  void setFuture(IgniteInternalFuture fut) {
     /**
      * @return Internal proxy.
      */
-    @Override
-    public GridCacheProxyImpl internalProxy() {
+    @Override public GridCacheProxyImpl internalProxy() {
         return new GridCacheProxyImpl<>(ctx, delegate, ctx.operationContextPerCall());
     }
 
@@ -1757,17 +1781,17 @@ public boolean isRestarting() {
     public void restart() {
         GridFutureAdapter restartFut = new GridFutureAdapter<>();
 
-        final GridFutureAdapter currentFut = this.restartFut.get();
+        final GridFutureAdapter curFut = this.restartFut.get();
 
-        boolean changed = this.restartFut.compareAndSet(currentFut, restartFut);
+        boolean changed = this.restartFut.compareAndSet(curFut, restartFut);
 
-        if (changed && currentFut != null)
+        if (changed && curFut != null)
             restartFut.listen(new IgniteInClosure>() {
-                @Override public void apply(IgniteInternalFuture future) {
-                    if (future.error() != null)
-                        currentFut.onDone(future.error());
+                @Override public void apply(IgniteInternalFuture fut) {
+                    if (fut.error() != null)
+                        curFut.onDone(fut.error());
                     else
-                        currentFut.onDone();
+                        curFut.onDone();
                 }
             });
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 1fd14f866744d..3bbe8f1e9ec6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -1977,6 +1977,22 @@ private void checkxEnabled() throws IgniteException {
     @SuppressWarnings("unchecked")
     public FieldsQueryCursor> querySqlFields(final GridCacheContext cctx, final SqlFieldsQuery qry,
         final boolean keepBinary) {
+        return querySqlFields(cctx, qry, keepBinary, true).get(0);
+    }
+
+    /**
+     * Query SQL fields.
+     *
+     * @param cctx Cache context.
+     * @param qry Query.
+     * @param keepBinary Keep binary flag.
+     * @param failOnMultipleStmts If {@code true} the method must throws exception when query contains
+     *      more then one SQL statement.
+     * @return Cursor.
+     */
+    @SuppressWarnings("unchecked")
+    public List>> querySqlFields(final GridCacheContext cctx, final SqlFieldsQuery qry,
+        final boolean keepBinary, final boolean failOnMultipleStmts) {
         checkxEnabled();
 
         validateSqlFieldsQuery(qry);
@@ -1994,37 +2010,39 @@ public FieldsQueryCursor> querySqlFields(final GridCacheContext cct
             final String schemaName = qry.getSchema() != null ? qry.getSchema() : idx.schema(cctx.name());
             final int mainCacheId = cctx.cacheId();
 
-            IgniteOutClosureX>> clo;
+            IgniteOutClosureX>>> clo;
 
             if (loc) {
-                clo = new IgniteOutClosureX>>() {
-                    @Override public FieldsQueryCursor> applyx() throws IgniteCheckedException {
+                clo = new IgniteOutClosureX>>>() {
+                    @Override public List>> applyx() throws IgniteCheckedException {
                         GridQueryCancel cancel = new GridQueryCancel();
 
-                        FieldsQueryCursor> cur;
+                        List>> cursors;
 
                         if (cctx.config().getQueryParallelism() > 1) {
                             qry.setDistributedJoins(true);
 
-                            cur = idx.queryDistributedSqlFields(schemaName, qry,
-                                keepBinary, cancel, mainCacheId, true).get(0);
+                            cursors = idx.queryDistributedSqlFields(schemaName, qry,
+                                keepBinary, cancel, mainCacheId, true);
                         }
                         else {
                             IndexingQueryFilter filter = idx.backupFilter(requestTopVer.get(), qry.getPartitions());
 
-                            cur = idx.queryLocalSqlFields(schemaName, qry, keepBinary, filter, cancel);
+                            cursors = new ArrayList<>(1);
+
+                            cursors.add(idx.queryLocalSqlFields(schemaName, qry, keepBinary, filter, cancel));
                         }
 
                         sendQueryExecutedEvent(qry.getSql(), qry.getArgs(), cctx.name());
 
-                        return cur;
+                        return cursors;
                     }
                 };
             }
             else {
-                clo = new IgniteOutClosureX>>() {
-                    @Override public FieldsQueryCursor> applyx() throws IgniteCheckedException {
-                        return idx.queryDistributedSqlFields(schemaName, qry, keepBinary, null, mainCacheId, true).get(0);
+                clo = new IgniteOutClosureX>>>() {
+                    @Override public List>> applyx() throws IgniteCheckedException {
+                        return idx.queryDistributedSqlFields(schemaName, qry, keepBinary, null, mainCacheId, failOnMultipleStmts);
                     }
                 };
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java
index 51bf7d6f10009..1daa1f2bd9ca7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.query.FieldsQueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.F;
@@ -84,22 +85,27 @@ private VisorQueryJob(VisorQueryTaskArg arg, boolean debug) {
 
                 long start = U.currentTimeMillis();
 
-                FieldsQueryCursor> qryCursor;
+                List>> qryCursors;
 
                 String cacheName = arg.getCacheName();
 
                 if (F.isEmpty(cacheName))
-                    qryCursor = ignite.context().query().querySqlFieldsNoCache(qry, true);
+                    qryCursors = ignite.context().query().querySqlFieldsNoCache(qry, true, false);
                 else {
                     IgniteCache c = ignite.cache(cacheName);
 
                     if (c == null)
                         throw new SQLException("Fail to execute query. Cache not found: " + cacheName);
 
-                    qryCursor = c.withKeepBinary().query(qry);
+                    qryCursors = ((IgniteCacheProxy)c.withKeepBinary()).queryMultipleStatements(qry);
                 }
 
-                VisorQueryCursor> cur = new VisorQueryCursor<>(qryCursor);
+                // In case of multiple statements leave opened only last cursor.
+                for (int i = 0; i < qryCursors.size() - 1; i++)
+                    U.closeQuiet(qryCursors.get(i));
+
+                // In case of multiple statements return last cursor as result.
+                VisorQueryCursor> cur = new VisorQueryCursor<>(F.last(qryCursors));
 
                 Collection meta = cur.fieldsMeta();
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 349ee92d5da6f..9cf86d08cea97 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -886,6 +886,9 @@ public GridQueryFieldsResult queryLocalSqlFields(final String schemaName, final
 
         final PreparedStatement stmt = preparedStatementWithParams(conn, qry, params, true);
 
+        if (GridSqlQueryParser.checkMultipleStatements(stmt))
+            throw new IgniteSQLException("Multiple statements queries are not supported for local queries");
+
         Prepared p = GridSqlQueryParser.prepared(stmt);
 
         if (DmlStatementsProcessor.isDmlStatement(p)) {
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index 61d7510c23da3..52dee60b90d4d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -408,10 +408,13 @@ public class GridSqlQueryParser {
     /** */
     private static final Getter REMAINING;
 
+    /** */
+    public static final String ORG_H2_COMMAND_COMMAND_LIST = "org.h2.command.CommandList";
+
     static {
         try {
             CLS_COMMAND_LIST = (Class)CommandContainer.class.getClassLoader()
-                .loadClass("org.h2.command.CommandList");
+                .loadClass(ORG_H2_COMMAND_COMMAND_LIST);
 
             LIST_COMMAND = getter(CLS_COMMAND_LIST, "command");
 
@@ -516,6 +519,16 @@ public GridSqlQueryParser(boolean useOptimizedSubqry) {
         optimizedTableFilterOrder = useOptimizedSubqry ? new HashMap() : null;
     }
 
+    /**
+     * @param stmt Prepared statement to check.
+     * @return {@code true} in case of multiple statements.
+     */
+    public static boolean checkMultipleStatements(PreparedStatement stmt) {
+        Command cmd = COMMAND.get((JdbcPreparedStatement)stmt);
+
+        return ORG_H2_COMMAND_COMMAND_LIST.equals(cmd.getClass().getName());
+    }
+
     /**
      * @param stmt Prepared statement.
      * @return Parsed select.
@@ -540,7 +553,7 @@ public static PreparedWithRemaining preparedWithRemaining(PreparedStatement stmt
         else {
             Class cmdCls = cmd.getClass();
 
-            if (cmdCls.getName().equals("org.h2.command.CommandList")) {
+            if (cmdCls.getName().equals(ORG_H2_COMMAND_COMMAND_LIST)) {
                 return new PreparedWithRemaining(PREPARED.get(LIST_COMMAND.get(cmd)), REMAINING.get(cmd));
             }
             else
@@ -1495,7 +1508,7 @@ private static int parseIntParam(String name, String val) {
         try {
             return Integer.parseInt(val);
         }
-        catch (NumberFormatException e) {
+        catch (NumberFormatException ignored) {
             throw new IgniteSQLException("Parameter value must be an integer [name=" + name + ", value=" + val + ']',
                 IgniteQueryErrorCode.PARSING);
         }
@@ -2013,4 +2026,4 @@ public String remainingSql() {
             return remainingSql;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
index 6344cba5baf50..bbd3d0d9e85eb 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
@@ -24,21 +24,26 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Callable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
@@ -48,6 +53,15 @@
  * Tests cross cache queries.
  */
 public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final String PART_CACHE_NAME = "partitioned";
+
+    /** */
+    private static final String REPL_PROD_CACHE_NAME = "replicated-prod";
+
+    /** */
+    private static final String REPL_STORE_CACHE_NAME = "replicated-store";
+
     /** */
     private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
 
@@ -65,9 +79,9 @@ public class GridCacheCrossCacheQuerySelfTest extends GridCommonAbstractTest {
         c.setDiscoverySpi(disco);
 
         c.setCacheConfiguration(
-            createCache("partitioned", CacheMode.PARTITIONED, Integer.class, FactPurchase.class),
-            createCache("replicated-prod", CacheMode.REPLICATED, Integer.class, DimProduct.class),
-            createCache("replicated-store", CacheMode.REPLICATED, Integer.class, DimStore.class));
+            createCache(PART_CACHE_NAME, CacheMode.PARTITIONED, Integer.class, FactPurchase.class),
+            createCache(REPL_PROD_CACHE_NAME, CacheMode.REPLICATED, Integer.class, DimProduct.class),
+            createCache(REPL_STORE_CACHE_NAME, CacheMode.REPLICATED, Integer.class, DimStore.class));
 
         return c;
     }
@@ -115,8 +129,7 @@ private static CacheConfiguration createCache(String name, CacheMode mode, Class
      * @throws Exception If failed.
      */
     public void testTwoStepGroupAndAggregates() throws Exception {
-        IgniteInternalCache cache =
-            ((IgniteKernal)ignite).getCache("partitioned");
+        IgniteInternalCache cache = ((IgniteKernal)ignite).getCache(PART_CACHE_NAME);
 
         GridQueryProcessor qryProc = ((IgniteKernal) ignite).context().query();
 
@@ -216,7 +229,7 @@ public void testTwoStepGroupAndAggregates() throws Exception {
      * @throws Exception If failed.
      */
     public void testApiQueries() throws Exception {
-        IgniteCache c = ignite.cache("partitioned");
+        IgniteCache c = ignite.cache(PART_CACHE_NAME);
 
         c.query(new SqlFieldsQuery("select cast(? as varchar) from FactPurchase").setArgs("aaa")).getAll();
 
@@ -227,9 +240,46 @@ public void testApiQueries() throws Exception {
         assertEquals("aaa", res.get(0).get(0));
     }
 
-//    @Override protected long getTestTimeout() {
-//        return 10 * 60 * 1000;
-//    }
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMultiStatement() throws Exception {
+        final IgniteInternalCache cache = ((IgniteKernal)ignite).getCache(PART_CACHE_NAME);
+
+        final GridQueryProcessor qryProc = ((IgniteKernal) ignite).context().query();
+
+        final SqlFieldsQuery qry = new SqlFieldsQuery(
+            "insert into FactPurchase(_key, id, productId, storeId, price) values (555, 555, 555, 555, 555);" +
+            "select count(*) from FactPurchase"
+        );
+
+        GridTestUtils.assertThrows(log,
+            new Callable() {
+                @Override public Object call() throws Exception {
+                    qryProc.querySqlFields(cache.context(), qry, false, true);
+
+                    return null;
+                }
+            }, IgniteSQLException.class, "Multiple statements queries are not supported");
+
+        List>> cursors = qryProc.querySqlFields(cache.context(), qry, false, false);
+
+        assertEquals(2, cursors.size());
+
+        for(FieldsQueryCursor> cur : cursors)
+            U.closeQuiet(cur);
+
+        qry.setLocal(true);
+
+        GridTestUtils.assertThrows(log,
+            new Callable() {
+                @Override public Object call() throws Exception {
+                    qryProc.querySqlFields(cache.context(), qry, false, false);
+
+                    return null;
+                }
+            }, IgniteSQLException.class, "Multiple statements queries are not supported for local queries");
+    }
 
     /**
      * @param l List.
@@ -246,8 +296,8 @@ private static int i(List l, int idx){
     private void fillCaches() throws IgniteCheckedException {
         int idGen = 0;
 
-        GridCacheAdapter dimCacheProd = ((IgniteKernal)ignite).internalCache("replicated-prod");
-        GridCacheAdapter dimCacheStore = ((IgniteKernal)ignite).internalCache("replicated-store");
+        GridCacheAdapter dimCacheProd = ((IgniteKernal)ignite).internalCache(REPL_PROD_CACHE_NAME);
+        GridCacheAdapter dimCacheStore = ((IgniteKernal)ignite).internalCache(REPL_STORE_CACHE_NAME);
 
         List dimStores = new ArrayList<>();
 
@@ -273,17 +323,17 @@ private void fillCaches() throws IgniteCheckedException {
             dimProds.add(v);
         }
 
-        GridCacheAdapter factCache = ((IgniteKernal)ignite).internalCache("partitioned");
+        GridCacheAdapter factCache = ((IgniteKernal)ignite).internalCache(PART_CACHE_NAME);
 
         Collections.sort(dimStores, new Comparator() {
             @Override public int compare(DimStore o1, DimStore o2) {
-                return o1.getId() > o2.getId() ? 1 : o1.getId() < o2.getId() ? -1 : 0;
+                return Integer.compare(o1.getId(), o2.getId());
             }
         });
 
         Collections.sort(dimProds, new Comparator() {
             @Override public int compare(DimProduct o1, DimProduct o2) {
-                return o1.getId() > o2.getId() ? 1 : o1.getId() < o2.getId() ? -1 : 0;
+                return Integer.compare(o1.getId(), o2.getId());
             }
         });
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
index f3deec9d41194..fb38c46b15dd3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
@@ -74,13 +74,13 @@
  */
 public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
     /** Client node index. */
-    private final static int CLIENT = 2;
+    private static final int CLIENT = 2;
 
     /** */
-    private final static String INDEXED_CACHE_NAME = CACHE_NAME + "_idx";
+    private static final String INDEXED_CACHE_NAME = CACHE_NAME + "_idx";
 
     /** */
-    private final static String INDEXED_CACHE_NAME_2 = INDEXED_CACHE_NAME + "_2";
+    private static final String INDEXED_CACHE_NAME_2 = INDEXED_CACHE_NAME + "_2";
 
     /** Data region name. */
     public static final String DATA_REGION_NAME = "my_data_region";

From 36cc822935387b2150e690c29bc6992dca0563f7 Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin 
Date: Thu, 18 Jan 2018 11:49:08 +0700
Subject: [PATCH 07/65] IGNITE-7306 Web Console: Fixed export data from tables.
 (cherry picked from commit 1bb60ec)

---
 .../app/components/grid-export/component.js   | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/modules/web-console/frontend/app/components/grid-export/component.js b/modules/web-console/frontend/app/components/grid-export/component.js
index 18e41c7f14ffe..d312959b16d9a 100644
--- a/modules/web-console/frontend/app/components/grid-export/component.js
+++ b/modules/web-console/frontend/app/components/grid-export/component.js
@@ -33,21 +33,31 @@ export default {
 
         export() {
             const data = [];
-            const columnHeaders = this.uiGridExporterService.getColumnHeaders(this.gridApi.grid, this.uiGridExporterConstants.VISIBLE);
+            const grid = this.gridApi.grid;
+            const exportColumnHeaders = this.uiGridExporterService.getColumnHeaders(grid, this.uiGridExporterConstants.VISIBLE);
 
-            _.forEach(this.gridApi.grid.rows, (row) => {
+            grid.rows.forEach((row) => {
                 if (!row.visible)
                     return;
 
                 const values = [];
-                _.forEach(columnHeaders, ({ name }) => {
-                    values.push({ value: row.entity[name] });
+
+                exportColumnHeaders.forEach((exportCol) => {
+                    const col = grid.columns.find(({ field }) => field === exportCol.name);
+
+                    if (!col || !col.visible || col.colDef.exporterSuppressExport === true)
+                        return;
+
+                    const value = grid.getCellValue(row, col);
+
+                    values.push({ value });
                 });
 
                 data.push(values);
             });
 
-            const csvContent = this.uiGridExporterService.formatAsCsv(columnHeaders, data, this.CSV.getSeparator());
+            const csvContent = this.uiGridExporterService.formatAsCsv(exportColumnHeaders, data, this.CSV.getSeparator());
+
             this.uiGridExporterService.downloadFile(this.gridApi.grid.options.exporterCsvFilename, csvContent, this.gridApi.grid.options.exporterOlderExcelCompatibility);
         }
     },

From d753298b4012894b56f5c9218325211cd84a21d5 Mon Sep 17 00:00:00 2001
From: Peter Ivanov 
Date: Thu, 18 Jan 2018 09:18:53 +0300
Subject: [PATCH 08/65] IGNITE-7107 Apache Ignite RPM packages

* added changelog to package specification

This closes #3396
---
 packaging/rpm/SPECS/apache-ignite.spec | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/packaging/rpm/SPECS/apache-ignite.spec b/packaging/rpm/SPECS/apache-ignite.spec
index 45d72b77a959c..b90007350568b 100644
--- a/packaging/rpm/SPECS/apache-ignite.spec
+++ b/packaging/rpm/SPECS/apache-ignite.spec
@@ -204,3 +204,11 @@ ln -sf %{_var}/log/%{name} %{buildroot}%{_sharedstatedir}/%{name}/log
 %doc %{name}-*/NOTICE
 %doc %{name}-*/RELEASE_NOTES.txt
 %license %{name}-*/LICENSE
+
+#-------------------------------------------------------------------------------
+#
+# Changelog
+#
+%changelog
+* Wed Jan 17 2018 GridGain Systems  - 2.4.0-1
+- Initial package release

From 63445893f1bc75dc9777184499f7eabc1d4e51b1 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov 
Date: Thu, 18 Jan 2018 11:36:18 +0300
Subject: [PATCH 09/65] IGNITE-3935 Use PeerDeployAware for streamer
 transformer - Fixes #3378.

Signed-off-by: Alexey Goncharuk 
---
 .../apache/ignite/IgniteSystemProperties.java |  15 +++
 .../ignite/stream/StreamTransformer.java      |  60 ++++++++-
 .../p2p/P2PStreamingClassLoaderTest.java      | 121 ++++++++++++++++++
 .../testsuites/IgniteP2PSelfTestSuite.java    |   2 +
 .../tests/p2p/NoopCacheEntryProcessor.java    |  36 ++++++
 5 files changed, 229 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/p2p/P2PStreamingClassLoaderTest.java
 create mode 100644 modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/NoopCacheEntryProcessor.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 1bbaacdd5315b..9d520fbac772d 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -23,11 +23,13 @@
 import java.util.Map;
 import java.util.Properties;
 import javax.net.ssl.HostnameVerifier;
+import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.internal.client.GridClient;
 import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.stream.StreamTransformer;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -566,6 +568,19 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_SERVICES_COMPATIBILITY_MODE = "IGNITE_SERVICES_COMPATIBILITY_MODE";
 
+    /**
+     * Manages backward compatibility of {@link StreamTransformer#from(CacheEntryProcessor)} method.
+     * 

+     * If the property is {@code true}, then the wrapped {@link CacheEntryProcessor} won't be able to be loaded over
+     * P2P class loading.
+     * 

+     * If the property is {@code false}, then another implementation of {@link StreamTransformer} will be returned,
+     * that fixes P2P class loading for {@link CacheEntryProcessor}, but it will be incompatible with old versions
+     * of Ignite.
+     */
+    public static final String IGNITE_STREAM_TRANSFORMER_COMPATIBILITY_MODE =
+        "IGNITE_STREAM_TRANSFORMER_COMPATIBILITY_MODE";
+
     /**
      * When set to {@code true} tree-based data structures - {@code TreeMap} and {@code TreeSet} - will not be
      * wrapped into special holders introduced to overcome serialization issue caused by missing {@code Comparable}
diff --git a/modules/core/src/main/java/org/apache/ignite/stream/StreamTransformer.java b/modules/core/src/main/java/org/apache/ignite/stream/StreamTransformer.java
index 176973eab14ce..9951c7cc9e7f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/stream/StreamTransformer.java
+++ b/modules/core/src/main/java/org/apache/ignite/stream/StreamTransformer.java
@@ -24,7 +24,10 @@
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.internal.util.lang.GridPeerDeployAware;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
  * Convenience adapter to transform update existing values in streaming cache
@@ -39,6 +42,10 @@ public abstract class StreamTransformer implements StreamReceiver, E
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** Compatibility mode flag. */
+    private static final boolean compatibilityMode =
+        IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_STREAM_TRANSFORMER_COMPATIBILITY_MODE);
+
     /** {@inheritDoc} */
     @Override public void receive(IgniteCache cache, Collection> entries) throws IgniteException {
         for (Map.Entry entry : entries)
@@ -52,10 +59,53 @@ public abstract class StreamTransformer implements StreamReceiver, E
      * @return Stream transformer.
      */
     public static  StreamTransformer from(final CacheEntryProcessor ep) {
-        return new StreamTransformer() {
-            @Override public Object process(MutableEntry entry, Object... args) throws EntryProcessorException {
-                return ep.process(entry, args);
-            }
-        };
+        if (compatibilityMode)
+            return new StreamTransformer() {
+                @Override public Object process(MutableEntry entry, Object... args) throws EntryProcessorException {
+                    return ep.process(entry, args);
+                }
+            };
+        else
+            return new EntryProcessorWrapper<>(ep);
+    }
+
+    /**
+     * @param  Key type.
+     * @param  Value type.
+     */
+    private static class EntryProcessorWrapper extends StreamTransformer implements GridPeerDeployAware {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private CacheEntryProcessor ep;
+
+        /** */
+        private transient ClassLoader ldr;
+
+        /**
+         * @param ep Entry processor.
+         */
+        EntryProcessorWrapper(CacheEntryProcessor ep) {
+            this.ep = ep;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object process(MutableEntry entry, Object... args) throws EntryProcessorException {
+            return ep.process(entry, args);
+        }
+
+        /** {@inheritDoc} */
+        @Override public Class deployClass() {
+            return ep.getClass();
+        }
+
+        /** {@inheritDoc} */
+        @Override public ClassLoader classLoader() {
+            if (ldr == null)
+                ldr = U.detectClassLoader(deployClass());
+
+            return ldr;
+        }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/p2p/P2PStreamingClassLoaderTest.java b/modules/core/src/test/java/org/apache/ignite/p2p/P2PStreamingClassLoaderTest.java
new file mode 100644
index 0000000000000..a5418132369d7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/p2p/P2PStreamingClassLoaderTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.p2p;
+
+import java.lang.reflect.Constructor;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.configuration.DeploymentMode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.stream.StreamTransformer;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+/** */
+@GridCommonTest(group = "P2P")
+public class P2PStreamingClassLoaderTest extends GridCommonAbstractTest {
+    /** */
+    private static final String ENTRY_PROCESSOR_CLASS_NAME = "org.apache.ignite.tests.p2p.NoopCacheEntryProcessor";
+
+    /** */
+    private static final String CACHE_NAME = "cache";
+
+    /**
+     * Current deployment mode. Used in {@link #getConfiguration(String)}.
+     */
+    private DeploymentMode depMode;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (igniteInstanceName.startsWith("client"))
+            cfg.setClientMode(true);
+
+        cfg.setDeploymentMode(depMode);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception if error occur
+     */
+    @SuppressWarnings("unchecked")
+    private void processTest() throws Exception {
+        try {
+            startGrid("server");
+            Ignite client = startGrid("client");
+
+            ClassLoader ldr = getExternalClassLoader();
+
+            Class epCls = ldr.loadClass(ENTRY_PROCESSOR_CLASS_NAME);
+
+            Constructor epCtr = epCls.getConstructor();
+
+            CacheEntryProcessor ep = (CacheEntryProcessor)epCtr.newInstance();
+
+            IgniteCache cache = client.createCache(CACHE_NAME);
+
+            try (IgniteDataStreamer streamer = client.dataStreamer(CACHE_NAME)) {
+                streamer.receiver(StreamTransformer.from(ep));
+
+                streamer.addData(1, "1");
+            }
+
+            assertEquals("1", cache.get(1));
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * Test GridDeploymentMode.PRIVATE mode.
+     *
+     * @throws Exception if error occur.
+     */
+    public void testPrivateMode() throws Exception {
+        depMode = DeploymentMode.PRIVATE;
+
+        processTest();
+    }
+
+    /**
+     * Test {@link DeploymentMode#CONTINUOUS} mode.
+     *
+     * @throws Exception if error occur.
+     */
+    public void testContinuousMode() throws Exception {
+        depMode = DeploymentMode.CONTINUOUS;
+
+        processTest();
+    }
+
+    /**
+     * Test GridDeploymentMode.SHARED mode.
+     *
+     * @throws Exception if error occur.
+     */
+    public void testSharedMode() throws Exception {
+        depMode = DeploymentMode.SHARED;
+
+        processTest();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java
index 3c50bafca7d08..524283bdbfcc4 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.p2p.GridP2PSameClassLoaderSelfTest;
 import org.apache.ignite.p2p.GridP2PTimeoutSelfTest;
 import org.apache.ignite.p2p.GridP2PUndeploySelfTest;
+import org.apache.ignite.p2p.P2PStreamingClassLoaderTest;
 import org.apache.ignite.p2p.SharedDeploymentTest;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -73,6 +74,7 @@ public static TestSuite suite(Set ignoredTests) throws Exception {
         suite.addTest(new TestSuite(GridP2PMissedResourceCacheSizeSelfTest.class));
         suite.addTest(new TestSuite(GridP2PContinuousDeploymentSelfTest.class));
         suite.addTest(new TestSuite(DeploymentClassLoaderCallableTest.class));
+        suite.addTest(new TestSuite(P2PStreamingClassLoaderTest.class));
         suite.addTest(new TestSuite(SharedDeploymentTest.class));
         GridTestUtils.addTestIfNeeded(suite, GridDeploymentMessageCountSelfTest.class, ignoredTests);
 
diff --git a/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/NoopCacheEntryProcessor.java b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/NoopCacheEntryProcessor.java
new file mode 100644
index 0000000000000..c473d2f482489
--- /dev/null
+++ b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/NoopCacheEntryProcessor.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tests.p2p;
+
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.cache.CacheEntryProcessor;
+
+/**
+ * @param  Key type.
+ * @param  Value type.
+ */
+@SuppressWarnings("unchecked")
+public class NoopCacheEntryProcessor implements CacheEntryProcessor {
+    /** {@inheritDoc} */
+    @Override public Object process(MutableEntry e, Object... args) throws EntryProcessorException {
+        e.setValue(args[0]);
+
+        return null;
+    }
+}

From f3f9f2a24b23027cf0c835140322e41a788932ae Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn 
Date: Thu, 18 Jan 2018 12:05:12 +0300
Subject: [PATCH 10/65] IGNITE-7413 Fix SqlDmlExample

This closes #3389
---
 .../apache/ignite/examples/sql/SqlDmlExample.java    | 12 +++---------
 .../Apache.Ignite.Examples/Sql/SqlDmlExample.cs      |  5 +----
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/examples/src/main/java/org/apache/ignite/examples/sql/SqlDmlExample.java b/examples/src/main/java/org/apache/ignite/examples/sql/SqlDmlExample.java
index cb2a04a30013c..dbb1088e888f1 100644
--- a/examples/src/main/java/org/apache/ignite/examples/sql/SqlDmlExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/sql/SqlDmlExample.java
@@ -119,15 +119,9 @@ private static void update(IgniteCache personCache) {
      * @param personCache Person cache.
      */
     private static void delete(IgniteCache personCache) {
-        String sql =
-            "delete from Person " +
-            "where id in (" +
-                "select p.id " +
-                "from Person p, \"" + ORG_CACHE + "\".Organization as o " +
-                "where o.name != ? and p.orgId = o.id" +
-            ")";
-
-        personCache.query(new SqlFieldsQuery(sql).setArgs("ASF")).getAll();
+        String sql = "delete from Person where orgId != ?";
+
+        personCache.query(new SqlFieldsQuery(sql).setArgs(1)).getAll();
     }
 
     /**
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlDmlExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlDmlExample.cs
index a37db76c2f858..7b84fc348a001 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlDmlExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlDmlExample.cs
@@ -153,10 +153,7 @@ private static void Update(ICache employeeCache)
         /// Employee cache.
         private static void Delete(ICache employeeCache)
         {
-            var qry = new SqlFieldsQuery(string.Format(
-                "delete from Employee where _key in (" +
-                "select emp._key from Employee emp, \"{0}\".Organization org " +
-                "where org.Name != ? and org._key = emp.organizationId)", OrganizationCacheName), "ASF");
+            var qry = new SqlFieldsQuery("delete from Employee where organizationId != ?", 1);
 
             employeeCache.Query(qry);
         }

From 1daa7c41bf1460a4d9a2b0c26a7a317f2fca3fb7 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov 
Date: Thu, 18 Jan 2018 17:14:53 +0700
Subject: [PATCH 11/65] IGNITE-7461 UI tools: Actualized data storage
 configuration. (cherry picked from commit 577e632)

---
 .../node/VisorDataStorageConfiguration.java   | 47 ++++++++++++++-----
 modules/web-console/backend/app/mongo.js      |  5 +-
 .../generator/ConfigurationGenerator.js       | 14 ++++--
 .../configuration/clusters/data-storage.pug   |  9 ++--
 4 files changed, 54 insertions(+), 21 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorDataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorDataStorageConfiguration.java
index 8470fe10f38ab..d0e0f915312ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorDataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorDataStorageConfiguration.java
@@ -65,9 +65,6 @@ public class VisorDataStorageConfiguration extends VisorDataTransferObject {
     /** Lock wait time. */
     private long lockWaitTime;
 
-    /** */
-    private long checkpointPageBufSize;
-
     /** */
     private int checkpointThreads;
 
@@ -125,6 +122,12 @@ public class VisorDataStorageConfiguration extends VisorDataTransferObject {
     /** If true, threads that generate dirty pages too fast during ongoing checkpoint will be throttled. */
     private boolean writeThrottlingEnabled;
 
+    /** Size of WAL buffer. */
+    private int walBufSize;
+
+    /** If true, system filters and compresses WAL archive in background. */
+    private boolean walCompactionEnabled;
+
     /**
      * Default constructor.
      */
@@ -165,6 +168,7 @@ public VisorDataStorageConfiguration(DataStorageConfiguration cfg) {
         metricsEnabled = cfg.isMetricsEnabled();
         walMode = cfg.getWalMode();
         walTlbSize = cfg.getWalThreadLocalBufferSize();
+        walBufSize = cfg.getWalBufferSize();
         walFlushFreq = cfg.getWalFlushFrequency();
         walFsyncDelay = cfg.getWalFsyncDelayNanos();
         walRecordIterBuffSize = cfg.getWalRecordIteratorBufferSize();
@@ -174,6 +178,7 @@ public VisorDataStorageConfiguration(DataStorageConfiguration cfg) {
         metricsRateTimeInterval = cfg.getMetricsRateTimeInterval();
         walAutoArchiveAfterInactivity = cfg.getWalAutoArchiveAfterInactivity();
         writeThrottlingEnabled = cfg.isWriteThrottlingEnabled();
+        walCompactionEnabled = cfg.isWalCompactionEnabled();
     }
 
     /**
@@ -232,13 +237,6 @@ public long getCheckpointFrequency() {
         return checkpointFreq;
     }
 
-    /**
-     * @return Checkpointing page buffer size in bytes.
-     */
-    public long getCheckpointPageBufferSize() {
-        return checkpointPageBufSize;
-    }
-
     /**
      * @return Number of checkpointing threads.
      */
@@ -379,6 +377,24 @@ public boolean isWriteThrottlingEnabled() {
         return writeThrottlingEnabled;
     }
 
+    /**
+     * @return Size of WAL buffer.
+     */
+    public int getWalBufferSize() {
+        return walBufSize;
+    }
+
+    /**
+     * @return If true, system filters and compresses WAL archive in background
+     */
+    public boolean isWalCompactionEnabled() {
+        return walCompactionEnabled;
+    }
+
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         out.writeLong(sysRegionInitSize);
@@ -390,7 +406,7 @@ public boolean isWriteThrottlingEnabled() {
         U.writeString(out, storagePath);
         out.writeLong(checkpointFreq);
         out.writeLong(lockWaitTime);
-        out.writeLong(0);
+        out.writeLong(0);  // Write stub for removed checkpointPageBufSize.
         out.writeInt(checkpointThreads);
         U.writeEnum(out, checkpointWriteOrder);
         out.writeInt(walHistSize);
@@ -410,6 +426,8 @@ public boolean isWriteThrottlingEnabled() {
         out.writeLong(metricsRateTimeInterval);
         out.writeLong(walAutoArchiveAfterInactivity);
         out.writeBoolean(writeThrottlingEnabled);
+        out.writeInt(walBufSize);
+        out.writeBoolean(walCompactionEnabled);
     }
 
     /** {@inheritDoc} */
@@ -423,7 +441,7 @@ public boolean isWriteThrottlingEnabled() {
         storagePath = U.readString(in);
         checkpointFreq = in.readLong();
         lockWaitTime = in.readLong();
-        checkpointPageBufSize = in.readLong();
+        in.readLong(); // Read stub for removed checkpointPageBufSize.
         checkpointThreads = in.readInt();
         checkpointWriteOrder = CheckpointWriteOrder.fromOrdinal(in.readByte());
         walHistSize = in.readInt();
@@ -443,6 +461,11 @@ public boolean isWriteThrottlingEnabled() {
         metricsRateTimeInterval = in.readLong();
         walAutoArchiveAfterInactivity = in.readLong();
         writeThrottlingEnabled = in.readBoolean();
+
+        if (protoVer > V1) {
+            walBufSize = in.readInt();
+            walCompactionEnabled = in.readBoolean();
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index 0ea4b88e737d4..189f45bdf28db 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -1010,7 +1010,6 @@ const defineSchema = (mongoose) => {
             metricsEnabled: Boolean,
             alwaysWriteFullPages: Boolean,
             checkpointFrequency: Number,
-            checkpointPageBufferSize: Number,
             checkpointThreads: Number,
             checkpointWriteOrder: {type: String, enum: ['RANDOM', 'SEQUENTIAL']},
             walPath: String,
@@ -1023,12 +1022,14 @@ const defineSchema = (mongoose) => {
             walFsyncDelayNanos: Number,
             walRecordIteratorBufferSize: Number,
             lockWaitTime: Number,
+            walBufferSize: Number,
             walThreadLocalBufferSize: Number,
             metricsSubIntervalCount: Number,
             metricsRateTimeInterval: Number,
             fileIOFactory: {type: String, enum: ['RANDOM', 'ASYNC']},
             walAutoArchiveAfterInactivity: Number,
-            writeThrottlingEnabled: Boolean
+            writeThrottlingEnabled: Boolean,
+            walCompactionEnabled: Boolean
         },
         memoryConfiguration: {
             systemCacheInitialSize: Number,
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index cf14293393039..9269fc930f74f 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -1427,15 +1427,18 @@ export default class IgniteConfigurationGenerator {
 
         storageBean.stringProperty('storagePath')
             .intProperty('checkpointFrequency')
-            .longProperty('checkpointPageBufferSize')
             .intProperty('checkpointThreads')
             .enumProperty('walMode')
             .stringProperty('walPath')
             .stringProperty('walArchivePath')
             .intProperty('walSegments')
             .intProperty('walSegmentSize')
-            .intProperty('walHistorySize')
-            .longProperty('walFlushFrequency')
+            .intProperty('walHistorySize');
+
+        if (available('2.4.0'))
+            storageBean.intProperty('walBufferSize');
+
+        storageBean.longProperty('walFlushFrequency')
             .longProperty('walFsyncDelayNanos')
             .intProperty('walRecordIteratorBufferSize')
             .longProperty('lockWaitTime')
@@ -1447,6 +1450,9 @@ export default class IgniteConfigurationGenerator {
             .boolProperty('alwaysWriteFullPages')
             .boolProperty('writeThrottlingEnabled');
 
+        if (available('2.4.0'))
+            storageBean.boolProperty('walCompactionEnabled');
+
         const fileIOFactory = _.get(dataStorageCfg, 'fileIOFactory');
 
         let factoryBean;
@@ -1932,7 +1938,7 @@ export default class IgniteConfigurationGenerator {
             propName = (near ? 'nearEviction' : 'eviction') + 'Policy';
         }
 
-        const bean = new Bean(beanProps.cls, 'evictionPlc', beanProps.src, dflt);
+        const bean = new Bean(beanProps.cls, propName, beanProps.src, dflt);
 
         bean.intProperty('batchSize')
             .intProperty('maxMemorySize')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/data-storage.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/data-storage.pug
index 82c6dbebf7546..5112591a7f981 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/data-storage.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/data-storage.pug
@@ -174,9 +174,6 @@ include /app/helpers/jade/mixins
                 .settings-row
                     +number('Checkpoint frequency:', `${model}.checkpointFrequency`, '"DataStorageCheckpointFrequency"', 'true', '180000', '1',
                     'Frequency which is a minimal interval when the dirty pages will be written to the Persistent Store')
-                .settings-row
-                    +number('Checkpoint page buffer size:', `${model}.checkpointPageBufferSize`, '"DataStorageCheckpointPageBufferSize"', 'true', '268435456', '0',
-                    'Amount of memory allocated for a checkpointing temporary buffer')
                 .settings-row
                     +number('Checkpoint threads:', `${model}.checkpointThreads`, '"DataStorageCheckpointThreads"', 'true', '4', '1', 'A number of threads to use for the checkpoint purposes')
                 .settings-row
@@ -215,6 +212,9 @@ include /app/helpers/jade/mixins
                     +number('WAL segment size:', `${model}.walSegmentSize`, '"DataStorageWalSegmentSize"', 'true', '67108864', '0', 'Size of a WAL segment')
                 .settings-row
                     +number('WAL history size:', `${model}.walHistorySize`, '"DataStorageWalHistorySize"', 'true', '20', '1', 'A total number of checkpoints to keep in the WAL history')
+                .settings-row(ng-show='$ctrl.available("2.4.0")')
+                    +number('WAL buffer size:', `${model}.walBufferSize`, '"DataStorageWalBufferSize"', 'true', 'WAL segment size / 4', '1',
+                    'Size of WAL buffer')
                 .settings-row
                     +number('WAL flush frequency:', `${model}.walFlushFrequency`, '"DataStorageWalFlushFrequency"', 'true', '2000', '1',
                     'How often will be fsync, in milliseconds. In background mode, exist thread which do fsync by timeout')
@@ -257,5 +257,8 @@ include /app/helpers/jade/mixins
                 .settings-row
                     +checkbox('Write throttling enabled', `${model}.writeThrottlingEnabled`, '"DataStorageWriteThrottlingEnabled"',
                     'Throttle threads that generate dirty pages too fast during ongoing checkpoint')
+                .settings-row(ng-show='$ctrl.available("2.4.0")')
+                    +checkbox('Enable WAL compaction', `${model}.walCompactionEnabled`, '"DataStorageWalCompactionEnabled"',
+                    'If true, system filters and compresses WAL archive in background')
             .col-sm-6
                 +preview-xml-java(model, 'clusterDataStorageConfiguration')

From cf0080210d24d9dd8b057f959446fac5f8a4ca01 Mon Sep 17 00:00:00 2001
From: dpavlov 
Date: Thu, 18 Jan 2018 13:53:29 +0300
Subject: [PATCH 12/65] IGNITE-7380 Implemented pluggable Direct IO - Fixes
 #3226.

Signed-off-by: Alexey Goncharuk 
---
 .../apache/ignite/IgniteSystemProperties.java |   8 +
 .../cache/GridCacheSharedContext.java         |  21 +-
 .../GridCacheDatabaseSharedManager.java       |  11 +-
 .../cache/persistence/file/FileIO.java        |   7 +-
 .../cache/persistence/file/FilePageStore.java |   3 +-
 .../file/FilePageStoreManager.java            |  47 +-
 .../file/FileVersionCheckingFactory.java      |  32 +-
 .../wal/FileWriteAheadLogManager.java         |  34 +-
 .../IgniteClusterActivateDeactivateTest.java  |   2 +-
 .../IgniteDataStorageMetricsSelfTest.java     |   2 +-
 .../IgnitePdsClientNearCachePutGetTest.java   |  33 +-
 .../IgnitePdsDynamicCacheTest.java            |   2 +-
 ...itePdsSingleNodePutGetPersistenceTest.java |  15 -
 .../db/IgnitePdsCacheRestoreTest.java         |   2 +-
 ...iteCheckpointDirtyPagesForLowLoadTest.java |  29 +-
 .../file/IgnitePdsCheckpointSimpleTest.java   | 100 ++++
 .../db/file/IgnitePdsEvictionTest.java        |   2 +-
 .../database/IgniteDbAbstractTest.java        |  29 +-
 .../IgniteDbDynamicCacheSelfTest.java         |  11 +-
 .../IgniteDbMemoryLeakAbstractTest.java       |   2 +-
 .../database/IgniteDbPutGetAbstractTest.java  |   1 +
 .../ignite/testsuites/IgnitePdsTestSuite.java |  34 +-
 .../testsuites/IgnitePdsTestSuite2.java       |  29 +-
 modules/direct-io/pom.xml                     | 122 +++++
 .../persistence/file/AlignedBuffers.java      |  72 +++
 .../file/AlignedBuffersDirectFileIO.java      | 511 ++++++++++++++++++
 .../AlignedBuffersDirectFileIOFactory.java    | 177 ++++++
 .../persistence/file/IgniteNativeIoLib.java   | 405 ++++++++++++++
 .../file/LinuxNativeIoPlugin.java}            |  26 +-
 .../file/LinuxNativeIoPluginProvider.java     | 238 ++++++++
 .../org.apache.ignite.plugin.PluginProvider   |   1 +
 .../IgniteNativeIoWithNoPersistenceTest.java  |  76 +++
 .../IgnitePdsNativeIoTestSuite.java           |  38 ++
 .../IgnitePdsNativeIoTestSuite2.java          |  36 ++
 pom.xml                                       |   1 +
 35 files changed, 2032 insertions(+), 127 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimpleTest.java
 create mode 100644 modules/direct-io/pom.xml
 create mode 100644 modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffers.java
 create mode 100644 modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
 create mode 100644 modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java
 create mode 100644 modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoLib.java
 rename modules/{core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbClientNearCachePutGetTest.java => direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/LinuxNativeIoPlugin.java} (63%)
 create mode 100644 modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/LinuxNativeIoPluginProvider.java
 create mode 100644 modules/direct-io/src/main/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
 create mode 100644 modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoWithNoPersistenceTest.java
 create mode 100644 modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite.java
 create mode 100644 modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 9d520fbac772d..833773a15b7bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -809,6 +809,14 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_GRID_CLIENT_LOG_ENABLED = "IGNITE_GRID_CLIENT_LOG_ENABLED";
 
+    /**
+     * When set to {@code true}, direct IO may be enabled. Direct IO enabled only if JAR file with corresponding
+     * feature is available in classpath and OS and filesystem settings allows to enable this mode.
+     * Default is {@code true}.
+     */
+    public static final String IGNITE_DIRECT_IO_ENABLED = "IGNITE_DIRECT_IO_ENABLED";
+
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index 5bf1343720b66..306723bb2f4a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -104,8 +104,8 @@ public class GridCacheSharedContext {
     /** Deployment manager. */
     private GridCacheDeploymentManager depMgr;
 
-    /** Write ahead log manager. */
-    private IgniteWriteAheadLogManager walMgr;
+    /** Write ahead log manager. {@code Null} if persistence is not enabled. */
+    @Nullable private IgniteWriteAheadLogManager walMgr;
 
     /** Database manager. */
     private IgniteCacheDatabaseSharedManager dbMgr;
@@ -113,8 +113,8 @@ public class GridCacheSharedContext {
     /** Snp manager. */
     private IgniteCacheSnapshotManager snpMgr;
 
-    /** Page store manager. */
-    private IgnitePageStoreManager pageStoreMgr;
+    /** Page store manager. {@code Null} if persistence is not enabled. */
+    @Nullable private IgnitePageStoreManager pageStoreMgr;
 
     /** Affinity manager. */
     private CacheAffinitySharedManager affMgr;
@@ -169,6 +169,8 @@ public class GridCacheSharedContext {
      * @param txMgr Transaction manager.
      * @param verMgr Version manager.
      * @param mvccMgr MVCC manager.
+     * @param pageStoreMgr Page store manager. {@code Null} if persistence is not enabled.
+     * @param walMgr WAL manager. {@code Null} if persistence is not enabled.
      * @param depMgr Deployment manager.
      * @param exchMgr Exchange manager.
      * @param affMgr Affinity manager.
@@ -182,8 +184,8 @@ public GridCacheSharedContext(
         IgniteTxManager txMgr,
         GridCacheVersionManager verMgr,
         GridCacheMvccManager mvccMgr,
-        IgnitePageStoreManager pageStoreMgr,
-        IgniteWriteAheadLogManager walMgr,
+        @Nullable IgnitePageStoreManager pageStoreMgr,
+        @Nullable IgniteWriteAheadLogManager walMgr,
         IgniteCacheDatabaseSharedManager dbMgr,
         IgniteCacheSnapshotManager snpMgr,
         GridCacheDeploymentManager depMgr,
@@ -398,6 +400,7 @@ private boolean restartOnDisconnect(GridCacheSharedManager mgr) {
      * @param jtaMgr JTA manager.
      * @param verMgr Version manager.
      * @param mvccMgr MVCC manager.
+     * @param pageStoreMgr Page store manager. {@code Null} if persistence is not enabled.
      * @param depMgr Deployment manager.
      * @param exchMgr Exchange manager.
      * @param affMgr Affinity manager.
@@ -409,7 +412,7 @@ private void setManagers(List> mgrs,
         CacheJtaManagerAdapter jtaMgr,
         GridCacheVersionManager verMgr,
         GridCacheMvccManager mvccMgr,
-        IgnitePageStoreManager pageStoreMgr,
+        @Nullable IgnitePageStoreManager pageStoreMgr,
         IgniteWriteAheadLogManager walMgr,
         IgniteCacheDatabaseSharedManager dbMgr,
         IgniteCacheSnapshotManager snpMgr,
@@ -667,9 +670,9 @@ public IgniteCacheSnapshotManager snapshot() {
     }
 
     /**
-     * @return Page store manager.
+     * @return Page store manager. {@code Null} if persistence is not enabled.
      */
-    public IgnitePageStoreManager pageStore() {
+    @Nullable public IgnitePageStoreManager pageStore() {
         return pageStoreMgr;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 359959401f9a9..1b0eb6d399a09 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -304,7 +304,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Checkpoint runner thread pool. If null tasks are to be run in single thread */
     @Nullable private ExecutorService asyncRunner;
 
-    /** Buffer for the checkpoint threads. */
+    /** Thread local with buffers for the checkpoint threads. Each buffer represent one page for durable memory. */
     private ThreadLocal threadBuf;
 
     /** */
@@ -2435,6 +2435,15 @@ private static String checkpointFileName(long cpTs, UUID cpId, CheckpointEntryTy
         return cpTs + "-" + cpId + "-" + type + ".bin";
     }
 
+    /**
+     * Replace thread local with buffers. Thread local should provide direct buffer with one page in length.
+     *
+     * @param threadBuf new thread-local with buffers for the checkpoint threads.
+     */
+    public void setThreadBuf(final ThreadLocal threadBuf) {
+        this.threadBuf = threadBuf;
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
index 849f03a51453a..73e44b0d9af1f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
@@ -52,7 +52,8 @@ public interface FileIO extends AutoCloseable {
      *
      * @param destBuf Destination byte buffer.
      *
-     * @return Number of read bytes.
+     * @return Number of read bytes, or -1 if the
+     *          given position is greater than or equal to the file's current size
      *
      * @throws IOException If some I/O error occurs.
      */
@@ -65,7 +66,9 @@ public interface FileIO extends AutoCloseable {
      * @param destBuf Destination byte buffer.
      * @param position Starting position of file.
      *
-     * @return Number of read bytes.
+     * @return Number of read bytes, possibly zero, or -1 if the
+     *          given position is greater than or equal to the file's current
+     *          size
      *
      * @throws IOException If some I/O error occurs.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 053ab2b605498..0ed9167bb8c4e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -451,7 +451,8 @@ private void init() throws IgniteCheckedException {
 
             assert pageBuf.capacity() == pageSize;
             assert pageBuf.position() == 0;
-            assert pageBuf.order() == ByteOrder.nativeOrder();
+            assert pageBuf.order() == ByteOrder.nativeOrder() : "Page buffer order " + pageBuf.order()
+                + " should be same with " + ByteOrder.nativeOrder();
             assert PageIO.getType(pageBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(pageId);
             assert PageIO.getVersion(pageBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(pageId);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index d406df66422cb..deacf79004aa3 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -36,8 +36,8 @@
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -94,8 +94,20 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
     /** */
     private final IgniteConfiguration igniteCfg;
 
+    /**
+     * File IO factory for page store, by default is taken from {@link #dsCfg}.
+     * May be overriden by block read/write.
+     */
+    private FileIOFactory pageStoreFileIoFactory;
+
+    /**
+     * File IO factory for page store V1 and for fast checking page store (non block read).
+     * By default is taken from {@link #dsCfg}.
+     */
+    private FileIOFactory pageStoreV1FileIoFactory;
+
     /** */
-    private DataStorageConfiguration dsCfg;
+    private final DataStorageConfiguration dsCfg;
 
     /** Absolute directory for file page store. Includes consistent id based folder. */
     private File storeWorkDir;
@@ -117,6 +129,8 @@ public FilePageStoreManager(GridKernalContext ctx) {
         assert dsCfg != null;
 
         this.dsCfg = dsCfg;
+
+        pageStoreV1FileIoFactory = pageStoreFileIoFactory = dsCfg.getFileIOFactory();
     }
 
     /** {@inheritDoc} */
@@ -373,8 +387,8 @@ private CacheStoreHolder initDir(File cacheWorkDir, int grpId, int partitions) t
         if (dirExisted && !idxFile.exists())
             grpsWithoutIdx.add(grpId);
 
-        FileVersionCheckingFactory pageStoreFactory = new FileVersionCheckingFactory(
-            dsCfg.getFileIOFactory(), igniteCfg.getDataStorageConfiguration());
+        FilePageStoreFactory pageStoreFactory = new FileVersionCheckingFactory(
+            pageStoreFileIoFactory, pageStoreV1FileIoFactory, igniteCfg.getDataStorageConfiguration());
 
         FilePageStore idxStore = pageStoreFactory.createPageStore(PageMemory.FLAG_IDX, idxFile);
 
@@ -686,6 +700,31 @@ public PageStore getStore(int grpId, int partId) throws IgniteCheckedException {
         return store;
     }
 
+    /**
+     * @param pageStoreFileIoFactory File IO factory to override default, may be used for blocked read-write.
+     * @param pageStoreV1FileIoFactory File IO factory for reading V1 page store and for fast touching page files
+     *      (non blocking).
+     */
+    public void setPageStoreFileIOFactories(final FileIOFactory pageStoreFileIoFactory,
+        final FileIOFactory pageStoreV1FileIoFactory) {
+        this.pageStoreFileIoFactory = pageStoreFileIoFactory;
+        this.pageStoreV1FileIoFactory = pageStoreV1FileIoFactory;
+    }
+
+    /**
+     * @return File IO factory currently selected for page store.
+     */
+    public FileIOFactory getPageStoreFileIoFactory() {
+        return pageStoreFileIoFactory;
+    }
+
+    /**
+     * @return Durable memory page size in bytes.
+     */
+    public int pageSize() {
+        return dsCfg.getPageSize();
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
index bab2cf07fe65d..bb25ab0b844ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
@@ -32,30 +32,46 @@ public class FileVersionCheckingFactory implements FilePageStoreFactory {
     public static final String LATEST_VERSION_OVERRIDE_PROPERTY = "file.page.store.latest.version.override";
 
     /** Latest page store version. */
-    public final static int LATEST_VERSION = 2;
+    public static final int LATEST_VERSION = 2;
 
     /** Factory to provide I/O interfaces for read/write operations with files. */
     private final FileIOFactory fileIOFactory;
 
+    /**
+     * Factory to provide I/O interfaces for read/write operations with files.
+     * This is backup factory for V1 page store.
+     */
+    private FileIOFactory fileIOFactoryStoreV1;
+
     /** Memory configuration. */
     private final DataStorageConfiguration memCfg;
 
     /**
-     * @param fileIOFactory File io factory.
+     * @param fileIOFactory File IO factory.
+     * @param fileIOFactoryStoreV1 File IO factory for V1 page store and for version checking.
      * @param memCfg Memory configuration.
      */
-    public FileVersionCheckingFactory(
-        FileIOFactory fileIOFactory, DataStorageConfiguration memCfg) {
+    public FileVersionCheckingFactory(FileIOFactory fileIOFactory, FileIOFactory fileIOFactoryStoreV1,
+        DataStorageConfiguration memCfg) {
         this.fileIOFactory = fileIOFactory;
+        this.fileIOFactoryStoreV1 = fileIOFactoryStoreV1;
         this.memCfg = memCfg;
     }
 
+    /**
+     * @param fileIOFactory File IO factory for V1 & V2 page store and for version checking.
+     * @param memCfg Memory configuration.
+     */
+    public FileVersionCheckingFactory(FileIOFactory fileIOFactory, DataStorageConfiguration memCfg) {
+        this(fileIOFactory, fileIOFactory, memCfg);
+    }
+
     /** {@inheritDoc} */
     @Override public FilePageStore createPageStore(byte type, File file) throws IgniteCheckedException {
         if (!file.exists())
             return createPageStore(type, file, latestVersion());
 
-        try (FileIO fileIO = fileIOFactory.create(file)) {
+        try (FileIO fileIO = fileIOFactoryStoreV1.create(file)) {
             int minHdr = FilePageStore.HEADER_SIZE;
 
             if (fileIO.size() < minHdr)
@@ -101,16 +117,16 @@ public int latestVersion() {
      * @param file File.
      * @param ver Version.
      */
-    public FilePageStore createPageStore(byte type, File file, int ver) throws IgniteCheckedException {
+    public FilePageStore createPageStore(byte type, File file, int ver) {
         switch (ver) {
             case FilePageStore.VERSION:
-                return new FilePageStore(type, file, fileIOFactory, memCfg);
+                return new FilePageStore(type, file, fileIOFactoryStoreV1, memCfg);
 
             case FilePageStoreV2.VERSION:
                 return new FilePageStoreV2(type, file, fileIOFactory, memCfg);
 
             default:
-                throw new IllegalArgumentException("Unknown version of file page store: " + ver);
+                throw new IllegalArgumentException("Unknown version of file page store: " + ver + " for file [" + file.getAbsolutePath() + "]");
         }
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 5ae0226b9b451..444a207ff4b66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -227,7 +227,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     /** */
     private final boolean alwaysWriteFullPages;
 
-    /** WAL segment size in bytes */
+    /** WAL segment size in bytes. . This is maximum value, actual segments may be shorter. */
     private final long maxWalSegmentSize;
 
     /** */
@@ -309,7 +309,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     private AtomicLong lastRecordLoggedMs = new AtomicLong();
 
     /**
-     * Cancellable task for {@link WALMode#BACKGROUND}, should be cancelled at shutdown Null for non background modes
+     * Cancellable task for {@link WALMode#BACKGROUND}, should be cancelled at shutdown.
+     * Null for non background modes.
      */
     @Nullable private volatile GridTimeoutProcessor.CancelableTask backgroundFlushSchedule;
 
@@ -322,6 +323,11 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     /** WAL writer worker. */
     private WALWriter walWriter;
 
+    /**
+     * Listener invoked for each segment file IO initializer.
+     */
+    @Nullable private volatile IgniteInClosure createWalFileListener;
+
     /**
      * @param ctx Kernal context.
      */
@@ -1048,6 +1054,11 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
         try {
             FileIO fileIO = ioFactory.create(curFile);
 
+            IgniteInClosure lsnr = createWalFileListener;
+
+            if (lsnr != null)
+                lsnr.apply(fileIO);
+
             try {
                 int serVer = serializerVer;
 
@@ -1137,6 +1148,10 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageE
                 try {
                     fileIO = ioFactory.create(nextFile);
 
+                    IgniteInClosure lsnr = createWalFileListener;
+                    if (lsnr != null)
+                        lsnr.apply(fileIO);
+
                     if (mmap) {
                         try {
                             MappedByteBuffer buf = fileIO.map((int)maxWalSegmentSize);
@@ -1340,6 +1355,21 @@ private void checkEnvironment() throws StorageException {
                 "previous error)", envFailed);
     }
 
+    /**
+     * Setup listener for WAL segment write File IO creation.
+     * @param createWalFileListener Listener to be invoked for new segment file IO creation.
+     */
+    public void setCreateWalFileListener(@Nullable IgniteInClosure createWalFileListener) {
+        this.createWalFileListener = createWalFileListener;
+    }
+
+    /**
+     * @return {@link #maxWalSegmentSize}.
+     */
+    public long maxWalSegmentSize() {
+        return maxWalSegmentSize;
+    }
+
     /**
      * File archiver operates on absolute segment indexes. For any given absolute segment index N we can calculate the
      * work WAL segment: S(N) = N % dsCfg.walSegments. When a work segment is finished, it is given to the archiver. If
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 267cb18c2bed8..27739a27a2308 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -122,7 +122,7 @@ else if (testDiscoSpi)
         }
 
         DataStorageConfiguration memCfg = new DataStorageConfiguration();
-        memCfg.setPageSize(1024);
+        memCfg.setPageSize(4 * 1024);
         memCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
             .setMaxSize(10 * 1024 * 1024)
             .setPersistenceEnabled(persistenceEnabled()));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
index 93fa24ef5f036..5609995ac8ab6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
@@ -85,7 +85,7 @@ public class IgniteDataStorageMetricsSelfTest extends GridCommonAbstractTest {
                 .setMetricsEnabled(true)
                 .setName("no-persistence"))
             .setWalMode(WALMode.LOG_ONLY)
-            .setPageSize(1024)
+            .setPageSize(4 * 1024)
             .setMetricsEnabled(true);
 
         cfg.setDataStorageConfiguration(memCfg);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsClientNearCachePutGetTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsClientNearCachePutGetTest.java
index 130a91c3ae7d0..8bc91558ba1d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsClientNearCachePutGetTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsClientNearCachePutGetTest.java
@@ -17,41 +17,24 @@
 
 package org.apache.ignite.internal.processors.cache.persistence;
 
-import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.processors.database.IgniteDbClientNearCachePutGetTest;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+import org.apache.ignite.internal.processors.database.IgniteDbPutGetAbstractTest;
 
 /**
  *
  */
-public class IgnitePdsClientNearCachePutGetTest extends IgniteDbClientNearCachePutGetTest {
+public class IgnitePdsClientNearCachePutGetTest extends IgniteDbPutGetAbstractTest {
     /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        cfg.setDataStorageConfiguration(
-            new DataStorageConfiguration()
-                .setWalMode(WALMode.LOG_ONLY)
-        );
-
-        return cfg;
+    @Override protected int gridCount() {
+        return 1;
     }
 
     /** {@inheritDoc} */
-    @Override protected void beforeTest() throws Exception {
-        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
-
-        super.beforeTest();
+    @Override protected boolean indexingEnabled() {
+        return false;
     }
 
     /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        super.afterTest();
-
-        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+    @Override protected boolean withClientNearCache() {
+        return true;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDynamicCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDynamicCacheTest.java
index 7e0cf8270043e..ba1933f68cca6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDynamicCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDynamicCacheTest.java
@@ -49,7 +49,7 @@ public class IgnitePdsDynamicCacheTest extends IgniteDbDynamicCacheSelfTest {
             .setDefaultDataRegionConfiguration(
                 new DataRegionConfiguration().setMaxSize(200 * 1024 * 1024).setPersistenceEnabled(true))
             .setWalMode(WALMode.LOG_ONLY)
-            .setPageSize(1024);
+            .setPageSize(4 * 1024);
 
         cfg.setDataStorageConfiguration(memCfg);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsSingleNodePutGetPersistenceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsSingleNodePutGetPersistenceTest.java
index 18e31fcbdcba2..59a6f84035174 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsSingleNodePutGetPersistenceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsSingleNodePutGetPersistenceTest.java
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache.persistence;
 
-import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.processors.database.IgniteDbSingleNodePutGetTest;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -29,18 +26,6 @@
  *
  */
 public class IgnitePdsSingleNodePutGetPersistenceTest extends IgniteDbSingleNodePutGetTest {
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        cfg.setDataStorageConfiguration(
-            new DataStorageConfiguration()
-            .setWalMode(WALMode.LOG_ONLY)
-        );
-
-        return cfg;
-    }
-
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java
index 577cf9a27376a..7da765b61b2dc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsCacheRestoreTest.java
@@ -61,7 +61,7 @@ public class IgnitePdsCacheRestoreTest extends GridCommonAbstractTest {
         DataStorageConfiguration memCfg = new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(
                 new DataRegionConfiguration().setMaxSize(10 * 1024 * 1024).setPersistenceEnabled(true))
-            .setPageSize(1024)
+            .setPageSize(4 * 1024)
             .setWalMode(WALMode.LOG_ONLY);
 
         memCfg.setDataRegionConfigurations(new DataRegionConfiguration()
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
index 34662a7c69c9f..782949ffffe14 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -130,11 +131,21 @@ public void testManyCachesAndNotManyPuts() throws Exception {
 
                 ignite.cache(fullname).put(d, d);
 
-                log.info("Put to " + fullname + " value " + d);
+                if (log.isInfoEnabled())
+                    log.info("Put to cache [" + fullname + "] value " + d);
 
-                db.wakeupForCheckpoint("").get();
+                final int timeout = 5000;
+                try {
+                    db.wakeupForCheckpoint("").get(timeout, TimeUnit.MILLISECONDS);
+                }
+                catch (IgniteFutureTimeoutCheckedException e) {
+                    continue;
+                }
 
-                int currCpPages = waitForCurrentCheckpointPagesCounterUpdated(db);
+                int currCpPages = waitForCurrentCheckpointPagesCounterUpdated(db, timeout);
+
+                if (currCpPages < 0)
+                    continue;
 
                 pageCntObserved.add(currCpPages);
 
@@ -158,15 +169,23 @@ public void testManyCachesAndNotManyPuts() throws Exception {
     }
 
     /**
+     * Waits counter of pages will be set up. If it is not changed for timeout milliseconds, method returns negative
+     * value.
+     *
      * @param db DB shared manager.
-     * @return counter when it becomes non-zero.
+     * @param timeout milliseconds to wait.
+     * @return counter when it becomes non-zero, negative value indicates timeout during wait for update.
      */
-    private int waitForCurrentCheckpointPagesCounterUpdated(GridCacheDatabaseSharedManager db) {
+    private int waitForCurrentCheckpointPagesCounterUpdated(GridCacheDatabaseSharedManager db, int timeout) {
         int currCpPages = 0;
+        long start = System.currentTimeMillis();
 
         while (currCpPages == 0) {
             LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1));
             currCpPages = db.currentCheckpointPagesCount();
+
+            if (currCpPages == 0 && ((System.currentTimeMillis() - start) > timeout))
+                return -1;
         }
 
         return currCpPages;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimpleTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimpleTest.java
new file mode 100644
index 0000000000000..4655fccf376f7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimpleTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.file;
+
+import com.google.common.base.Strings;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Puts data into grid, waits for checkpoint to start and then verifies data
+ */
+public class IgnitePdsCheckpointSimpleTest extends GridCommonAbstractTest {
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        DataRegionConfiguration regCfg = new DataRegionConfiguration().setPersistenceEnabled(true);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+            .setPageSize(4 * 1024)
+            .setDefaultDataRegionConfiguration(regCfg)
+            .setCheckpointFrequency(TimeUnit.SECONDS.toMillis(10));
+
+        return cfg.setDataStorageConfiguration(dsCfg);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * Checks if same data can be loaded after checkpoint.
+     * @throws Exception if failed.
+     */
+    public void testRecoveryAfterCpEnd() throws Exception {
+        IgniteEx ignite = startGrid(0);
+
+        ignite.active(true);
+
+        IgniteCache cache = ignite.getOrCreateCache("cache");
+
+        for (int i = 0; i < 10000; i++)
+            cache.put(i, valueWithRedundancyForKey(i));
+
+        ignite.context().cache().context().database().waitForCheckpoint("test");
+
+        stopAllGrids();
+
+        IgniteEx igniteRestart = startGrid(0);
+        igniteRestart.active(true);
+
+        IgniteCache cacheRestart = igniteRestart.getOrCreateCache("cache");
+
+        for (int i = 0; i < 10000; i++)
+            assertEquals(valueWithRedundancyForKey(i), cacheRestart.get(i));
+
+        stopAllGrids();
+    }
+
+    /**
+     * @param i key.
+     * @return value with extra data, which allows to verify
+     */
+    @NotNull private String valueWithRedundancyForKey(int i) {
+        return Strings.repeat(Integer.toString(i), 10);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
index 1b86e3da056bf..117be9bb189a1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
@@ -52,7 +52,7 @@ public class IgnitePdsEvictionTest extends GridCommonAbstractTest {
     private static final int NUMBER_OF_SEGMENTS = 64;
 
     /** */
-    private static final int PAGE_SIZE = 1024;
+    private static final int PAGE_SIZE = 4 * 1024;
 
     /** */
     private static final long CHUNK_SIZE = 1024 * 1024;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 1d5b624b429b7..9ce3077beed30 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -22,21 +22,23 @@
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.cluster.IgniteClusterEx;
 import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
 
 /**
  *
@@ -73,7 +75,12 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
         if (isLargePage())
             dbCfg.setPageSize(16 * 1024);
         else
-            dbCfg.setPageSize(1024);
+            dbCfg.setPageSize(4 * 1024);
+
+        dbCfg.setWalMode(WALMode.LOG_ONLY);
+
+        dbCfg.setDefaultDataRegionConfiguration(
+            new DataRegionConfiguration().setPersistenceEnabled(true).setName("default"));
 
         configure(dbCfg);
 
@@ -151,7 +158,7 @@ protected void configure(IgniteConfiguration cfg){
     /**
      * @param mCfg DataStorageConfiguration.
      */
-    protected void configure(DataStorageConfiguration mCfg){
+    protected void configure(DataStorageConfiguration mCfg) {
         // No-op.
     }
 
@@ -164,7 +171,7 @@ protected boolean withClientNearCache() {
 
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
-        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+        GridTestUtils.deleteDbFiles();
 
         startGrids(gridCount());
 
@@ -178,7 +185,15 @@ protected boolean withClientNearCache() {
 
         assert gridCount() > 0;
 
-        grid(0).active(true);
+        final IgniteClusterEx cluster = grid(0).cluster();
+
+        if (log.isInfoEnabled())
+            log.info("BTL before activation: " + cluster.currentBaselineTopology());
+
+        cluster.active(true);
+
+        if (log.isInfoEnabled())
+            log.info("BTL after activation: " + cluster.currentBaselineTopology());
 
         awaitPartitionMapExchange();
     }
@@ -189,7 +204,7 @@ protected boolean withClientNearCache() {
 
         stopAllGrids();
 
-        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+        GridTestUtils.deleteDbFiles();
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbDynamicCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbDynamicCacheSelfTest.java
index e5c0e8ac3e34d..00c22406ad961 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbDynamicCacheSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbDynamicCacheSelfTest.java
@@ -31,6 +31,8 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
 /**
  *
  */
@@ -40,7 +42,7 @@ public class IgniteDbDynamicCacheSelfTest extends GridCommonAbstractTest {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
         DataStorageConfiguration memCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration(
-            new DataRegionConfiguration().setMaxSize(200 * 1024 * 1024));
+            new DataRegionConfiguration().setMaxSize(200 * 1024 * 1024).setPersistenceEnabled(true));
 
         cfg.setDataStorageConfiguration(memCfg);
 
@@ -57,6 +59,13 @@ public class IgniteDbDynamicCacheSelfTest extends GridCommonAbstractTest {
         stopAllGrids();
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         super.afterTest();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index c4e8bee951efc..81b5515f29044 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -79,7 +79,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
     @Override protected void configure(DataStorageConfiguration mCfg) {
         mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
 
-        long size = (1024 * (isLargePage() ? 16 : 1) + 24) * pagesMax();
+        long size = (1024 * (isLargePage() ? 16 : 4) + 24) * pagesMax();
 
         mCfg.setDefaultDataRegionConfiguration(
             new DataRegionConfiguration().setMaxSize(Math.max(size, MIN_PAGE_CACHE_SIZE)).setName("default"));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
index 0b0c7633b6817..84455bab45f83 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetAbstractTest.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 59425c8df5691..8e022795b6626 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -33,7 +33,6 @@
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottleSmokeTest;
 import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBufferTest;
-import org.apache.ignite.internal.processors.database.IgniteDbClientNearCachePutGetTest;
 import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest;
 import org.apache.ignite.internal.processors.database.IgniteDbMultiNodePutGetTest;
 import org.apache.ignite.internal.processors.database.IgniteDbPutGetWithCacheStoreTest;
@@ -52,25 +51,43 @@ public class IgnitePdsTestSuite extends TestSuite {
     public static TestSuite suite() throws Exception {
         TestSuite suite = new TestSuite("Ignite Persistent Store Test Suite");
 
+        addRealPageStoreTests(suite);
+
         // Basic PageMemory tests.
         suite.addTestSuite(PageMemoryImplNoLoadTest.class);
         suite.addTestSuite(IndexStoragePageMemoryImplTest.class);
         suite.addTestSuite(IgnitePdsEvictionTest.class);
         suite.addTestSuite(PageMemoryImplTest.class);
 
-        // Checkpointing smoke-test.
-        suite.addTestSuite(IgnitePdsCheckpointSimulationWithRealCpDisabledTest.class);
-
         // BTree tests with store page memory.
         suite.addTestSuite(BPlusTreePageMemoryImplTest.class);
         suite.addTestSuite(BPlusTreeReuseListPageMemoryImplTest.class);
 
+        suite.addTestSuite(SegmentedRingByteBufferTest.class);
+
+        // Write throttling
+        suite.addTestSuite(PagesWriteThrottleSmokeTest.class);
+
+        return suite;
+    }
+
+    /**
+     * Fills {@code suite} with PDS test subset, which operates with real page store and does actual disk operations.
+     *
+     * @param suite suite to add tests into.
+     */
+    public static void addRealPageStoreTests(TestSuite suite) {
+        // Basic PageMemory tests.
+        suite.addTestSuite(IgnitePdsEvictionTest.class);
+
+        // Checkpointing smoke-test.
+        suite.addTestSuite(IgnitePdsCheckpointSimulationWithRealCpDisabledTest.class);
+
         // Basic API tests.
         suite.addTestSuite(IgniteDbSingleNodePutGetTest.class);
         suite.addTestSuite(IgniteDbMultiNodePutGetTest.class);
         suite.addTestSuite(IgniteDbSingleNodeTinyPutGetTest.class);
         suite.addTestSuite(IgniteDbDynamicCacheSelfTest.class);
-        suite.addTestSuite(IgniteDbClientNearCachePutGetTest.class);
 
         // Persistence-enabled.
         suite.addTestSuite(IgnitePdsSingleNodePutGetPersistenceTest.class);
@@ -83,12 +100,5 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgnitePdsCacheRestoreTest.class);
 
         suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
-
-        suite.addTestSuite(SegmentedRingByteBufferTest.class);
-
-        // Write throttling
-        suite.addTestSuite(PagesWriteThrottleSmokeTest.class);
-
-        return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 8dc318a94a3a5..3852a16283f4f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -50,13 +50,30 @@
 public class IgnitePdsTestSuite2 extends TestSuite {
     /**
      * @return Suite.
-     * @throws Exception If failed.
      */
-    public static TestSuite suite() throws Exception {
+    public static TestSuite suite() {
         TestSuite suite = new TestSuite("Ignite persistent Store Test Suite 2");
 
         // Integrity test.
         suite.addTestSuite(IgniteDataIntegrityTests.class);
+
+        addRealPageStoreTests(suite);
+
+        // BaselineTopology tests
+        suite.addTestSuite(IgniteAllBaselineNodesOnlineFullApiSelfTest.class);
+        suite.addTestSuite(IgniteOfflineBaselineNodeFullApiSelfTest.class);
+        suite.addTestSuite(IgniteOnlineNodeOutOfBaselineFullApiSelfTest.class);
+
+        return suite;
+    }
+
+    /**
+     * Fills {@code suite} with PDS test subset, which operates with real page store and does actual disk operations.
+     *
+     * @param suite suite to add tests into.
+     */
+    public static void addRealPageStoreTests(TestSuite suite) {
+        // Integrity test.
         suite.addTestSuite(IgnitePdsRecoveryAfterFileCorruptionTest.class);
         suite.addTestSuite(IgnitePdsPageSizesTest.class);
 
@@ -89,6 +106,7 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteWalFlushLogOnlySelfTest.class);
 
+        // Test suite uses Standalone WAL iterator to verify PDS content.
         suite.addTestSuite(IgniteWalReaderTest.class);
 
         suite.addTestSuite(IgnitePdsExchangeDuringCheckpointTest.class);
@@ -98,15 +116,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteWalSerializerVersionTest.class);
 
-        // BaselineTopology tests
-        suite.addTestSuite(IgniteAllBaselineNodesOnlineFullApiSelfTest.class);
-        suite.addTestSuite(IgniteOfflineBaselineNodeFullApiSelfTest.class);
-        suite.addTestSuite(IgniteOnlineNodeOutOfBaselineFullApiSelfTest.class);
-
         suite.addTestSuite(WalCompactionTest.class);
 
         suite.addTestSuite(IgniteCheckpointDirtyPagesForLowLoadTest.class);
-
-        return suite;
     }
 }
diff --git a/modules/direct-io/pom.xml b/modules/direct-io/pom.xml
new file mode 100644
index 0000000000000..1d246722e108d
--- /dev/null
+++ b/modules/direct-io/pom.xml
@@ -0,0 +1,122 @@
+
+
+
+
+
+
+    4.0.0
+
+    
+        org.apache.ignite
+        ignite-parent
+        1
+        ../../parent
+    
+
+    ignite-direct-io
+    2.4.0-SNAPSHOT
+    http://ignite.apache.org
+
+    
+
+        
+            org.apache.ignite
+            ignite-core
+            ${project.version}
+        
+
+
+        
+            org.apache.ignite
+            ignite-core
+            ${project.version}
+            test-jar
+            test
+        
+
+        
+            org.apache.ignite
+            ignite-indexing
+            ${project.version}
+        
+
+        
+            org.springframework
+            spring-beans
+            ${spring.version}
+            test
+        
+
+        
+            org.springframework
+            spring-context
+            ${spring.version}
+            test
+        
+
+        
+            net.java.dev.jna
+            jna
+            4.5.0
+        
+
+        
+            com.google.guava
+            guava
+            ${guava.version}
+            test
+        
+
+        
+            log4j
+            log4j
+            test
+        
+
+        
+            com.thoughtworks.xstream
+            xstream
+            1.4.8
+            test
+        
+
+        
+            org.mockito
+            mockito-all
+            1.9.5
+            test
+        
+    
+
+
+    
+        
+            
+                org.apache.maven.plugins
+                maven-deploy-plugin
+                2.8.2
+                
+                    true
+                
+            
+        
+    
+
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffers.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffers.java
new file mode 100644
index 0000000000000..4bc7b08417a7c
--- /dev/null
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffers.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import com.sun.jna.NativeLong;
+import com.sun.jna.Pointer;
+import com.sun.jna.ptr.PointerByReference;
+import java.nio.ByteBuffer;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.util.GridUnsafe;
+
+/**
+ * Utility class for work with aligned buffers.
+ */
+@SuppressWarnings("WeakerAccess")
+public class AlignedBuffers {
+    /**
+     * Allocates align memory for use with O_DIRECT and returns native byte buffer.
+     * @param fsBlockSize alignment, FS ans OS block size.
+     * @param size capacity.
+     * @return byte buffer, to be released by {@link #free(ByteBuffer)}.
+     */
+    public static ByteBuffer allocate(int fsBlockSize, int size) {
+        assert fsBlockSize > 0;
+        assert size > 0;
+
+        PointerByReference refToPtr = new PointerByReference();
+
+        int retVal = IgniteNativeIoLib.posix_memalign(refToPtr, new NativeLong(fsBlockSize),
+            new NativeLong(size));
+
+        if (retVal != 0)
+            throw new IgniteOutOfMemoryException("Failed to allocate memory: " + IgniteNativeIoLib.strerror(retVal));
+
+        return GridUnsafe.wrapPointer(Pointer.nativeValue(refToPtr.getValue()), size);
+    }
+
+    /**
+     * Frees the memory space used by direct buffer, which must have been returned by a previous call
+     * {@link #allocate(int, int)}.
+     *
+     * @param buf direct buffer to free.
+     */
+    public static void free(ByteBuffer buf) {
+        free(GridUnsafe.bufferAddress(buf));
+    }
+
+    /**
+     * Frees the memory space pointed to by {@code addr} - address of buffer, which must have been returned by a
+     * previous call {@link #allocate(int, int)}.
+     *
+     * @param addr direct buffer address to free.
+     */
+    public static void free(long addr) {
+        IgniteNativeIoLib.free(new Pointer(addr));
+    }
+}
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
new file mode 100644
index 0000000000000..62e1db3e95985
--- /dev/null
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import com.sun.jna.Native;
+import com.sun.jna.NativeLong;
+import com.sun.jna.Pointer;
+import java.io.File;
+import java.io.IOException;
+import java.nio.BufferOverflowException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.file.OpenOption;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
+import org.jsr166.ConcurrentHashMap8;
+
+/**
+ * Limited capabilities Direct IO, which enables file write and read using aligned buffers and O_DIRECT mode.
+ *
+ * Works only for Linux
+ */
+public class AlignedBuffersDirectFileIO implements FileIO {
+    /** Negative value for file offset: read/write starting from current file position */
+    private static final int FILE_POS_USE_CURRENT = -1;
+
+    /** File system & linux block size. Minimal amount of data can be written using DirectIO. */
+    private final int fsBlockSize;
+
+    /** Durable memory Page size. Can have greater value than {@link #fsBlockSize}. */
+    private final int pageSize;
+
+    /** File. */
+    private final File file;
+
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /** Thread local with buffers with capacity = one page {@link #pageSize} and aligned using {@link #fsBlockSize}. */
+    private ThreadLocal tlbOnePageAligned;
+
+    /** Managed aligned buffers. Used to check if buffer is applicable for direct IO our data should be copied. */
+    private ConcurrentHashMap8 managedAlignedBuffers;
+
+    /** File descriptor. */
+    private int fd = -1;
+
+    /** Number of instances to cache */
+    private static final int CACHED_LONGS = 512;
+
+    /** Value used as divisor in {@link #nativeLongCache}. Native longs divisible by this value will be cached. */
+    private static final int NL_CACHE_DIVISOR = 4096;
+
+    /** Native long instance cache. */
+    private static final AtomicReferenceArray nativeLongCache = new AtomicReferenceArray<>(CACHED_LONGS);
+
+    /**
+     * Creates Direct File IO.
+     *
+     * @param fsBlockSize FS/OS block size.
+     * @param pageSize Durable memory Page size.
+     * @param file File to open.
+     * @param modes Open options (flags).
+     * @param tlbOnePageAligned Thread local with buffers with capacity = one page {@code pageSize} and aligned using
+     * {@code fsBlockSize}.
+     * @param managedAlignedBuffers Managed aligned buffers map, used to check if buffer is known.
+     * @param log Logger.
+     * @throws IOException if file open failed.
+     */
+    AlignedBuffersDirectFileIO(
+        int fsBlockSize,
+        int pageSize,
+        File file,
+        OpenOption[] modes,
+        ThreadLocal tlbOnePageAligned,
+        ConcurrentHashMap8 managedAlignedBuffers,
+        IgniteLogger log)
+        throws IOException {
+        this.log = log;
+        this.fsBlockSize = fsBlockSize;
+        this.pageSize = pageSize;
+        this.file = file;
+        this.tlbOnePageAligned = tlbOnePageAligned;
+        this.managedAlignedBuffers = managedAlignedBuffers;
+
+        String pathname = file.getAbsolutePath();
+
+        int openFlags = setupOpenFlags(modes, log, true);
+        int mode = IgniteNativeIoLib.DEFAULT_OPEN_MODE;
+        int fd = IgniteNativeIoLib.open(pathname, openFlags, mode);
+
+        if (fd < 0) {
+            int error = Native.getLastError();
+            String msg = "Error opening file [" + pathname + "] with flags [0x"
+                + String.format("%2X", openFlags) + ": DIRECT & " + Arrays.asList(modes)
+                + "], got error [" + error + ": " + getLastError() + "]";
+
+            if (error == IgniteNativeIoLib.E_INVAL) {
+                openFlags = setupOpenFlags(modes, log, false);
+                fd = IgniteNativeIoLib.open(pathname, openFlags, mode);
+
+                if (fd > 0) {
+                    U.warn(log, "Disable Direct IO mode for path " + file.getParentFile() +
+                        "(probably incompatible file system selected, for example, tmpfs): " + msg);
+
+                    this.fd = fd;
+
+                    return;
+                }
+            }
+
+            throw new IOException(msg);
+        }
+
+        this.fd = fd;
+    }
+
+    /**
+     * Convert Java open options to native flags.
+     *
+     * @param modes java options.
+     * @param log logger.
+     * @param enableDirect flag for enabling option {@link IgniteNativeIoLib#O_DIRECT} .
+     * @return native flags for open method.
+     */
+    private static int setupOpenFlags(OpenOption[] modes, IgniteLogger log, boolean enableDirect) {
+        int flags = enableDirect ? IgniteNativeIoLib.O_DIRECT : 0;
+        List openOptionList = Arrays.asList(modes);
+
+        for (OpenOption mode : openOptionList) {
+            if (mode == StandardOpenOption.READ) {
+                flags |= openOptionList.contains(StandardOpenOption.WRITE)
+                    ? IgniteNativeIoLib.O_RDWR
+                    : IgniteNativeIoLib.O_RDONLY;
+            }
+            else if (mode == StandardOpenOption.WRITE) {
+                flags |= openOptionList.contains(StandardOpenOption.READ)
+                    ? IgniteNativeIoLib.O_RDWR
+                    : IgniteNativeIoLib.O_WRONLY;
+            }
+            else if (mode == StandardOpenOption.CREATE)
+                flags |= IgniteNativeIoLib.O_CREAT;
+            else if (mode == StandardOpenOption.TRUNCATE_EXISTING)
+                flags |= IgniteNativeIoLib.O_TRUNC;
+            else if (mode == StandardOpenOption.SYNC)
+                flags |= IgniteNativeIoLib.O_SYNC;
+            else
+                log.error("Unsupported open option [" + mode + "]");
+        }
+
+        return flags;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long position() throws IOException {
+        long position = IgniteNativeIoLib.lseek(fdCheckOpened(), 0, IgniteNativeIoLib.SEEK_CUR);
+
+        if (position < 0) {
+            throw new IOException(String.format("Error checking file [%s] position: %s",
+                file, getLastError()));
+        }
+
+        return position;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void position(long newPosition) throws IOException {
+        if (IgniteNativeIoLib.lseek(fdCheckOpened(), newPosition, IgniteNativeIoLib.SEEK_SET) < 0) {
+            throw new IOException(String.format("Error setting file [%s] position to [%s]: %s",
+                file, Long.toString(newPosition), getLastError()));
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read(ByteBuffer destBuf) throws IOException {
+        return read(destBuf, FILE_POS_USE_CURRENT);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read(ByteBuffer destBuf, long filePosition) throws IOException {
+        int size = checkSizeIsPadded(destBuf.remaining());
+
+        if (isKnownAligned(destBuf))
+            return readIntoAlignedBuffer(destBuf, filePosition);
+
+        boolean useTlb = size == pageSize;
+        ByteBuffer alignedBuf = useTlb ? tlbOnePageAligned.get() : AlignedBuffers.allocate(fsBlockSize, size);
+
+        try {
+            assert alignedBuf.position() == 0: "Temporary aligned buffer is in incorrect state: position is set incorrectly";
+            assert alignedBuf.limit() == size: "Temporary aligned buffer is in incorrect state: limit is set incorrectly";
+
+            int loaded = readIntoAlignedBuffer(alignedBuf, filePosition);
+
+            alignedBuf.flip();
+
+            if (loaded > 0)
+                  destBuf.put(alignedBuf);
+
+            return loaded;
+        }
+        finally {
+            alignedBuf.clear();
+
+            if (!useTlb)
+                AlignedBuffers.free(alignedBuf);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read(byte[] buf, int off, int len) throws IOException {
+        return read(ByteBuffer.wrap(buf, off, len));
+    }
+
+    /** {@inheritDoc} */
+    @Override public int write(ByteBuffer srcBuf) throws IOException {
+        return write(srcBuf, FILE_POS_USE_CURRENT);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int write(ByteBuffer srcBuf, long filePosition) throws IOException {
+        int size = checkSizeIsPadded(srcBuf.remaining());
+
+        if (isKnownAligned(srcBuf))
+            return writeFromAlignedBuffer(srcBuf, filePosition);
+
+        boolean useTlb = size == pageSize;
+        ByteBuffer alignedBuf = useTlb ? tlbOnePageAligned.get() : AlignedBuffers.allocate(fsBlockSize, size);
+
+        try {
+            assert alignedBuf.position() == 0 : "Temporary aligned buffer is in incorrect state: position is set incorrectly";
+            assert alignedBuf.limit() == size : "Temporary aligned buffer is in incorrect state: limit is set incorrectly";
+
+            int initPos = srcBuf.position();
+
+            alignedBuf.put(srcBuf);
+            alignedBuf.flip();
+
+            srcBuf.position(initPos); // will update later from write results
+
+            int written = writeFromAlignedBuffer(alignedBuf, filePosition);
+
+            if (written > 0)
+                srcBuf.position(initPos + written);
+
+            return written;
+        }
+        finally {
+            alignedBuf.clear();
+
+            if (!useTlb)
+                AlignedBuffers.free(alignedBuf);
+        }
+    }
+
+    /**
+     * Checks if we can run fast path: we got well known buffer is already aligned.
+     *
+     * @param srcBuf buffer to check if it is known buffer.
+     * @return {@code true} if this buffer was allocated with alignment, may be used directly.
+     */
+    private boolean isKnownAligned(ByteBuffer srcBuf) {
+        return srcBuf.isDirect()
+            && managedAlignedBuffers != null
+            && managedAlignedBuffers.containsKey(GridUnsafe.bufferAddress(srcBuf));
+    }
+
+    /**
+     * Check if size is appropriate for aligned/direct IO.
+     *
+     * @param size buffer size to write, should be divisible by {@link #fsBlockSize}.
+     * @return size from parameter.
+     * @throws IOException if provided size can't be written using direct IO.
+     */
+    private int checkSizeIsPadded(int size) throws IOException {
+        if (size % fsBlockSize != 0) {
+            throw new IOException(
+                String.format("Unable to apply DirectIO for read/write buffer [%d] bytes on file system " +
+                    "block size [%d]. Consider setting %s.setPageSize(%d) or disable Direct IO.",
+                    size, fsBlockSize, DataStorageConfiguration.class.getSimpleName(), fsBlockSize));
+        }
+
+        return size;
+    }
+
+    /**
+     * Checks if file is opened and returns descriptor.
+     *
+     * @return file descriptor.
+     * @throws IOException if file not opened.
+     */
+    private int fdCheckOpened() throws IOException {
+        if (fd < 0)
+            throw new IOException(String.format("Error %s not opened", file));
+
+        return fd;
+    }
+
+    /**
+     * Read bytes from file using Native IO and aligned buffers.
+     *
+     * @param destBuf Destination aligned byte buffer.
+     * @param filePos Starting position of file. Providing {@link #FILE_POS_USE_CURRENT} means it is required
+     *     to read from current file position.
+     * @return number of bytes read from file, or -1 if tried to read past EOF for file.
+     * @throws IOException if reading failed.
+     */
+    private int readIntoAlignedBuffer(ByteBuffer destBuf, long filePos) throws IOException {
+        int pos = destBuf.position();
+        int limit = destBuf.limit();
+        int toRead = pos <= limit ? limit - pos : 0;
+
+        if (toRead == 0)
+            return 0;
+
+        if ((pos + toRead) > destBuf.capacity())
+            throw new BufferOverflowException();
+
+        int rd;
+        Pointer ptr = bufferPtrAtPosition(destBuf, pos);
+
+        if (filePos == FILE_POS_USE_CURRENT)
+            rd = IgniteNativeIoLib.read(fdCheckOpened(), ptr, nl(toRead)).intValue();
+        else
+            rd = IgniteNativeIoLib.pread(fdCheckOpened(), ptr, nl(toRead), nl(filePos)).intValue();
+
+        if (rd == 0)
+            return -1; //Tried to read past EOF for file
+
+        if (rd < 0)
+            throw new IOException(String.format("Error during reading file [%s] from position [%s] : %s",
+                file, filePos == FILE_POS_USE_CURRENT ? "current" : Long.toString(filePos), getLastError()));
+
+        destBuf.position(pos + rd);
+
+        return rd;
+    }
+
+    /**
+     * Writes the aligned native buffer starting at {@code buf} to the file at offset
+     * {@code filePos}. The file offset is not changed.
+     *
+     * @param srcBuf pointer to buffer.
+     * @param filePos position in file to write data. Providing {@link #FILE_POS_USE_CURRENT} means it is required
+     *     to read from current file position.
+     * @return the number of bytes written.
+     */
+    private int writeFromAlignedBuffer(ByteBuffer srcBuf, long filePos) throws IOException {
+        int pos = srcBuf.position();
+        int limit = srcBuf.limit();
+        int toWrite = pos <= limit ? limit - pos : 0;
+
+        if (toWrite == 0)
+            return 0;
+
+        int wr;
+        Pointer ptr = bufferPtrAtPosition(srcBuf, pos);
+
+        if (filePos == FILE_POS_USE_CURRENT)
+            wr = IgniteNativeIoLib.write(fdCheckOpened(), ptr, nl(toWrite)).intValue();
+        else
+            wr = IgniteNativeIoLib.pwrite(fdCheckOpened(), ptr, nl(toWrite), nl(filePos)).intValue();
+
+        if (wr < 0) {
+            throw new IOException(String.format("Error during writing file [%s] to position [%s]: %s",
+                file, filePos == FILE_POS_USE_CURRENT ? "current" : Long.toString(filePos), getLastError()));
+        }
+
+        if ((pos + wr) > limit) {
+            throw new IllegalStateException(String.format("Write illegal state for file [%s]: pos=%d, wr=%d, limit=%d",
+                file, pos, wr, limit));
+        }
+
+        srcBuf.position(pos + wr);
+
+        return wr;
+    }
+
+    /**
+     * @param val value to box to native long.
+     * @return native long.
+     */
+    @NotNull private static NativeLong nl(long val) {
+        if (val % NL_CACHE_DIVISOR == 0 && val < CACHED_LONGS * NL_CACHE_DIVISOR) {
+            int cacheIdx = (int)(val / NL_CACHE_DIVISOR);
+
+            NativeLong curCached = nativeLongCache.get(cacheIdx);
+
+            if (curCached != null)
+                return curCached;
+
+            NativeLong nl = new NativeLong(val);
+
+            nativeLongCache.compareAndSet(cacheIdx, null, nl);
+
+            return nl;
+        }
+        return new NativeLong(val);
+    }
+
+    /**
+     * Retrieve last error set by the OS as string. This corresponds to and errno on
+     * *nix platforms.
+     * @return displayable string with OS error info.
+     */
+    private static String getLastError() {
+        return IgniteNativeIoLib.strerror(Native.getLastError());
+    }
+
+    /**
+     * Gets address in memory for direct aligned buffer taking into account its current {@code position()} as offset.
+     * Produces warnings if data or offset seems to be not aligned.
+     *
+     * @param buf Direct aligned buffer.
+     * @param pos position, used as offset for resulting pointer.
+     * @return Buffer memory address.
+     */
+    @NotNull private Pointer bufferPtrAtPosition(ByteBuffer buf, int pos) {
+        long alignedPointer = GridUnsafe.bufferAddress(buf);
+
+        if (pos < 0)
+            throw new IllegalArgumentException();
+
+        if (pos > buf.capacity())
+            throw new BufferOverflowException();
+
+        if ((alignedPointer + pos) % fsBlockSize != 0) {
+            U.warn(log, String.format("IO Buffer Pointer [%d] and/or offset [%d] seems to be not aligned " +
+                "for FS block size [%d]. Direct IO may fail.", alignedPointer, buf.position(), fsBlockSize));
+        }
+
+        return new Pointer(alignedPointer + pos);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void write(byte[] buf, int off, int len) throws IOException {
+        write(ByteBuffer.wrap(buf, off, len));
+    }
+
+    /** {@inheritDoc} */
+    @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
+        throw new UnsupportedOperationException("AsynchronousFileChannel doesn't support mmap.");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void force() throws IOException {
+        if (IgniteNativeIoLib.fsync(fdCheckOpened()) < 0)
+            throw new IOException(String.format("Error fsync()'ing %s, got %s", file, getLastError()));
+    }
+
+    /** {@inheritDoc} */
+    @Override public long size() throws IOException {
+        return file.length();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clear() throws IOException {
+        truncate(0);
+    }
+
+    /**
+     * Truncates this channel's file to the given size.
+     *
+     * 
 If the given size is less than the file's current size then the file
+     * is truncated, discarding any bytes beyond the new end of the file. If
+     * the given size is greater than or equal to the file's current size then
+     * the file is not modified. In either case, if this channel's file
+     * position is greater than the given size then it is set to that size.
+     * 

+     *
+     * @param  size The new size, a non-negative byte count
+     *
+     */
+    private void truncate(long size) throws IOException {
+        if (IgniteNativeIoLib.ftruncate(fdCheckOpened(), size) < 0)
+            throw new IOException(String.format("Error truncating file %s, got %s", file, getLastError()));
+
+        if (position() > size)
+            position(size);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() throws IOException {
+        if (IgniteNativeIoLib.close(fdCheckOpened()) < 0)
+            throw new IOException(String.format("Error closing %s, got %s", file, getLastError()));
+
+        fd = -1;
+    }
+}
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java
new file mode 100644
index 0000000000000..b9d11cd587b1d
--- /dev/null
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.OpenOption;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
+import org.jsr166.ConcurrentHashMap8;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/**
+ * Direct native IO factory for block IO operations on aligned memory structures.

+ * This limited functionality is used for page store operations.

+ * Note:  This type of IO not applicable for WAL or other files.
 

+ * This IO tries to minimize cache effects of the I/O (page caching by OS). 
 

+ * In general this will degrade performance, but it is useful in special
+ * situations, such as when applications do their own caching.

+ */
+public class AlignedBuffersDirectFileIOFactory implements FileIOFactory {
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /** Page size from durable memory. */
+    private final int pageSize;
+
+    /** Backup factory for files in case native is not available or not applicable. */
+    private final FileIOFactory backupFactory;
+
+    /** File system/os block size, negative value if library init was failed. */
+    private final int fsBlockSize;
+
+    /** Use backup factory, {@code true} if direct IO setup failed. */
+    private boolean useBackupFactory;
+
+    /** Thread local with buffers with capacity = one page {@code pageSize} and aligned using {@code fsBlockSize}. */
+    private ThreadLocal tlbOnePageAligned;
+
+    /**
+     * Managed aligned buffers. This collection is used to free buffers, an for checking if buffer is known to be
+     * already aligned.
+     */
+    private final ConcurrentHashMap8 managedAlignedBuffers = new ConcurrentHashMap8<>();
+
+    /**
+     * Creates direct native IO factory.
+     *
+     * @param log Logger.
+     * @param storePath Storage path, used to check FS settings.
+     * @param pageSize durable memory page size.
+     * @param backupFactory fallback factory if init failed.
+     */
+    public AlignedBuffersDirectFileIOFactory(
+        final IgniteLogger log,
+        final File storePath,
+        final int pageSize,
+        final FileIOFactory backupFactory) {
+        this.log = log;
+        this.pageSize = pageSize;
+        this.backupFactory = backupFactory;
+
+        useBackupFactory = true;
+        fsBlockSize = IgniteNativeIoLib.getFsBlockSize(storePath.getAbsolutePath(), log);
+
+        if(!IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_DIRECT_IO_ENABLED, true)) {
+            if (log.isInfoEnabled())
+                log.info("Direct IO is explicitly disabled by system property");
+
+            return;
+        }
+
+        if (fsBlockSize > 0) {
+            int blkSize = fsBlockSize;
+
+            if (pageSize % blkSize != 0) {
+                U.warn(log, String.format("Unable to setup Direct IO for Ignite [pageSize=%d bytes;" +
+                        " file system block size=%d]. For speeding up Ignite consider setting %s.setPageSize(%d)." +
+                        " Direct IO is disabled",
+                    pageSize, blkSize, DataStorageConfiguration.class.getSimpleName(), blkSize));
+            }
+            else {
+                useBackupFactory = false;
+
+                tlbOnePageAligned = new ThreadLocal() {
+                    @Override protected ByteBuffer initialValue() {
+                        return createManagedBuffer(pageSize);
+                    }
+                };
+
+                if (log.isInfoEnabled()) {
+                    log.info(String.format("Direct IO is enabled for block IO operations on aligned memory structures." +
+                        " [block size = %d, durable memory page size = %d]", blkSize, pageSize));
+                }
+            }
+        }
+        else {
+            if (log.isInfoEnabled()) {
+                log.info(String.format("Direct IO library is not available on current operating system [%s]." +
+                    " Direct IO is not enabled.", System.getProperty("os.version")));
+            }
+        }
+
+    }
+
+    /**
+     * Note:  Use only if {@link #isDirectIoAvailable()}.
+     *
+     * @param size buffer size to allocate.
+     * @return new byte buffer.
+     */
+    @NotNull ByteBuffer createManagedBuffer(int size) {
+        assert !useBackupFactory : "Direct IO is disabled, aligned managed buffer creation is disabled now";
+        assert managedAlignedBuffers != null : "Direct buffers not available";
+
+        ByteBuffer allocate = AlignedBuffers.allocate(fsBlockSize, size).order(ByteOrder.nativeOrder());
+
+        managedAlignedBuffers.put(GridUnsafe.bufferAddress(allocate), Thread.currentThread());
+
+        return allocate;
+    }
+
+    /** {@inheritDoc} */
+    @Override public FileIO create(File file) throws IOException {
+        return create(file, CREATE, READ, WRITE);
+    }
+
+    /** {@inheritDoc} */
+    @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+        if (useBackupFactory)
+            return backupFactory.create(file, modes);
+
+        return new AlignedBuffersDirectFileIO(fsBlockSize, pageSize, file, modes, tlbOnePageAligned, managedAlignedBuffers, log);
+
+    }
+
+    /**
+     * @return {@code true} if Direct IO can be used on current OS and file system settings
+     */
+    boolean isDirectIoAvailable() {
+        return !useBackupFactory;
+    }
+
+    /**
+     * Managed aligned buffers and its associated threads. This collection is used to free buffers, an for checking if
+     * buffer is known to be already aligned.
+     *
+     * @return map address->thread.
+     */
+    ConcurrentHashMap8 managedAlignedBuffers() {
+        return managedAlignedBuffers;
+    }
+}
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoLib.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoLib.java
new file mode 100644
index 0000000000000..47f1e6a03d72b
--- /dev/null
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoLib.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import com.sun.jna.Native;
+import com.sun.jna.NativeLong;
+import com.sun.jna.Platform;
+import com.sun.jna.Pointer;
+import com.sun.jna.ptr.PointerByReference;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Native IO library based on *nix C library, enabled for Linux, kernel version >= 2.4.10. 

+ * 

+ * Uses JNA library (https://github.com/java-native-access/jna) to access native calls. 

+ * 

+ */
+@SuppressWarnings({"OctalInteger", "WeakerAccess"})
+public class IgniteNativeIoLib {
+    /** Open for reading only. */
+    public static final int O_RDONLY = 00;
+
+    /** Open for writing only. */
+    public static final int O_WRONLY = 01;
+
+    /** Open for reading and writing. */
+    public static final int O_RDWR = 02;
+
+    /** File shall be created. If the file exists, this flag has no effect. */
+    public static final int O_CREAT = 0100;
+
+    /** If the file exists and is a regular file length shall be truncated to 0. */
+    public static final int O_TRUNC = 01000;
+
+    /** Try to minimize cache effects of the I/O to and from this file.  */
+    public static final int O_DIRECT = 040000;
+
+    /**
+     * Write operations on the file will complete according to the requirements of synchronized I/O file integrity
+     * completion. By the time write(2) (or similar) returns, the output data and associated file metadata have been
+     * transferred to the underlying hardware.
+     */
+    public static final int O_SYNC = 04000000;
+
+    /**
+     * The specified data will not be accessed in the near future. See fadvise.h and "man 2 posix_fadvise".
+    */
+    public static final int POSIX_FADV_DONTNEED = 4;
+
+    /** Flag for newly created files: user has read permission. */
+    public static final int S_IRUSR = 00400;
+
+    /** Flag for newly created files: user has write permission. */
+    public static final int S_IWUSR = 00200;
+
+    /** Flag for newly created files: group has read permission. */
+    public static final int S_IRGRP = 00040;
+
+    /** Flag for newly created files: others have read permission. */
+    public static final int S_IROTH = 00004;
+
+    /** Default access mask for newly created files. */
+    public static final int DEFAULT_OPEN_MODE = S_IRUSR | S_IWUSR | S_IROTH | S_IRGRP;
+
+    /** Invalid argument. */
+    public static final int E_INVAL = 22;
+
+    /** Seek option: set file offset to offset */
+    public static final int SEEK_SET = 0;
+
+    /** Seek option: change file position to offset */
+    public static final int SEEK_CUR = 1;
+
+    /** JNA library available and initialized. Always {@code false} for non linux systems. */
+    private static boolean jnaAvailable;
+
+    /** JNA library initialization exception. To be logged to Ignite logger later. */
+    @Nullable private static Exception ex;
+
+    static {
+        if (Platform.isLinux()) {
+            try {
+                if (checkLinuxVersion()) {
+                    Native.register(Platform.C_LIBRARY_NAME);
+                    jnaAvailable = true;
+                }
+                else
+                    jnaAvailable = false;
+            }
+            catch (Exception e) {
+                ex = e;
+                jnaAvailable = false;
+            }
+        }
+        else
+            jnaAvailable = false;
+    }
+
+    /**
+     * O_DIRECT  support was added under Linux in kernel version 2.4.10.
+     *
+     * @return {@code true} if O_DIRECT is supported, kernel version >= 2.4.10
+     */
+    private static boolean checkLinuxVersion() {
+        final String osVer = System.getProperty("os.version");
+
+        if (osVer == null)
+            return false;
+
+        List verIntComps = new ArrayList<>();
+
+        for (StringTokenizer tokenizer = new StringTokenizer(osVer, ".-"); tokenizer.hasMoreTokens(); ) {
+            String verComp = tokenizer.nextToken();
+
+            if (verComp.matches("\\d*"))
+                verIntComps.add(Integer.parseInt(verComp));
+        }
+
+        if (verIntComps.isEmpty())
+            return false;
+
+        final int verIdx = 0;
+        final int majorRevIdx = 1;
+        final int minorRevIdx = 2;
+
+        if (verIntComps.get(verIdx) > 2)
+            return true;
+        else if (verIntComps.get(verIdx) == 2) {
+            int compsCnt = verIntComps.size();
+
+            if (compsCnt > majorRevIdx && verIntComps.get(majorRevIdx) > 4)
+                return true;
+            else if (compsCnt > minorRevIdx
+                && verIntComps.get(majorRevIdx) == 4
+                && verIntComps.get(minorRevIdx) >= 10)
+                return true;
+        }
+        return false;
+    }
+
+
+    /**
+     * Calculate Lowest Common Multiplier.
+     * @param a first value.
+     * @param b second value.
+     */
+    private static long lcm(final long a, final long b) {
+        return (a * b) / gcf(a, b);
+    }
+
+    /**
+     * Calculate Greatest Common Factor.
+     * @param a first value.
+     * @param b second value.
+     */
+    private static long gcf(final long a, final long b) {
+        if (b == 0)
+            return a;
+        else
+            return gcf(b, a % b);
+    }
+
+    /**
+     * Determines FS and OS block size. Returns file system block size for use with storageDir see "man 3 posix_memalign"
+     *
+     * @param storageDir storage path, base path to check (FS) configuration parameters.
+     * @param log Logger.
+     * @return 
  • FS block size to be used in Direct IO and memory alignments.
    • 
+     * 
  • or -1 Operating System is not applicable for enabling Direct IO.
    • 
+     * 
  • and -1 if failed to determine block size.
    • 
+     * 
  • and -1 if JNA is not available or init failed.
    •  

+     */
+    public static int getFsBlockSize(final String storageDir, final IgniteLogger log) {
+        if (ex != null) {
+            U.warn(log, "Failed to initialize O_DIRECT support at current OS: " + ex.getMessage(), ex);
+
+            return -1;
+        }
+
+        if (!jnaAvailable)
+            return -1;
+
+        int fsBlockSize = -1;
+        int _PC_REC_XFER_ALIGN = 0x11;
+        int pcAlign = pathconf(storageDir, _PC_REC_XFER_ALIGN).intValue();
+
+        if (pcAlign > 0)
+            fsBlockSize = pcAlign;
+
+        int pageSize = getpagesize();
+
+        fsBlockSize = (int)lcm(fsBlockSize, pageSize);
+
+        // just being completely paranoid: (512 is the rule for 2.6+ kernels)
+        fsBlockSize = (int)lcm(fsBlockSize, 512);
+
+        if (log.isInfoEnabled())
+            log.info(String.format("Page size configuration for storage path [%s]: %d;" +
+                    " Linux memory page size: %d;" +
+                    " Selected FS block size : %d.",
+                storageDir, pcAlign, pageSize, fsBlockSize));
+
+        // lastly, a sanity check
+        if (fsBlockSize <= 0 || ((fsBlockSize & (fsBlockSize - 1)) != 0)) {
+            U.warn(log, "File system block size should be a power of two, was found to be " + fsBlockSize +
+                " Disabling O_DIRECT support");
+
+            return -1;
+        }
+
+        if (log.isInfoEnabled())
+            log.info("Selected FS block size : " + fsBlockSize);
+
+        return fsBlockSize;
+    }
+
+    /**
+     * @return Flag indicating JNA library available and initialized. Always {@code false} for non linux systems.
+     */
+    public static boolean isJnaAvailable() {
+        return jnaAvailable;
+    }
+
+    /**
+     * Open a file. See "man 3 open".
+     *
+     * @param pathname pathname naming the file.
+     * @param flags flag/open options. Flags are constructed by a bitwise-inclusive OR of flags.
+     * @param mode create file mode creation mask.
+     * @return file descriptor.
+     */
+    public static native int open(String pathname, int flags, int mode);
+
+    /**
+     * See "man 2 close".
+     *
+     * @param fd The file descriptor of the file to close.
+     * @return 0 on success, -1 on error.
+     */
+    public static native int close(int fd);
+
+    /**
+     * Writes up to {@code cnt} bytes to the buffer starting at {@code buf} to the file descriptor {@code fd} at offset
+     * {@code offset}. The file offset is not changed. See "man 2 pwrite".
+     *
+     * @param fd file descriptor.
+     * @param buf pointer to buffer with data.
+     * @param cnt bytes to write.
+     * @param off position in file to write data.
+     * @return the number of bytes written. Note that is not an error for a successful call to transfer fewer bytes than
+     * requested.
+     */
+    public static native NativeLong pwrite(int fd, Pointer buf, NativeLong cnt, NativeLong off);
+
+    /**
+     * Writes up to {@code cnt} bytes to the buffer starting at {@code buf} to the file descriptor {@code fd}.
+     * The file offset is changed. See "man 2 write".
+     *
+     * @param fd file descriptor.
+     * @param buf pointer to buffer with data.
+     * @param cnt bytes to write.
+     * @return the number of bytes written. Note that is not an error for a successful call to transfer fewer bytes than
+     * requested.
+     */
+    public static native NativeLong write(int fd, Pointer buf, NativeLong cnt);
+
+    /**
+     * Reads up to {@code cnt} bytes from file descriptor {@code fd} at offset {@code off} (from the start of the file)
+     * into the buffer starting at {@code buf}. The file offset is not changed. See "man 2 pread".
+     *
+     * @param fd file descriptor.
+     * @param buf pointer to buffer to place the data.
+     * @param cnt bytes to read.
+     * @return On success, the number of bytes read is returned (zero indicates end of file), on error, -1 is returned,
+     * and errno is set appropriately.
+     */
+    public static native NativeLong pread(int fd, Pointer buf, NativeLong cnt, NativeLong off);
+
+    /**
+     * Reads up to {@code cnt} bytes from file descriptor {@code fd} into the buffer starting at {@code buf}. The file
+     * offset is changed. See "man 2 read".
+     *
+     * @param fd file descriptor.
+     * @param buf pointer to buffer to place the data.
+     * @param cnt bytes to read.
+     * @return On success, the number of bytes read is returned (zero indicates end of file), on error, -1 is returned,
+     * and errno is set appropriately.
+     */
+    public static native NativeLong read(int fd, Pointer buf, NativeLong cnt);
+
+    /**
+     * Synchronize a file's in-core state with storage device. See "man 2 fsync".
+     * @param fd file descriptor.
+     * @return On success return zero. On error, -1 is returned, and errno is set appropriately.
+     */
+    public static native int fsync(int fd);
+
+    /**
+     * Allocates size bytes and places the address of the allocated memory in {@code memptr}.
+     * The address of the allocated memory will be a multiple of {@code alignment}.
+     *
+     * See "man 3 posix_memalign".
+     * @param memptr out memory pointer.
+     * @param alignment memory alignment,  must be a power of two and a multiple of sizeof(void *).
+     * @param size size of buffer.
+     * @return returns zero on success, or one of the error values.
+     */
+    public static native int posix_memalign(PointerByReference memptr, NativeLong alignment, NativeLong size);
+
+    /**
+     * Frees the memory space pointed to by ptr, which must have been returned by a previous call to native allocation
+     * methods. POSIX requires that memory obtained from {@link #posix_memalign} can be freed using free. See "man 3
+     * free".
+     *
+     * @param ptr pointer to free.
+     */
+    public static native void free(Pointer ptr);
+
+    /**
+     * Function returns a string that describes the error code passed in the argument {@code errnum}. See "man 3
+     * strerror".
+     *
+     * @param errnum error code.
+     * @return displayable error information.
+     */
+    public static native String strerror(int errnum);
+
+    /**
+     * Return path (FS) configuration parameter value. 

+     * Helps to determine alignment restrictions, for example, on buffers used for direct block device I/O. 

+     * POSIX specifies the pathconf(path,_PC_REC_XFER_ALIGN) call that tells what alignment is needed.
+     *
+     * @param path base path to check settings.
+     * @param name variable name to query.
+     */
+    public static native NativeLong pathconf(String path, int name);
+
+    /**
+     * The function getpagesize() returns the number of bytes in a memory
+     * page, where "page" is a fixed-length block, the unit for memory
+     * allocation and file mapping
+     */
+    public static native int getpagesize();
+
+    /**
+     * Allows to announce an intention to access file data in a specific pattern in the future, thus allowing the
+     * kernel to perform appropriate optimizations.
+     *
+     * The advice applies to a (not necessarily existent) region starting at
+     * {@code off} and extending for {@code len} bytes (or until the end of the file if len is 0)
+     * within the file referred to by fd.
+     *
+     * See "man 2 posix_fadvise".
+     *
+     * @param fd file descriptor.
+     * @param off region start.
+     * @param len region end.
+     * @param flag advice (option) to apply.
+     * @return On success, zero is returned.  On error, an error number is returned.
+     */
+    public static native int posix_fadvise(int fd, long off, long len, int flag);
+
+    /**
+     * Causes regular file referenced by fd to be truncated to a size of precisely length bytes.
+     *
+     * If the file previously was larger than this size, the extra data is lost.
+     * If the file previously was shorter, it is extended, and the extended part reads as null bytes ('\0').
+     * The file offset is not changed.
+     *
+     * @param fd  file descriptor.
+     * @param len required length.
+     * @return On success, zero is returned. On error, -1 is returned, and errno is set appropriately.
+     */
+    public static native int ftruncate(int fd, long len);
+
+    /**
+     * Repositions the file offset of the open file description associated with the file descriptor {@code fd}
+     * to the argument offset according to the directive {@code whence}
+     * @param fd file descriptor.
+     * @param off required position offset.
+     * @param whence position base.
+     * @return  On error, the value -1 is returned and errno is set to indicate the error.
+     */
+    public static native long lseek(int fd, long off, int whence);
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbClientNearCachePutGetTest.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/LinuxNativeIoPlugin.java
similarity index 63%
rename from modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbClientNearCachePutGetTest.java
rename to modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/LinuxNativeIoPlugin.java
index aa0819021e629..32c63dfdc064a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbClientNearCachePutGetTest.java
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/LinuxNativeIoPlugin.java
@@ -13,27 +13,13 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  */
 
-package org.apache.ignite.internal.processors.database;
-
-/**
- *
- */
-public class IgniteDbClientNearCachePutGetTest extends IgniteDbPutGetAbstractTest {
-    /** {@inheritDoc} */
-    @Override protected int gridCount() {
-        return 1;
-    }
+package org.apache.ignite.internal.processors.cache.persistence.file;
 
-    /** {@inheritDoc} */
-    @Override protected boolean indexingEnabled() {
-        return false;
-    }
+import org.apache.ignite.plugin.IgnitePlugin;
 
-    /** {@inheritDoc} */
-    @Override protected boolean withClientNearCache() {
-        return true;
-    }
-}
+/** Noop plugin. See {@link IgniteNativeIoLib}. */
+public class LinuxNativeIoPlugin implements IgnitePlugin {
+    // No-op.
+}
\ No newline at end of file
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/LinuxNativeIoPluginProvider.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/LinuxNativeIoPluginProvider.java
new file mode 100644
index 0000000000000..918ff5cdbf177
--- /dev/null
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/LinuxNativeIoPluginProvider.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.FileDescriptor;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.CachePluginContext;
+import org.apache.ignite.plugin.CachePluginProvider;
+import org.apache.ignite.plugin.ExtensionRegistry;
+import org.apache.ignite.plugin.IgnitePlugin;
+import org.apache.ignite.plugin.PluginContext;
+import org.apache.ignite.plugin.PluginProvider;
+import org.apache.ignite.plugin.PluginValidationException;
+import org.jetbrains.annotations.Nullable;
+import org.jsr166.ConcurrentHashMap8;
+
+/**
+ * Plugin provider for setting up {@link IgniteNativeIoLib}.
+ */
+public class LinuxNativeIoPluginProvider implements PluginProvider {
+    /** Managed buffers map from address to thread requested buffer. */
+    @Nullable private ConcurrentHashMap8 managedBuffers;
+
+    /** Logger. */
+    private IgniteLogger log;
+
+    /** {@inheritDoc} */
+    @Override public String name() {
+        return "Ignite Native I/O Plugin [Direct I/O]";
+    }
+
+    /** {@inheritDoc} */
+    @Override public String version() {
+        return "";
+    }
+
+    /** {@inheritDoc} */
+    @Override public String copyright() {
+        return "Copyright(C) Apache Software Foundation";
+    }
+
+    /** {@inheritDoc} */
+    @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public CachePluginProvider createCacheProvider(CachePluginContext ctx) {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void start(PluginContext ctx) {
+        final Ignite ignite = ctx.grid();
+
+        log = ignite.log();
+        managedBuffers = setupDirect((IgniteEx)ignite);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void stop(boolean cancel) {
+        freeDirectBuffers();
+    }
+
+    /**
+     * Free direct thread local buffer allocated for Direct IO user's threads.
+     */
+    private void freeDirectBuffers() {
+        ConcurrentHashMap8 buffers = managedBuffers;
+
+        if (buffers == null)
+            return;
+
+        managedBuffers = null;
+
+        if (log.isDebugEnabled())
+            log.debug("Direct IO buffers to be freed: " + buffers.size());
+
+        for (Map.Entry next : buffers.entrySet()) {
+            Thread th = next.getValue();
+            Long addr = next.getKey();
+
+            if (log.isDebugEnabled())
+                log.debug(String.format("Free Direct IO buffer [address=%d; Thread=%s; alive=%s]",
+                    addr, th != null ? th.getName() : "", th != null && th.isAlive()));
+
+            AlignedBuffers.free(addr);
+        }
+
+        buffers.clear();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onIgniteStart() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onIgniteStop(boolean cancel) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Serializable provideDiscoveryData(UUID nodeId) {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void receiveDiscoveryData(UUID nodeId, Serializable data) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void validateNewNode(ClusterNode node) throws PluginValidationException {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Object createComponent(PluginContext ctx, Class cls) {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgnitePlugin plugin() {
+        return new LinuxNativeIoPlugin();
+    }
+
+    /**
+     * @param ignite Ignite starting up.
+     * @return Managed aligned buffers and its associated threads. This collection is used to free buffers. May return
+     * {@code null}.
+     */
+    @Nullable private ConcurrentHashMap8 setupDirect(IgniteEx ignite) {
+        GridCacheSharedContext cacheCtx = ignite.context().cache().context();
+        IgnitePageStoreManager ignitePageStoreMgr = cacheCtx.pageStore();
+
+        if (ignitePageStoreMgr == null)
+            return null;
+
+        if (!(ignitePageStoreMgr instanceof FilePageStoreManager))
+            return null;
+
+        final FilePageStoreManager pageStore = (FilePageStoreManager)ignitePageStoreMgr;
+        FileIOFactory backupIoFactory = pageStore.getPageStoreFileIoFactory();
+
+        final AlignedBuffersDirectFileIOFactory factory = new AlignedBuffersDirectFileIOFactory(
+            ignite.log(),
+            pageStore.workDir(),
+            pageStore.pageSize(),
+            backupIoFactory);
+
+        final FileWriteAheadLogManager walMgr = (FileWriteAheadLogManager)cacheCtx.wal();
+
+        if (walMgr != null && IgniteNativeIoLib.isJnaAvailable()) {
+            walMgr.setCreateWalFileListener(new IgniteInClosure() {
+                @Override public void apply(FileIO fileIO) {
+                    adviceFileDontNeed(fileIO, walMgr.maxWalSegmentSize());
+                }
+            });
+        }
+
+        if (!factory.isDirectIoAvailable())
+            return null;
+
+        GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)cacheCtx.database();
+
+        db.setThreadBuf(new ThreadLocal() {
+            @Override protected ByteBuffer initialValue() {
+                return factory.createManagedBuffer(pageStore.pageSize());
+            }
+        });
+
+        pageStore.setPageStoreFileIOFactories(factory, backupIoFactory);
+
+        return factory.managedAlignedBuffers();
+    }
+
+    /**
+     * Apply advice: The specified data will not be accessed in the near future.
+     *
+     * Useful for WAL segments to indicate file content won't be loaded.
+     *
+     * @param fileIO file to advice.
+     * @param size expected size of file.
+     */
+    private void adviceFileDontNeed(FileIO fileIO, long size)   {
+        try {
+            if(fileIO instanceof RandomAccessFileIO) {
+                RandomAccessFileIO chIo = (RandomAccessFileIO)fileIO;
+
+                FileChannel ch = U.field(chIo, "ch");
+
+                FileDescriptor fd = U.field(ch, "fd");
+
+                int fdVal = U.field(fd, "fd");
+
+                int retVal = IgniteNativeIoLib.posix_fadvise(fdVal, 0, size, IgniteNativeIoLib.POSIX_FADV_DONTNEED);
+
+                if (retVal != 0) {
+                    U.warn(log, "Unable to apply fadvice on WAL file descriptor [fd=" + fdVal + "]:" +
+                        IgniteNativeIoLib.strerror(retVal));
+                }
+            }
+        }
+        catch (Exception e) {
+            U.warn(log, "Unable to advice on WAL file descriptor: [" + e.getMessage() + "]", e);
+        }
+    }
+}
diff --git a/modules/direct-io/src/main/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider b/modules/direct-io/src/main/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
new file mode 100644
index 0000000000000..ee1ccd885fab8
--- /dev/null
+++ b/modules/direct-io/src/main/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
@@ -0,0 +1 @@
+org.apache.ignite.internal.processors.cache.persistence.file.LinuxNativeIoPluginProvider
\ No newline at end of file
diff --git a/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoWithNoPersistenceTest.java b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoWithNoPersistenceTest.java
new file mode 100644
index 0000000000000..981e0d59a633b
--- /dev/null
+++ b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteNativeIoWithNoPersistenceTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import com.google.common.base.Strings;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Checks if Direct IO can be set up if no persistent store is configured
+ */
+public class IgniteNativeIoWithNoPersistenceTest extends GridCommonAbstractTest {
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration configuration = super.getConfiguration(igniteInstanceName);
+
+        configuration.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()));
+
+        return configuration;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * Checks simple launch with native IO.
+     * @throws Exception if failed
+     */
+    public void testDirectIoHandlesNoPersistentGrid() throws Exception {
+        IgniteEx ignite = startGrid(0);
+
+        ignite.active(true);
+
+        IgniteCache cache = ignite.getOrCreateCache("cache");
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, valueForKey(i));
+
+
+        stopAllGrids();
+    }
+
+    /**
+     * @param i key.
+     * @return value with extra data, which allows to verify
+     */
+    @NotNull private String valueForKey(int i) {
+        return Strings.repeat(Integer.toString(i), 10);
+    }
+}
diff --git a/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite.java b/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite.java
new file mode 100644
index 0000000000000..48454eaa8d6ea
--- /dev/null
+++ b/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.file.IgniteNativeIoWithNoPersistenceTest;
+
+/**
+ * Subset of {@link IgnitePdsTestSuite} suite test, started with direct-oi jar in classpath.
+ */
+public class IgnitePdsNativeIoTestSuite extends TestSuite {
+    /**
+     * @return Suite.
+     */
+    public static TestSuite suite() {
+        TestSuite suite = new TestSuite("Ignite Persistent Store Test Suite (with Direct IO)");
+
+        IgnitePdsTestSuite.addRealPageStoreTests(suite);
+
+        suite.addTestSuite(IgniteNativeIoWithNoPersistenceTest.class);
+
+        return suite;
+    }
+}
\ No newline at end of file
diff --git a/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java b/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java
new file mode 100644
index 0000000000000..54dd7d3689ab7
--- /dev/null
+++ b/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+
+/**
+ * Same as {@link IgnitePdsTestSuite2} but is started with direct-oi jar in classpath.
+ */
+public class IgnitePdsNativeIoTestSuite2 extends TestSuite {
+    /**
+     * @return Suite.
+     * @throws Exception If failed.
+     */
+    public static TestSuite suite() throws Exception {
+        TestSuite suite = new TestSuite("Ignite Persistent Store Test Suite 2 (Native IO)");
+
+        IgnitePdsTestSuite2.addRealPageStoreTests(suite);
+
+        return suite;
+    }
+}
diff --git a/pom.xml b/pom.xml
index 1d8637c54a5ca..eab1ec8275707 100644
--- a/pom.xml
+++ b/pom.xml
@@ -52,6 +52,7 @@
         modules/tools
         modules/core
         modules/dev-utils
+        modules/direct-io
         modules/hadoop
         modules/extdata/p2p
         modules/extdata/uri

From dd06d0bd7ef266bfbe156e858b312d1ac86e8982 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn 
Date: Thu, 18 Jan 2018 15:55:49 +0300
Subject: [PATCH 13/65] IGNITE-7465 .NET: Fix SqlDdlExample failure with
 standalone node

---
 .../examples/Apache.Ignite.Examples/Sql/SqlDdlExample.cs       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlDdlExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlDdlExample.cs
index 3f72842901a7d..64d267d8c65eb 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlDdlExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Sql/SqlDdlExample.cs
@@ -53,7 +53,8 @@ public static void Main()
                 // will appear in future versions, JDBC and ODBC drivers do not require it already).
                 var cacheCfg = new CacheConfiguration(DummyCacheName)
                 {
-                    SqlSchema = "PUBLIC"
+                    SqlSchema = "PUBLIC",
+                    CacheMode = CacheMode.Replicated
                 };
 
                 ICache cache = ignite.GetOrCreateCache(cacheCfg);

From 57479ec564e1761716da3d5f9feb7a64c396a9f9 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn 
Date: Thu, 18 Jan 2018 16:45:54 +0300
Subject: [PATCH 14/65] .NET: Fix CacheLocalTest.TestTxDeadlockDetection

---
 .../Cache/CacheAbstractTransactionalTest.cs                  | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
index 7a60e9abc2539..2602a02efb75c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
@@ -532,6 +532,11 @@ public void TestTxStateAndExceptions()
         [Test]
         public void TestTxDeadlockDetection()
         {
+            if (LocalCache())
+            {
+                return;
+            }
+
             var cache = Cache();
 
             var keys0 = Enumerable.Range(1, 100).ToArray();

From bd6be8a4653322905a3b63850c7e033ce3801ce5 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn 
Date: Thu, 18 Jan 2018 21:25:05 +0300
Subject: [PATCH 15/65] .NET: Thin client: Fix OP_BINARY_TYPE_GET schema
 passing format

---
 .../processors/platform/utils/PlatformUtils.java       |  8 +++++++-
 .../Impl/Binary/Metadata/BinaryType.cs                 | 10 +++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
index b65ca040fb527..2954a9e6c0e87 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
@@ -1144,7 +1144,13 @@ public static void writeBinaryMetadata(BinaryRawWriter writer, BinaryMetadata me
 
         for (BinarySchema schema : schemas) {
             writer.writeInt(schema.schemaId());
-            writer.writeIntArray(schema.fieldIds());
+
+            int[] ids = schema.fieldIds();
+            writer.writeInt(ids.length);
+
+            for (int id : ids) {
+                writer.writeInt(id);
+            }
         }
     }
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Metadata/BinaryType.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Metadata/BinaryType.cs
index 514f2e2a6323a..06794b5734692 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Metadata/BinaryType.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/Metadata/BinaryType.cs
@@ -172,7 +172,15 @@ public BinaryType(BinaryReader reader, bool readSchemas = false)
 
                 for (var i = 0; i < cnt; i++)
                 {
-                    _schema.Add(reader.ReadInt(), reader.ReadIntArray());
+                    var schemaId = reader.ReadInt();
+                    
+                    var ids = new int[reader.ReadInt()];
+                    for (var j = 0; j < ids.Length; j++)
+                    {
+                        ids[j] = reader.ReadInt();
+                    }
+
+                    _schema.Add(schemaId, ids);
                 }
             }
 

From 97564d160586d6d57d300937e6b8877994e35fc7 Mon Sep 17 00:00:00 2001
From: rkondakov 
Date: Fri, 19 Jan 2018 11:24:51 +0300
Subject: [PATCH 16/65] IGNITE-6456: Ability to separately enable or disable
 JDBC, ODBC and thin client endpoints. This closes #3309.

---
 .../ClientConnectorConfiguration.java         |  84 ++++++++++++++
 .../odbc/ClientListenerNioListener.java       | 105 +++++++++++++-----
 .../odbc/ClientListenerProcessor.java         |   3 +-
 .../utils/PlatformConfigurationUtils.java     |   9 +-
 ...nectorConfigurationValidationSelfTest.java |  43 +++++++
 .../Client/ClientConnectionTest.cs            |  17 +++
 .../IgniteConfigurationSerializerTest.cs      |   3 +
 .../IgniteConfigurationTest.cs                |   3 +
 .../ClientConnectorConfiguration.cs           |  45 ++++++++
 .../IgniteConfigurationSection.xsd            |  15 +++
 10 files changed, 297 insertions(+), 30 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/ClientConnectorConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/ClientConnectorConfiguration.java
index d2520d2373459..5827333b45a96 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/ClientConnectorConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/ClientConnectorConfiguration.java
@@ -72,6 +72,15 @@ public class ClientConnectorConfiguration {
     /** Idle timeout. */
     private long idleTimeout = DFLT_IDLE_TIMEOUT;
 
+    /** JDBC connections enabled flag. */
+    private boolean jdbcEnabled = true;
+
+    /** ODBC connections enabled flag. */
+    private boolean odbcEnabled = true;
+
+    /** JDBC connections enabled flag. */
+    private boolean thinCliEnabled = true;
+
     /**
      * Creates SQL connector configuration with all default values.
      */
@@ -301,6 +310,81 @@ public ClientConnectorConfiguration setIdleTimeout(long idleTimeout) {
         return this;
     }
 
+    /**
+     * Gets whether access through JDBC is enabled.
+     * 

+     * Defaults to {@code true}.
+     *
+     * @return Whether access through JDBC is enabled.
+     */
+    public boolean isJdbcEnabled() {
+        return jdbcEnabled;
+    }
+
+    /**
+     * Sets whether access through JDBC is enabled.
+     * 

+     * Defaults to {@code true}.
+     *
+     * @param jdbcEnabled Whether access through JDBC is enabled.
+     * @return {@code this} for chaining.
+     */
+    public ClientConnectorConfiguration setJdbcEnabled(boolean jdbcEnabled) {
+        this.jdbcEnabled = jdbcEnabled;
+
+        return this;
+    }
+
+    /**
+     * Gets whether access through ODBC is enabled.
+     * 

+     * Defaults to {@code true}.
+     *
+     * @return Whether access through ODBC is enabled.
+     */
+    public boolean isOdbcEnabled() {
+        return odbcEnabled;
+    }
+
+    /**
+     * Sets whether access through ODBC is enabled.
+     * 

+     * Defaults to {@code true}.
+     *
+     * @param odbcEnabled Whether access through ODBC is enabled.
+     * @return {@code this} for chaining.
+     */
+    public ClientConnectorConfiguration setOdbcEnabled(boolean odbcEnabled) {
+        this.odbcEnabled = odbcEnabled;
+
+        return this;
+    }
+
+    /**
+     * Gets whether access through thin client is enabled.
+     * 

+     * Defaults to {@code true}.
+     *
+     * @return Whether access through thin client is enabled.
+     */
+    public boolean isThinClientEnabled() {
+        return thinCliEnabled;
+    }
+
+    /**
+     * Sets whether access through thin client is enabled.
+     * 

+     * Defaults to {@code true}.
+     *
+     * @param thinCliEnabled Whether access through thin client is enabled.
+     * @return {@code this} for chaining.
+     */
+    public ClientConnectorConfiguration setThinClientEnabled(boolean thinCliEnabled) {
+        this.thinCliEnabled = thinCliEnabled;
+
+        return this;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(ClientConnectorConfiguration.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
index b41e2401b4f82..43276a5be1a12 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
@@ -17,8 +17,9 @@
 
 package org.apache.ignite.internal.processors.odbc;
 
-import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.ClientConnectorConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.binary.BinaryReaderExImpl;
 import org.apache.ignite.internal.binary.BinaryWriterExImpl;
@@ -62,18 +63,24 @@ public class ClientListenerNioListener extends GridNioServerListenerAdapter srv0 = GridNioServer.builder()
                             .address(hostAddr)
                             .port(port)
-                            .listener(new ClientListenerNioListener(ctx, busyLock, maxOpenCursors))
+                            .listener(new ClientListenerNioListener(ctx, busyLock, cliConnCfg))
                             .logger(log)
                             .selectorCount(DFLT_SELECTOR_CNT)
                             .igniteInstanceName(ctx.igniteInstanceName())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
index c4c354e0fefb4..d5fe6f9a8b8ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
@@ -1600,7 +1600,10 @@ private static ClientConnectorConfiguration readClientConnectorConfiguration(Bin
                 .setTcpNoDelay(in.readBoolean())
                 .setMaxOpenCursorsPerConnection(in.readInt())
                 .setThreadPoolSize(in.readInt())
-                .setIdleTimeout(in.readLong());
+                .setIdleTimeout(in.readLong())
+                .setThinClientEnabled(in.readBoolean())
+                .setOdbcEnabled(in.readBoolean())
+                .setJdbcEnabled(in.readBoolean());
     }
 
     /**
@@ -1623,6 +1626,10 @@ private static void writeClientConnectorConfiguration(BinaryRawWriter w, ClientC
             w.writeInt(cfg.getMaxOpenCursorsPerConnection());
             w.writeInt(cfg.getThreadPoolSize());
             w.writeLong(cfg.getIdleTimeout());
+
+            w.writeBoolean(cfg.isThinClientEnabled());
+            w.writeBoolean(cfg.isOdbcEnabled());
+            w.writeBoolean(cfg.isJdbcEnabled());
         } else {
             w.writeBoolean(false);
         }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/ClientConnectorConfigurationValidationSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/ClientConnectorConfigurationValidationSelfTest.java
index c861e49c72674..141f4446d9f5c 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/ClientConnectorConfigurationValidationSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/client/ClientConnectorConfigurationValidationSelfTest.java
@@ -294,6 +294,49 @@ public void testDisabled() throws Exception {
         }, SQLException.class, null);
     }
 
+    /**
+     * Checks if JDBC connection enabled and others are disabled, JDBC still works.
+     *
+     * @throws Exception If failed.
+     */
+    public void testJdbcConnectionEnabled() throws Exception {
+        IgniteConfiguration cfg = baseConfiguration();
+
+        cfg.setClientConnectorConfiguration(new ClientConnectorConfiguration()
+            .setJdbcEnabled(true)
+            .setOdbcEnabled(false)
+            .setThinClientEnabled(false));
+
+        Ignition.start(cfg);
+
+        checkJdbc(null, ClientConnectorConfiguration.DFLT_PORT);
+    }
+
+    /**
+     * Checks if JDBC connection disabled and others are enabled, JDBC doesn't work.
+     *
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+    public void testJdbcConnectionDisabled() throws Exception {
+        IgniteConfiguration cfg = baseConfiguration();
+
+        cfg.setClientConnectorConfiguration(new ClientConnectorConfiguration()
+            .setJdbcEnabled(false)
+            .setOdbcEnabled(true)
+            .setThinClientEnabled(true));
+
+        Ignition.start(cfg);
+
+        GridTestUtils.assertThrows(log, new Callable() {
+            @Override public Void call() throws Exception {
+                checkJdbc(null, ClientConnectorConfiguration.DFLT_PORT);
+
+                return null;
+            }
+        }, SQLException.class, "Failed to connect to Ignite cluster");
+    }
+
     /**
      * Get base node configuration.
      *
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
index 999fdf8b76021..57eb97cee6836 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
@@ -157,6 +157,23 @@ public void TestDisabledConnector()
                 Assert.AreEqual("Failed to establish Ignite thin client connection, " +
                                 "examine inner exceptions for details.", ex.Message.Substring(0, 88));
             }
+
+            // Disable only thin client.
+            servCfg = new IgniteConfiguration(TestUtils.GetTestConfiguration())
+            {
+                ClientConnectorConfiguration = new ClientConnectorConfiguration
+                {
+                    ThinClientEnabled = false
+                }
+            };
+
+            using (Ignition.Start(servCfg))
+            {
+                var ex = Assert.Throws(() => Ignition.StartClient(clientCfg));
+                Assert.AreEqual("Client handshake failed: 'Thin client connection is not allowed, " +
+                                "see ClientConnectorConfiguration.thinClientEnabled.'.", 
+                                ex.Message.Substring(0, 118));
+            }
         }
 
         /// 

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs
index 543bb6de4a2fc..da80e411527d0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs
@@ -898,6 +898,9 @@ private static IgniteConfiguration GetTestConfig()
                     SocketReceiveBufferSize = 5,
                     SocketSendBufferSize = 6,
                     TcpNoDelay = false,
+                    ThinClientEnabled = false,
+                    OdbcEnabled = false,
+                    JdbcEnabled = false,
                     ThreadPoolSize = 7,
                     IdleTimeout = TimeSpan.FromMinutes(5)
                 },
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
index 734b0cf7d7d51..c7072168f7ceb 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
@@ -596,6 +596,9 @@ private static void CheckDefaultProperties(ClientConnectorConfiguration cfg)
             Assert.AreEqual(ClientConnectorConfiguration.DefaultTcpNoDelay, cfg.TcpNoDelay);
             Assert.AreEqual(ClientConnectorConfiguration.DefaultThreadPoolSize, cfg.ThreadPoolSize);
             Assert.AreEqual(ClientConnectorConfiguration.DefaultIdleTimeout, cfg.IdleTimeout);
+            Assert.AreEqual(ClientConnectorConfiguration.DefaultThinClientEnabled, cfg.ThinClientEnabled);
+            Assert.AreEqual(ClientConnectorConfiguration.DefaultJdbcEnabled, cfg.JdbcEnabled);
+            Assert.AreEqual(ClientConnectorConfiguration.DefaultOdbcEnabled, cfg.OdbcEnabled);
         }
 
         /// 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Configuration/ClientConnectorConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Configuration/ClientConnectorConfiguration.cs
index e51a08cf6c6dc..090ebd145cf8a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Configuration/ClientConnectorConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Configuration/ClientConnectorConfiguration.cs
@@ -63,6 +63,21 @@ public class ClientConnectorConfiguration
         /// 
         public static readonly TimeSpan DefaultIdleTimeout = TimeSpan.Zero;
 
+        /// 
+        /// Default value for  property.
+        /// 
+        public const bool DefaultThinClientEnabled = true;
+
+        /// 
+        /// Default value for  property.
+        /// 
+        public const bool DefaultJdbcEnabled = true;
+
+        /// 
+        /// Default value for  property.
+        /// 
+        public const bool DefaultOdbcEnabled = true;
+
         /// 
         /// Initializes a new instance of the  class.
         /// 
@@ -76,6 +91,10 @@ public ClientConnectorConfiguration()
             MaxOpenCursorsPerConnection = DefaultMaxOpenCursorsPerConnection;
             ThreadPoolSize = DefaultThreadPoolSize;
             IdleTimeout = DefaultIdleTimeout;
+
+            ThinClientEnabled = DefaultThinClientEnabled;
+            OdbcEnabled = DefaultOdbcEnabled;
+            JdbcEnabled = DefaultJdbcEnabled;
         }
 
         /// 
@@ -94,6 +113,10 @@ internal ClientConnectorConfiguration(IBinaryRawReader reader)
             MaxOpenCursorsPerConnection = reader.ReadInt();
             ThreadPoolSize = reader.ReadInt();
             IdleTimeout = reader.ReadLongAsTimespan();
+
+            ThinClientEnabled = reader.ReadBoolean();
+            OdbcEnabled = reader.ReadBoolean();
+            JdbcEnabled = reader.ReadBoolean();
         }
 
         /// 
@@ -112,6 +135,10 @@ internal void Write(IBinaryRawWriter writer)
             writer.WriteInt(MaxOpenCursorsPerConnection);
             writer.WriteInt(ThreadPoolSize);
             writer.WriteTimeSpanAsLong(IdleTimeout);
+
+            writer.WriteBoolean(ThinClientEnabled);
+            writer.WriteBoolean(OdbcEnabled);
+            writer.WriteBoolean(JdbcEnabled);
         }
 
         /// 
@@ -172,5 +199,23 @@ internal void Write(IBinaryRawWriter writer)
         /// Zero or negative means no timeout.
         /// 
         public TimeSpan IdleTimeout { get; set; }
+
+        /// 
+        /// Gets or sets a value indicating whether thin client connector is enabled.
+        /// 
+        [DefaultValue(DefaultThinClientEnabled)]
+        public bool ThinClientEnabled { get; set; }
+
+        /// 
+        /// Gets or sets a value indicating whether JDBC connector is enabled.
+        /// 
+        [DefaultValue(DefaultJdbcEnabled)]
+        public bool JdbcEnabled { get; set; }
+
+        /// 
+        /// Gets or sets a value indicating whether ODBC connector is enabled.
+        /// 
+        [DefaultValue(DefaultOdbcEnabled)]
+        public bool OdbcEnabled { get; set; }
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
index 7deebc5f0c056..53cbb40f07824 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd
@@ -1454,6 +1454,21 @@
                                 Idle timeout for client connections on the server side. If no packets come within idle timeout, the connection is closed by the server. Zero or negative for no timeout.
                             
                         
+                        
+                            
+                                Enables thin client connector.
+                            
+                        
+                        
+                            
+                                Enables JDBC connector.
+                            
+                        
+                        
+                            
+                                Enables ODBC connector.
+                            
+                        
                     
                 
                 

From d50274ca8875c9680c12e8786ac355a787ba95e0 Mon Sep 17 00:00:00 2001
From: Yakov Zhdanov 
Date: Thu, 18 Jan 2018 20:57:17 +0300
Subject: [PATCH 17/65] Javadoc enhancements - added @see

---
 .../java/org/apache/ignite/cache/PartitionLossPolicy.java     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/PartitionLossPolicy.java b/modules/core/src/main/java/org/apache/ignite/cache/PartitionLossPolicy.java
index 16608a0587066..7c9b2a7ffa692 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/PartitionLossPolicy.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/PartitionLossPolicy.java
@@ -19,6 +19,7 @@
 
 import java.util.Collection;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -31,6 +32,9 @@
  * 

  * READ_ONLY_* and READ_WRITE_* policies do not automatically change partition state
  * and thus do not change rebalancing assignments for such partitions.
+ *
+ * @see Ignite#resetLostPartitions(Collection)
+ * @see IgniteCache#lostPartitions()
  */
 public enum PartitionLossPolicy {
     /**

From cb2d3cf22388ab19fb2d34ae5bdfc8f1b608db75 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin 
Date: Thu, 18 Jan 2018 17:14:26 +0300
Subject: [PATCH 18/65] IGNITE-7471 Use soft reference for checkpoint entry
 contents to avoid excessive memory usage

---
 .../GridCachePartitionExchangeManager.java    |   5 +-
 .../GridCacheDatabaseSharedManager.java       | 544 +++++++++++++-----
 .../wal/IgniteWalHistoryReservationsTest.java |  13 +-
 3 files changed, 422 insertions(+), 140 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7107125553604..ef972389fda8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -959,8 +959,11 @@ public void scheduleResendPartitions() {
      */
     private void refreshPartitions() {
         // TODO https://issues.apache.org/jira/browse/IGNITE-6857
-        if (cctx.snapshot().snapshotOperationInProgress())
+        if (cctx.snapshot().snapshotOperationInProgress()) {
+            scheduleResendPartitions();
+
             return;
+        }
 
         ClusterNode oldest = cctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 1b0eb6d399a09..e56b3521e8a64 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -21,6 +21,7 @@
 import java.io.FileFilter;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.lang.ref.SoftReference;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.channels.FileChannel;
@@ -41,6 +42,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -323,10 +325,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private final int maxCpHistMemSize;
 
     /** */
-    private Map>> reservedForExchange;
+    private Map>> reservedForExchange;
 
     /** */
-    private final ConcurrentMap, T2> reservedForPreloading = new ConcurrentHashMap<>();
+    private final ConcurrentMap, T2> reservedForPreloading = new ConcurrentHashMap<>();
 
     /** Snapshot manager. */
     private IgniteCacheSnapshotManager snapshotMgr;
@@ -1345,47 +1347,78 @@ private void restoreState() throws IgniteCheckedException {
 
         reservedForExchange = new HashMap<>();
 
-        for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
-            if (grp.isLocal())
-                continue;
+        Map> parts4CheckpointHistSearch = partsForCheckpointHistorySearch();
 
-            for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions()) {
-                if (part.state() != GridDhtPartitionState.OWNING || part.dataStore().fullSize() <= walRebalanceThreshold)
-                    continue;
+        Map> lastCheckpointEntry4Grp =
+            searchLastCheckpointEntryPerPartition(parts4CheckpointHistSearch);
 
-                CheckpointEntry cpEntry = searchCheckpointEntry(grp.groupId(), part.id(), null);
+        Map> grpPartsWithCnts = new HashMap<>();
 
-                try {
-                    if (cpEntry != null && cctx.wal().reserve(cpEntry.cpMark)) {
-                        Map> cacheMap = reservedForExchange.get(grp.groupId());
+        try {
+            for (Map.Entry> e : lastCheckpointEntry4Grp.entrySet()) {
+                Integer grpId = e.getKey();
+
+                for (Map.Entry e0 : e.getValue().entrySet()) {
+                    CheckpointEntry cpEntry = e0.getValue();
+
+                    Integer partId = e0.getKey();
+
+                    if (cctx.wal().reserve(cpEntry.cpMark)) {
+                        Map> grpChpState = reservedForExchange.get(grpId);
 
-                        if (cacheMap == null) {
-                            cacheMap = new HashMap<>();
+                        Map grpCnts = grpPartsWithCnts.get(grpId);
 
-                            reservedForExchange.put(grp.groupId(), cacheMap);
+                        if (grpChpState == null) {
+                            reservedForExchange.put(grpId, grpChpState = new HashMap<>());
+
+                            grpPartsWithCnts.put(grpId, grpCnts = new HashMap<>());
                         }
 
-                        cacheMap.put(part.id(), new T2<>(cpEntry.partitionCounter(grp.groupId(), part.id()), cpEntry.cpMark));
+                        Long partCnt = cpEntry.partitionCounter(cctx, grpId, partId);
+
+                        if (partCnt != null) {
+                            grpChpState.put(partId, new T2<>(partCnt, cpEntry.cpMark));
+
+                            grpCnts.put(partId, partCnt);
+                        }
+                        else
+                            cctx.wal().release(cpEntry.cpMark);
                     }
                 }
-                catch (IgniteCheckedException ex) {
-                    U.error(log, "Error while trying to reserve history", ex);
-                }
             }
         }
+        catch (IgniteCheckedException ex) {
+            U.error(log, "Error while trying to reserve history", ex);
+        }
 
-        Map> resMap = new HashMap<>();
+        return grpPartsWithCnts;
+    }
 
-        for (Map.Entry>> e : reservedForExchange.entrySet()) {
-            Map cacheMap = new HashMap<>();
+    /**
+     *
+     * @return Map of group id -> Set parts.
+     */
+    private Map> partsForCheckpointHistorySearch() {
+        Map> part4CheckpointHistSearch = new HashMap<>();
 
-            for (Map.Entry> e0 : e.getValue().entrySet())
-                cacheMap.put(e0.getKey(), e0.getValue().get1());
+        for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+            if (grp.isLocal())
+                continue;
 
-            resMap.put(e.getKey(), cacheMap);
+            for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions()) {
+                if (part.state() != GridDhtPartitionState.OWNING || part.dataStore().fullSize() <= walRebalanceThreshold)
+                    continue;
+
+                Set parts = part4CheckpointHistSearch.get(grp.groupId());
+
+                if (parts == null)
+                    part4CheckpointHistSearch.put(grp.groupId(), parts = new HashSet<>());
+
+                parts.add(part.id());
+            }
         }
 
-        return resMap;
+        return part4CheckpointHistSearch;
     }
 
     /** {@inheritDoc} */
@@ -1494,6 +1527,67 @@ public Map, T2> reservedForPreloading() {
         fut2.get();
     }
 
+    /**
+     * Tries to search for a WAL pointer for the given partition counter start.
+     *
+     * @return Checkpoint entry or {@code null} if failed to search.
+     */
+    private Map> searchLastCheckpointEntryPerPartition(
+        final Map> part4reserve
+    ) {
+        final Map> res = new HashMap<>();
+
+        for (Long cpTs : checkpointHist.checkpoints()) {
+            try {
+                final CheckpointEntry chpEntry = checkpointHist.entry(cpTs);
+
+                Map grpsState = chpEntry.groupState(cctx);
+
+                if (grpsState.isEmpty()){
+                    res.clear();
+
+                    continue;
+                }
+
+                for (Map.Entry> grps : part4reserve.entrySet()) {
+                    Integer grpId = grps.getKey();
+
+                    Map partToCheckPntEntry = res.get(grpId);
+
+                    CheckpointEntry.GroupState grpState = grpsState.get(grpId);
+
+                    if (grpState == null) {
+                        res.remove(grpId);
+
+                        continue;
+                    }
+
+                    if (partToCheckPntEntry == null)
+                        res.put(grpId, partToCheckPntEntry = new HashMap<>());
+
+                    for (Integer partId : grps.getValue()) {
+                        int idx = grpState.indexByPartition(partId);
+
+                        if (idx < 0)
+                            partToCheckPntEntry.remove(partId);
+                        else {
+                            if (partToCheckPntEntry.containsKey(partId))
+                                continue;
+
+                            partToCheckPntEntry.put(partId, chpEntry);
+                        }
+                    }
+                }
+            }
+            catch (IgniteCheckedException ignore) {
+                // Treat exception the same way as a gap.
+                res.clear();
+            }
+        }
+
+        return res;
+    }
+
     /**
      * Tries to search for a WAL pointer for the given partition counter start.
      *
@@ -1527,7 +1621,7 @@ public Map, T2> reservedForPreloading() {
             try {
                 CheckpointEntry entry = checkpointHist.entry(cpTs);
 
-                Long foundCntr = entry.partitionCounter(grpId, part);
+                Long foundCntr = entry.partitionCounter(cctx, grpId, part);
 
                 if (foundCntr != null) {
                     if (partCntrSince == null) {
@@ -1682,8 +1776,8 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
             return new FileWALPointer(buf.getLong(), buf.getInt(), buf.getInt());
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to read checkpoint pointer from marker file: " +
-                cpMarkerFile.getAbsolutePath(), e);
+            throw new IgniteCheckedException(
+                "Failed to read checkpoint pointer from marker file: " + cpMarkerFile.getAbsolutePath(), e);
         }
     }
 
@@ -2403,8 +2497,7 @@ private CheckpointEntry writeCheckpointEntry(
             if (!skipSync)
                 ch.force(true);
 
-            return type == CheckpointEntryType.START ?
-                new CheckpointEntry(cpTs, ptr, cpId, rec.cacheGroupStates()) : null;
+            return createCheckPointEntry(cpTs, ptr, cpId, rec, type);
         }
         catch (IOException e) {
             throw new IgniteCheckedException(e);
@@ -2444,6 +2537,41 @@ public void setThreadBuf(final ThreadLocal threadBuf) {
         this.threadBuf = threadBuf;
     }
 
+    /**
+     * @param cpTs Checkpoint timestamp.
+     * @param ptr Wal pointer of checkpoint.
+     * @param cpId Checkpoint ID.
+     * @param rec Checkpoint record.
+     * @param type Checkpoint type.
+     *
+     * @return Checkpoint entry.
+     */
+    private CheckpointEntry createCheckPointEntry(
+        long cpTs,
+        WALPointer ptr,
+        UUID cpId,
+        @Nullable CheckpointRecord rec,
+        CheckpointEntryType type
+    ) {
+        assert cpTs > 0;
+        assert ptr != null;
+        assert cpId != null;
+        assert type != null;
+
+        if (type != CheckpointEntryType.START)
+            return null;
+
+        CheckpointEntry entry;
+
+        Map cacheGrpStates = null;
+
+        // Create lazy checkpoint entry.
+        if ((checkpointHist.histMap.size() + 1 < maxCpHistMemSize) && rec != null)
+            cacheGrpStates = rec.cacheGroupStates();
+
+        return new CheckpointEntry(cpTs, ptr, cpId, cacheGrpStates);
+    }
+
     /**
      *
      */
@@ -3215,8 +3343,6 @@ private Checkpoint(
             @NotNull GridMultiCollectionWrapper cpPages,
             CheckpointProgress progress
         ) {
-            assert cpEntry == null || cpEntry.initGuard != 0;
-
             this.cpEntry = cpEntry;
             this.cpPages = cpPages;
             this.progress = progress;
@@ -3378,13 +3504,14 @@ private void loadHistory(File dir) throws IgniteCheckedException {
 
                         if (type == CheckpointEntryType.START) {
                             long cpTs = Long.parseLong(matcher.group(1));
+                            UUID cpId = UUID.fromString(matcher.group(2));
+
                             WALPointer ptr = readPointer(file, buf);
 
                             if (ptr == null)
                                 continue;
 
-                            // Create lazy checkpoint entry.
-                            CheckpointEntry entry = new CheckpointEntry(cpTs, ptr);
+                            CheckpointEntry entry = createCheckPointEntry(cpTs, ptr, cpId, null, type);
 
                             histMap.put(cpTs, entry);
                         }
@@ -3404,8 +3531,6 @@ private CheckpointEntry entry(Long cpTs) throws IgniteCheckedException {
             if (entry == null)
                 throw new IgniteCheckedException("Checkpoint entry was removed: " + cpTs);
 
-            entry.initIfNeeded(cctx);
-
             return entry;
         }
 
@@ -3519,102 +3644,44 @@ private boolean removeCheckpointFiles(CheckpointEntry cpEntry) {
 
             return fail;
         }
-
-        /**
-         * @param cacheId Cache ID.
-         * @param partId Partition ID.
-         * @return Reserved counter or null if couldn't reserve.
-         */
-        @Nullable private Long reserve(int cacheId, int partId) {
-            for (CheckpointEntry entry : histMap.values()) {
-                try {
-                    entry.initIfNeeded(cctx);
-
-                    if (entry.cacheGrpStates == null)
-                        continue;
-
-                    CacheState grpState = entry.cacheGrpStates.get(cacheId);
-
-                    if (grpState == null)
-                        continue;
-
-                    long partCntr = grpState.counterByPartition(partId);
-
-                    if (partCntr >= 0) {
-                        if (cctx.wal().reserve(entry.checkpointMark()))
-                            return partCntr;
-                    }
-                }
-                catch (Exception e) {
-                    U.error(log, "Error while trying to reserve history", e);
-                }
-            }
-
-            return null;
-        }
     }
 
     /**
-     *
+     * Checkpoint entry.
      */
     private static class CheckpointEntry {
-        /** */
-        private static final AtomicIntegerFieldUpdater initGuardUpdater =
-            AtomicIntegerFieldUpdater.newUpdater(CheckpointEntry.class, "initGuard");
-
         /** Checkpoint timestamp. */
         private long cpTs;
 
         /** Checkpoint end mark. */
         private WALPointer cpMark;
 
-        /** Initialization latch. */
-        private CountDownLatch initLatch;
-
-        /** */
-        @SuppressWarnings("unused")
-        private volatile int initGuard;
-
-        /** Checkpoint ID. Initialized lazily. */
+        /** Checkpoint ID. */
         private UUID cpId;
 
-        /** Cache states. Initialized lazily. */
-        private Map cacheGrpStates;
-
-        /** Initialization exception. */
-        private IgniteCheckedException initEx;
+        /** */
+        private volatile SoftReference grpStateLazyStore;
 
         /**
-         * Lazy entry constructor.
+         * Checkpoint entry constructor.
          *
-         * @param cpTs Checkpoint timestamp.
-         * @param cpMark Checkpoint end mark (WAL pointer).
-         */
-        private CheckpointEntry(long cpTs, WALPointer cpMark) {
-            assert cpMark != null;
-
-            this.cpTs = cpTs;
-            this.cpMark = cpMark;
-
-            initLatch = new CountDownLatch(1);
-        }
-
-        /**
-         * Creates complete entry.
+         * If {@code grpStates} is null then it will be inited lazy from wal pointer.
          *
          * @param cpTs Checkpoint timestamp.
          * @param cpMark Checkpoint mark pointer.
          * @param cpId Checkpoint ID.
          * @param cacheGrpStates Cache groups states.
          */
-        private CheckpointEntry(long cpTs, WALPointer cpMark, UUID cpId, Map cacheGrpStates) {
+        private CheckpointEntry(
+            long cpTs,
+            WALPointer cpMark,
+            UUID cpId,
+            @Nullable Map cacheGrpStates
+        ) {
             this.cpTs = cpTs;
             this.cpMark = cpMark;
             this.cpId = cpId;
-            this.cacheGrpStates = cacheGrpStates;
-
-            initGuard = 1;
-            initLatch = new CountDownLatch(0);
+            this.grpStateLazyStore = new SoftReference<>(new GroupStateLazyStore(cacheGrpStates));
         }
 
         /**
@@ -3653,57 +3720,260 @@ private String endFile() {
         }
 
         /**
+         * @param cctx Cache shred context.
+         */
+        public Map groupState(
+            GridCacheSharedContext cctx
+        ) throws IgniteCheckedException {
+            GroupStateLazyStore store = initIfNeeded(cctx);
+
+            return store.grpStates;
+        }
+
+        /**
+         * @param cctx Cache shred context.
+         * @return Group lazy store.
+         */
+        private GroupStateLazyStore initIfNeeded(GridCacheSharedContext cctx) throws IgniteCheckedException {
+            GroupStateLazyStore store = grpStateLazyStore.get();
+
+            if (store == null) {
+                store = new GroupStateLazyStore();
+
+                grpStateLazyStore = new SoftReference<>(store);
+            }
+
+            store.initIfNeeded(cctx, cpMark);
+
+            return store;
+        }
+
+        /**
+         * @param cctx Cache shared context.
          * @param grpId Cache group ID.
          * @param part Partition ID.
          * @return Partition counter or {@code null} if not found.
          */
-        private Long partitionCounter(int grpId, int part) {
-            assert initGuard != 0;
+        private Long partitionCounter(GridCacheSharedContext cctx, int grpId, int part) {
+            GroupStateLazyStore store;
 
-            if (initEx != null || cacheGrpStates == null)
+            try {
+                store = initIfNeeded(cctx);
+            }
+            catch (IgniteCheckedException e) {
                 return null;
+            }
 
-            CacheState state = cacheGrpStates.get(grpId);
+            return store.partitionCounter(grpId, part);
+        }
 
-            if (state != null) {
-                long cntr = state.counterByPartition(part);
+        /**
+         *
+         */
+        private static class GroupState {
+            /** */
+            private int[] parts;
+
+            /** */
+            private long[] cnts;
+
+            /** */
+            private int idx;
+
+            /**
+             * @param partsCnt Partitions count.
+             */
+            private GroupState(int partsCnt) {
+                parts = new int[partsCnt];
+                cnts = new long[partsCnt];
+            }
+
+            /**
+             * @param partId Partition ID to add.
+             * @param cntr Partition counter.
+             */
+            public void addPartitionCounter(int partId, long cntr) {
+                if (idx == parts.length)
+                    throw new IllegalStateException("Failed to add new partition to the partitions state " +
+                        "(no enough space reserved) [partId=" + partId + ", reserved=" + parts.length + ']');
+
+                if (idx > 0) {
+                    if (parts[idx - 1] >= partId)
+                        throw new IllegalStateException("Adding partition in a wrong order [prev=" + parts[idx - 1] +
+                            ", cur=" + partId + ']');
+                }
 
-                return cntr < 0 ? null : cntr;
+                parts[idx] = partId;
+
+                cnts[idx] = cntr;
+
+                idx++;
             }
 
-            return null;
+            /**
+             * Gets partition counter by partition ID.
+             *
+             * @param partId Partition ID.
+             * @return Partition update counter (will return {@code -1} if partition is not present in the record).
+             */
+            public long counterByPartition(int partId) {
+                int idx = indexByPartition(partId);
+
+                return idx >= 0 ? cnts[idx] : 0;
+            }
+
+            public long size(){
+                return idx;
+            }
+
+            /**
+             * @param partId Partition ID to search.
+             * @return Non-negative index of partition if found or negative value if not found.
+             */
+            private int indexByPartition(int partId) {
+                return Arrays.binarySearch(parts, 0, idx, partId);
+            }
+
+            /** {@inheritDoc} */
+            @Override public String toString() {
+                return "GroupState [cap=" + parts.length + ", size=" + idx + ']';
+            }
         }
 
         /**
-         * @throws IgniteCheckedException If failed to read WAL entry.
+         *  Group state lazy store.
          */
-        private void initIfNeeded(GridCacheSharedContext cctx) throws IgniteCheckedException {
-            if (initGuardUpdater.compareAndSet(this, 0, 1)) {
-                try (WALIterator it = cctx.wal().replay(cpMark)) {
-                    if (it.hasNextX()) {
-                        IgniteBiTuple tup = it.nextX();
+        private static class GroupStateLazyStore {
+            /** */
+            private static final AtomicIntegerFieldUpdater initGuardUpdater =
+                AtomicIntegerFieldUpdater.newUpdater(GroupStateLazyStore.class, "initGuard");
 
-                        CheckpointRecord rec = (CheckpointRecord)tup.get2();
+            /** Cache states. Initialized lazily. */
+            private Map grpStates;
 
-                        cpId = rec.checkpointId();
-                        cacheGrpStates = rec.cacheGroupStates();
-                    }
-                    else
-                        initEx = new IgniteCheckedException("Failed to find checkpoint record at " +
-                            "the given WAL pointer: " + cpMark);
-                }
-                catch (IgniteCheckedException e) {
-                    initEx = e;
+            /** */
+            private volatile CountDownLatch latch;
+
+            /** */
+            @SuppressWarnings("unused")
+            private volatile int initGuard;
+
+            /** Initialization exception. */
+            private IgniteCheckedException initEx;
+
+            /**
+             * Default constructor.
+             */
+            private GroupStateLazyStore() {
+                this(null);
+            }
+
+            /**
+             * @param cacheGrpStates Cache group state.
+             */
+            private GroupStateLazyStore(Map cacheGrpStates) {
+                CountDownLatch latch;
+
+                if (cacheGrpStates != null) {
+                    initGuard = 1;
+
+                    this.latch = new CountDownLatch(0);
                 }
-                finally {
-                    initLatch.countDown();
+                else
+                    this.latch = new CountDownLatch(1);
+
+                this.grpStates = remap(cacheGrpStates);
+            }
+
+            /**
+             * @param stateRec Cache group state.
+             */
+            private Map remap(Map stateRec) {
+                if (stateRec == null)
+                    return null;
+
+                Map grpStates = new HashMap<>(stateRec.size());
+
+                for (Integer grpId : stateRec.keySet()) {
+                    CacheState recState = stateRec.get(grpId);
+
+                    GroupState groupState = new GroupState(recState.size());
+
+                    for (int i = 0; i < recState.size(); i++) {
+                        groupState.addPartitionCounter(
+                            recState.partitionByIndex(i),
+                            recState.partitionCounterByIndex(i)
+                        );
+                    }
+
+                    grpStates.put(grpId, groupState);
                 }
+
+                return grpStates;
             }
-            else {
-                U.await(initLatch);
+
+            /**
+             * @param grpId Group id.
+             * @param part Partition id.
+             * @return Partition counter.
+             */
+            private Long partitionCounter(int grpId, int part) {
+                assert initGuard != 0 : initGuard;
 
                 if (initEx != null)
-                    throw initEx;
+                    return null;
+
+                GroupState state = grpStates.get(grpId);
+
+                if (state != null) {
+                    long cntr = state.counterByPartition(part);
+
+                    return cntr < 0 ? null : cntr;
+                }
+
+                return null;
+            }
+
+            /**
+             * @param cctx Cache shared context.
+             * @param ptr Checkpoint wal pointer.
+             * @throws IgniteCheckedException If failed to read WAL entry.
+             */
+            private void initIfNeeded(
+                GridCacheSharedContext cctx,
+                WALPointer ptr
+            ) throws IgniteCheckedException {
+                if (initGuardUpdater.compareAndSet(this, 0, 1)) {
+                    try (WALIterator it = cctx.wal().replay(ptr)) {
+                        if (it.hasNextX()) {
+                            IgniteBiTuple tup = it.nextX();
+
+                            CheckpointRecord rec = (CheckpointRecord)tup.get2();
+
+                            Map stateRec = rec.cacheGroupStates();
+
+                            if (stateRec != null)
+                                this.grpStates = remap(stateRec);
+                            else
+                                grpStates = Collections.emptyMap();
+                        }
+                        else
+                            initEx = new IgniteCheckedException(
+                                "Failed to find checkpoint record at the given WAL pointer: " + ptr);
+                    }
+                    catch (IgniteCheckedException e) {
+                        initEx = e;
+                    }
+                    finally {
+                        latch.countDown();
+                    }
+                }
+                else {
+                    U.await(latch);
+
+                    if (initEx != null)
+                        throw initEx;
+                }
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
index c6d58e5b9f780..66a8aa9c2a366 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
@@ -57,9 +58,13 @@ public class IgniteWalHistoryReservationsTest extends GridCommonAbstractTest {
 
         cfg.setClientMode(client);
 
+        cfg.setConsistentId("NODE$" + gridName.charAt(gridName.length() - 1));
+
         DataStorageConfiguration memCfg = new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(
-                new DataRegionConfiguration().setMaxSize(200 * 1024 * 1024).setPersistenceEnabled(true))
+                new DataRegionConfiguration()
+                    .setMaxSize(200 * 1024 * 1024)
+                    .setPersistenceEnabled(true))
             .setWalMode(WALMode.LOG_ONLY);
 
         cfg.setDataStorageConfiguration(memCfg);
@@ -104,10 +109,14 @@ public void testReservedOnExchange() throws Exception {
         final int entryCnt = 10_000;
         final int initGridCnt = 4;
 
-        final IgniteEx ig0 = (IgniteEx)startGrids(initGridCnt);
+        final IgniteEx ig0 = (IgniteEx)startGrids(initGridCnt + 1);
 
         ig0.active(true);
 
+        stopGrid(initGridCnt);
+
+        Assert.assertEquals(5, ig0.context().state().clusterState().baselineTopology().consistentIds().size());
+
         long start = U.currentTimeMillis();
 
         log.warning("Start loading");

From 3965923369870bb4e8e57e3332c1a1eb1e5f5ed3 Mon Sep 17 00:00:00 2001
From: rkondakov 
Date: Fri, 19 Jan 2018 12:00:55 +0300
Subject: [PATCH 19/65] IGNITE-6772: SQL exception messages became more
 informative. This closes #3342.

---
 .../internal/jdbc2/JdbcErrorsSelfTest.java    |  12 +-
 .../jdbc/JdbcErrorsAbstractSelfTest.java      | 257 +++++++++++++++---
 .../thin/JdbcThinComplexDmlDdlSelfTest.java   |   2 +-
 .../jdbc/thin/JdbcThinErrorsSelfTest.java     |   9 +-
 .../thin/JdbcThinNoDefaultSchemaTest.java     |   2 +-
 .../ignite/internal/jdbc/JdbcResultSet.java   |   2 +-
 .../ignite/internal/jdbc2/JdbcConnection.java |   2 +-
 .../ignite/internal/jdbc2/JdbcResultSet.java  |   2 +-
 .../cache/IgniteCacheProxyImpl.java           |   2 +-
 .../processors/query/h2/IgniteH2Indexing.java |   2 +-
 .../IgniteCacheSqlQueryErrorSelfTest.java     | 214 +++++++++++++++
 .../index/H2DynamicIndexingComplexTest.java   |   2 +-
 .../query/IgniteSqlSchemaIndexingTest.java    |  11 +-
 .../IgniteCacheQuerySelfTestSuite.java        |   2 +
 14 files changed, 465 insertions(+), 56 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheSqlQueryErrorSelfTest.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcErrorsSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcErrorsSelfTest.java
index d33e3a5a4764d..63f0c84a67f76 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcErrorsSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcErrorsSelfTest.java
@@ -41,13 +41,15 @@ public class JdbcErrorsSelfTest extends JdbcErrorsAbstractSelfTest {
      * @throws SQLException if failed.
      */
     public void testConnectionError() throws SQLException {
+        final String path = "jdbc:ignite:сfg://cache=test@/unknown/path";
+
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
-                DriverManager.getConnection("jdbc:ignite:сfg://cache=test@/unknown/path");
+                DriverManager.getConnection(path);
 
                 return null;
             }
-        }, "08001");
+        }, "08001", "No suitable driver found for " + path);
     }
 
     /**
@@ -55,13 +57,15 @@ public void testConnectionError() throws SQLException {
      * @throws SQLException if failed.
      */
     public void testInvalidConnectionStringFormat() throws SQLException {
+        final String cfgPath = "cache=";
+
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
                 // Empty config path yields an error.
-                DriverManager.getConnection("jdbc:ignite:cfg://cache=");
+                DriverManager.getConnection("jdbc:ignite:cfg://" + cfgPath);
 
                 return null;
             }
-        }, "08001");
+        }, "08001", "Failed to start Ignite node. Spring XML configuration path is invalid: " + cfgPath);
     }
 }
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcErrorsAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcErrorsAbstractSelfTest.java
index 6f6d6c52532d6..49746b690eed1 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcErrorsAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcErrorsAbstractSelfTest.java
@@ -81,7 +81,8 @@ public abstract class JdbcErrorsAbstractSelfTest extends GridCommonAbstractTest
      * @throws SQLException if failed.
      */
     public void testParsingErrors() throws SQLException {
-        checkErrorState("gibberish", "42000");
+        checkErrorState("gibberish", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"GIBBERISH[*] \"");
     }
 
     /**
@@ -89,7 +90,7 @@ public void testParsingErrors() throws SQLException {
      * @throws SQLException if failed.
      */
     public void testTableErrors() throws SQLException {
-        checkErrorState("DROP TABLE \"PUBLIC\".missing", "42000");
+        checkErrorState("DROP TABLE \"PUBLIC\".missing", "42000", "Table doesn't exist: MISSING");
     }
 
     /**
@@ -97,7 +98,7 @@ public void testTableErrors() throws SQLException {
      * @throws SQLException if failed.
      */
     public void testIndexErrors() throws SQLException {
-        checkErrorState("DROP INDEX \"PUBLIC\".missing", "42000");
+        checkErrorState("DROP INDEX \"PUBLIC\".missing", "42000", "Index doesn't exist: MISSING");
     }
 
     /**
@@ -105,9 +106,11 @@ public void testIndexErrors() throws SQLException {
      * @throws SQLException if failed.
      */
     public void testDmlErrors() throws SQLException {
-        checkErrorState("INSERT INTO \"test\".INTEGER(_key, _val) values(1, null)", "22004");
+        checkErrorState("INSERT INTO \"test\".INTEGER(_key, _val) values(1, null)", "22004",
+            "Value for INSERT, MERGE, or UPDATE must not be null");
 
-        checkErrorState("INSERT INTO \"test\".INTEGER(_key, _val) values(1, 'zzz')", "0700B");
+        checkErrorState("INSERT INTO \"test\".INTEGER(_key, _val) values(1, 'zzz')", "0700B",
+            "Value conversion failed [from=java.lang.String, to=java.lang.Integer]");
     }
 
     /**
@@ -115,7 +118,8 @@ public void testDmlErrors() throws SQLException {
      * @throws SQLException if failed.
      */
     public void testUnsupportedSql() throws SQLException {
-        checkErrorState("ALTER TABLE \"test\".Integer MODIFY COLUMN _key CHAR", "0A000");
+        checkErrorState("ALTER TABLE \"test\".Integer MODIFY COLUMN _key CHAR", "0A000",
+            "ALTER COLUMN is not supported");
     }
 
     /**
@@ -133,7 +137,7 @@ public void testConnectionClosed() throws SQLException {
 
                 return null;
             }
-        }, "08003");
+        }, "08003", "Connection is closed.");
 
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
@@ -145,7 +149,7 @@ public void testConnectionClosed() throws SQLException {
 
                 return null;
             }
-        }, "08003");
+        }, "08003", "Connection is closed.");
 
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
@@ -157,7 +161,7 @@ public void testConnectionClosed() throws SQLException {
 
                 return null;
             }
-        }, "08003");
+        }, "08003", "Connection is closed.");
 
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
@@ -171,7 +175,7 @@ public void testConnectionClosed() throws SQLException {
 
                 return null;
             }
-        }, "08003");
+        }, "08003", "Connection is closed.");
 
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
@@ -185,7 +189,7 @@ public void testConnectionClosed() throws SQLException {
 
                 return null;
             }
-        }, "08003");
+        }, "08003", "Connection is closed.");
 
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
@@ -199,7 +203,7 @@ public void testConnectionClosed() throws SQLException {
 
                 return null;
             }
-        }, "08003");
+        }, "08003", "Connection is closed.");
 
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
@@ -213,7 +217,7 @@ public void testConnectionClosed() throws SQLException {
 
                 return null;
             }
-        }, "08003");
+        }, "08003", "Connection is closed.");
 
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
@@ -227,7 +231,7 @@ public void testConnectionClosed() throws SQLException {
 
                 return null;
             }
-        }, "08003");
+        }, "08003", "Connection is closed.");
     }
 
     /**
@@ -247,7 +251,7 @@ public void testResultSetClosed() throws SQLException {
                     rs.getInt(1);
                 }
             }
-        }, "24000");
+        }, "24000", "Result set is closed");
     }
 
     /**
@@ -266,7 +270,7 @@ public void testInvalidIntFormat() throws SQLException {
                     rs.getLong(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to long");
     }
 
     /**
@@ -285,7 +289,7 @@ public void testInvalidLongFormat() throws SQLException {
                     rs.getLong(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to long");
     }
 
     /**
@@ -304,7 +308,7 @@ public void testInvalidFloatFormat() throws SQLException {
                     rs.getFloat(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to float");
     }
 
     /**
@@ -323,7 +327,7 @@ public void testInvalidDoubleFormat() throws SQLException {
                     rs.getDouble(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to double");
     }
 
     /**
@@ -342,7 +346,7 @@ public void testInvalidByteFormat() throws SQLException {
                     rs.getByte(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to byte");
     }
 
     /**
@@ -361,7 +365,7 @@ public void testInvalidShortFormat() throws SQLException {
                     rs.getShort(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to short");
     }
 
     /**
@@ -380,7 +384,7 @@ public void testInvalidBigDecimalFormat() throws SQLException {
                     rs.getBigDecimal(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to");
     }
 
     /**
@@ -399,7 +403,7 @@ public void testInvalidBooleanFormat() throws SQLException {
                     rs.getBoolean(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to boolean");
     }
 
     /**
@@ -418,7 +422,7 @@ public void testInvalidObjectFormat() throws SQLException {
                     rs.getObject(1, List.class);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to");
     }
 
     /**
@@ -437,7 +441,7 @@ public void testInvalidDateFormat() throws SQLException {
                     rs.getDate(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to date");
     }
 
     /**
@@ -456,7 +460,7 @@ public void testInvalidTimeFormat() throws SQLException {
                     rs.getTime(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to time");
     }
 
     /**
@@ -475,7 +479,7 @@ public void testInvalidTimestampFormat() throws SQLException {
                     rs.getTimestamp(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to timestamp");
     }
 
     /**
@@ -494,7 +498,7 @@ public void testInvalidUrlFormat() throws SQLException {
                     rs.getURL(1);
                 }
             }
-        }, "0700B");
+        }, "0700B", "Cannot convert to");
     }
 
     /**
@@ -516,7 +520,7 @@ public void testNotNullViolation() throws SQLException {
 
                             return null;
                         }
-                    }, "22004");
+                    }, "22004", "Null value is not allowed for column 'NAME'");
                 }
                 finally {
                     stmt.execute("DROP TABLE nulltest");
@@ -541,7 +545,8 @@ public void testNotNullRestrictionReadThroughCacheStore() throws SQLException {
                         "WITH \"template=" + CACHE_STORE_TEMPLATE + "\"");
                 }
             }
-        }, "0A000");
+        }, "0A000",
+            "NOT NULL constraint is not supported when CacheConfiguration.readThrough is enabled.");
     }
 
     /**
@@ -560,7 +565,151 @@ public void testNotNullRestrictionCacheInterceptor() throws SQLException {
                         "WITH \"template=" + CACHE_INTERCEPTOR_TEMPLATE + "\"");
                 }
             }
-        }, "0A000");
+        }, "0A000", "NOT NULL constraint is not supported when CacheConfiguration.interceptor is set.");
+    }
+
+    /**
+     * Checks wrong table name select error message.
+     *
+     * @throws SQLException If failed.
+     */
+    public void testSelectWrongTable() throws SQLException {
+        checkSqlErrorMessage("select from wrong", "42000",
+            "Failed to parse query. Table \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong column name select error message.
+     *
+     * @throws SQLException If failed.
+     */
+    public void testSelectWrongColumnName() throws SQLException {
+        checkSqlErrorMessage("select wrong from test", "42000",
+            "Failed to parse query. Column \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong syntax select error message.
+     *
+     * @throws SQLException If failed.
+     */
+    public void testSelectWrongSyntax() throws SQLException {
+        checkSqlErrorMessage("select from test where", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"SELECT FROM TEST WHERE[*]");
+    }
+
+    /**
+     * Checks wrong table name DML error message.
+     *
+     * @throws SQLException If failed.
+     */
+    public void testDmlWrongTable() throws SQLException {
+        checkSqlErrorMessage("insert into wrong (id, val) values (3, 'val3')", "42000",
+            "Failed to parse query. Table \"WRONG\" not found");
+
+        checkSqlErrorMessage("merge into wrong (id, val) values (3, 'val3')", "42000",
+            "Failed to parse query. Table \"WRONG\" not found");
+
+        checkSqlErrorMessage("update wrong set val = 'val3' where id = 2", "42000",
+            "Failed to parse query. Table \"WRONG\" not found");
+
+        checkSqlErrorMessage("delete from wrong where id = 2", "42000",
+            "Failed to parse query. Table \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong column name DML error message.
+     *
+     * @throws SQLException If failed.
+     */
+    public void testDmlWrongColumnName() throws SQLException {
+        checkSqlErrorMessage("insert into test (id, wrong) values (3, 'val3')", "42000",
+            "Failed to parse query. Column \"WRONG\" not found");
+
+        checkSqlErrorMessage("merge into test (id, wrong) values (3, 'val3')", "42000",
+            "Failed to parse query. Column \"WRONG\" not found");
+
+        checkSqlErrorMessage("update test set wrong = 'val3' where id = 2", "42000",
+            "Failed to parse query. Column \"WRONG\" not found");
+
+        checkSqlErrorMessage("delete from test where wrong = 2", "42000",
+            "Failed to parse query. Column \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong syntax DML error message.
+     *
+     * @throws SQLException If failed.
+     */
+    public void testDmlWrongSyntax() throws SQLException {
+        checkSqlErrorMessage("insert test (id, val) values (3, 'val3')", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"INSERT TEST[*] (ID, VAL)");
+
+        checkSqlErrorMessage("merge test (id, val) values (3, 'val3')", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"MERGE TEST[*] (ID, VAL)");
+
+        checkSqlErrorMessage("update test val = 'val3' where id = 2", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"UPDATE TEST VAL =[*] 'val3' WHERE ID = 2");
+
+        checkSqlErrorMessage("delete from test 1where id = 2", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"DELETE FROM TEST 1[*]WHERE ID = 2 ");
+    }
+
+    /**
+     * Checks wrong table name DDL error message.
+     *
+     * @throws SQLException If failed.
+     */
+    public void testDdlWrongTable() throws SQLException {
+        checkSqlErrorMessage("create table test (id int primary key, val varchar)", "42000",
+            "Table already exists: TEST");
+
+        checkSqlErrorMessage("drop table wrong", "42000",
+            "Table doesn't exist: WRONG");
+
+        checkSqlErrorMessage("create index idx1 on wrong (val)", "42000",
+            "Table doesn't exist: WRONG");
+
+        checkSqlErrorMessage("drop index wrong", "42000",
+            "Index doesn't exist: WRONG");
+
+        checkSqlErrorMessage("alter table wrong drop column val", "42000",
+            "Failed to parse query. Table \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong column name DDL error message.
+     *
+     * @throws SQLException If failed.
+     */
+    public void testDdlWrongColumnName() throws SQLException {
+        checkSqlErrorMessage("create index idx1 on test (wrong)", "42000",
+            "Column doesn't exist: WRONG");
+
+        checkSqlErrorMessage("alter table test drop column wrong", "42000",
+            "Failed to parse query. Column \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong syntax DDL error message.
+     *
+     * @throws SQLException If failed.
+     */
+    public void testDdlWrongSyntax() throws SQLException {
+        checkSqlErrorMessage("create table test2 (id int wrong key, val varchar)", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"CREATE TABLE TEST2 (ID INT WRONG[*]");
+
+        checkSqlErrorMessage("drop table test on", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"DROP TABLE TEST ON[*]");
+
+        checkSqlErrorMessage("create index idx1 test (val)", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"CREATE INDEX IDX1 TEST[*]");
+
+        checkSqlErrorMessage("drop index", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"DROP INDEX [*]");
+
+        checkSqlErrorMessage("alter table test drop column", "42000",
+            "Failed to parse query. Syntax error in SQL statement \"ALTER TABLE TEST DROP COLUMN [*]");
     }
 
     /**
@@ -576,14 +725,14 @@ public void testNotNullRestrictionCacheInterceptor() throws SQLException {
      * @throws SQLException if failed.
      */
     @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    private void checkErrorState(final String sql, String expState) throws SQLException {
+    private void checkErrorState(final String sql, String expState, String expMsg) throws SQLException {
         checkErrorState(new ConnClosure() {
             @Override public void run(Connection conn) throws Exception {
                 try (final PreparedStatement stmt = conn.prepareStatement(sql)) {
                     stmt.execute();
                 }
             }
-        }, expState);
+        }, expState, expMsg);
     }
 
     /**
@@ -593,7 +742,7 @@ private void checkErrorState(final String sql, String expState) throws SQLExcept
      * @throws SQLException if failed.
      */
     @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    protected void checkErrorState(final ConnClosure clo, String expState) throws SQLException {
+    protected void checkErrorState(final ConnClosure clo, String expState, String expMsg) throws SQLException {
         checkErrorState(new IgniteCallable() {
             @Override public Void call() throws Exception {
                 try (final Connection conn = getConnection()) {
@@ -604,7 +753,7 @@ protected void checkErrorState(final ConnClosure clo, String expState) throws SQ
                     return null;
                 }
             }
-        }, expState);
+        }, expState, expMsg);
     }
 
     /**
@@ -614,12 +763,46 @@ protected void checkErrorState(final ConnClosure clo, String expState) throws SQ
      * @throws SQLException if failed.
      */
     @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    protected void checkErrorState(final IgniteCallable clo, String expState) throws SQLException {
-        SQLException ex = (SQLException)GridTestUtils.assertThrows(null, clo, SQLException.class, null);
+    protected void checkErrorState(final IgniteCallable clo, String expState, String expMsg) throws SQLException {
+        SQLException ex = (SQLException)GridTestUtils.assertThrows(null, clo, SQLException.class, expMsg);
 
         assertEquals(expState, ex.getSQLState());
     }
 
+    /**
+     * Check SQL exception message and error code.
+     *
+     * @param sql Query string.
+     * @param expState Error code.
+     * @param expMsg Error message.
+     * @throws SQLException if failed.
+     */
+    private void checkSqlErrorMessage(final String sql, String expState, String expMsg) throws SQLException {
+        checkErrorState(new IgniteCallable() {
+            @Override public Void call() throws Exception {
+                try (final Connection conn = getConnection()) {
+                    conn.setSchema("PUBLIC");
+
+                    try (Statement stmt = conn.createStatement()) {
+                        stmt.executeUpdate("DROP TABLE IF EXISTS wrong");
+                        stmt.executeUpdate("DROP TABLE IF EXISTS test");
+
+                        stmt.executeUpdate("CREATE TABLE test (id INT PRIMARY KEY, val VARCHAR)");
+
+                        stmt.executeUpdate("INSERT INTO test (id, val) VALUES (1, 'val1')");
+                        stmt.executeUpdate("INSERT INTO test (id, val) VALUES (2, 'val2')");
+
+                        stmt.execute(sql);
+
+                        fail("Exception is expected");
+                    }
+
+                    return null;
+                }
+            }
+        }, expState, expMsg);
+    }
+
     /**
      * Runnable that accepts a {@link Connection} and can throw an exception.
      */
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSelfTest.java
index cdc91307bff41..7941abe7a85ea 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSelfTest.java
@@ -145,7 +145,7 @@ public void testCreateSelect() throws Exception {
 
                 return null;
             }
-        }, SQLException.class, "Failed to parse query: SELECT * from Person");
+        }, SQLException.class, "Table \"PERSON\" not found");
 
         sql(new UpdateChecker(0),
             "CREATE TABLE person (id int, name varchar, age int, company varchar, city varchar, " +
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinErrorsSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinErrorsSelfTest.java
index a286fc17a7698..90588c4e6e50b 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinErrorsSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinErrorsSelfTest.java
@@ -48,7 +48,7 @@ public void testConnectionError() throws SQLException {
 
                 return null;
             }
-        }, "08001");
+        }, "08001", "Failed to connect to Ignite cluster [host=unknown.host");
     }
 
     /**
@@ -63,7 +63,7 @@ public void testInvalidConnectionStringFormat() throws SQLException {
 
                 return null;
             }
-        }, "08001");
+        }, "08001", "Property cannot be upper than 65535");
     }
 
     /**
@@ -76,7 +76,7 @@ public void testInvalidIsolationLevel() throws SQLException {
             @Override public void run(Connection conn) throws Exception {
                 conn.setTransactionIsolation(1000);
             }
-        }, "0700E");
+        }, "0700E", "Invalid transaction isolation level.");
     }
 
     /**
@@ -103,6 +103,9 @@ public void testBatchUpdateException() throws SQLException {
                 assertArrayEquals("", new int[] {1, 1, Statement.EXECUTE_FAILED}, e.getUpdateCounts());
 
                 assertEquals("42000", e.getSQLState());
+
+                assertTrue("Unexpected error message: " + e.getMessage(), e.getMessage() != null &&
+                    e.getMessage().contains("Failed to parse query. Column \"ID1\" not found"));
             }
         }
     }
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinNoDefaultSchemaTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinNoDefaultSchemaTest.java
index a1be582003e49..11aef9fa87faf 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinNoDefaultSchemaTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinNoDefaultSchemaTest.java
@@ -217,7 +217,7 @@ public void testSetSchema() throws Exception {
 
                     return null;
                 }
-            }, SQLException.class, "Failed to parse query");
+            }, SQLException.class, "Table \"INTEGER\" not found");
 
             conn.setSchema("\"cache1\"");
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcResultSet.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcResultSet.java
index ee1547129bd3d..544207e52b460 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcResultSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcResultSet.java
@@ -1523,7 +1523,7 @@ else if (cls == String.class)
             throw new SQLException("Invalid column index: " + colIdx);
         }
         catch (ClassCastException ignored) {
-            throw new SQLException("Value is an not instance of " + cls.getName());
+            throw new SQLException("Cannot convert to " + cls.getSimpleName().toLowerCase());
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
index 29cb6a1669dea..b51e0b95084ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
@@ -248,7 +248,7 @@ public JdbcConnection(String url, Properties props) throws SQLException {
         catch (Exception e) {
             close();
 
-            throw convertToSqlException(e, "Failed to start Ignite node.", SqlStateCode.CLIENT_CONNECTION_FAILED);
+            throw convertToSqlException(e, "Failed to start Ignite node. " + e.getMessage(), SqlStateCode.CLIENT_CONNECTION_FAILED);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
index e2ff5d866dffb..e6e84880ea2eb 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java
@@ -1556,7 +1556,7 @@ else if (cls == String.class)
             throw new SQLException("Invalid column index: " + colIdx);
         }
         catch (ClassCastException ignored) {
-            throw new SQLException("Value is an not instance of " + cls.getName(), SqlStateCode.CONVERSION_FAILED);
+            throw new SQLException("Cannot convert to " + cls.getSimpleName().toLowerCase(), SqlStateCode.CONVERSION_FAILED);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index 1cbd2b0909343..dc6793b2313e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -615,7 +615,7 @@ private QueryCursor> queryContinuous(ContinuousQuery qry, bool
             if (e instanceof CacheException)
                 throw (CacheException)e;
 
-            throw new CacheException(e);
+            throw new CacheException(e.getMessage(), e);
         }
     }
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 9cf86d08cea97..38a04c4a9328e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -1519,7 +1519,7 @@ private List>> tryQueryDistributedSqlFieldsNative(Stri
                                 cachesCreated = true;
                             }
                             else
-                                throw new IgniteSQLException("Failed to parse query: " + sqlQry,
+                                throw new IgniteSQLException("Failed to parse query. " + e.getMessage(),
                                     IgniteQueryErrorCode.PARSING, e);
                         }
                     }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheSqlQueryErrorSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheSqlQueryErrorSelfTest.java
new file mode 100644
index 0000000000000..09790854b2d03
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheSqlQueryErrorSelfTest.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.concurrent.Callable;
+import javax.cache.CacheException;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Java API query error messages test.
+ */
+public class IgniteCacheSqlQueryErrorSelfTest  extends GridCacheAbstractSelfTest {
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 1;
+    }
+
+    /**
+     * Checks wrong table name select error message.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectWrongTable() throws Exception {
+        checkSqlErrorMessage("select from wrong",
+            "Failed to parse query. Table \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong column name select error message.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectWrongColumnName() throws Exception {
+        checkSqlErrorMessage("select wrong from test",
+            "Failed to parse query. Column \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong syntax select error message.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSelectWrongSyntax() throws Exception {
+        checkSqlErrorMessage("select from test where",
+            "Failed to parse query. Syntax error in SQL statement \"SELECT FROM TEST WHERE[*]");
+    }
+
+    /**
+     * Checks wrong table name DML error message.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDmlWrongTable() throws Exception {
+        checkSqlErrorMessage("insert into wrong (id, val) values (3, 'val3')",
+            "Failed to parse query. Table \"WRONG\" not found");
+
+        checkSqlErrorMessage("merge into wrong (id, val) values (3, 'val3')",
+            "Failed to parse query. Table \"WRONG\" not found");
+
+        checkSqlErrorMessage("update wrong set val = 'val3' where id = 2",
+            "Failed to parse query. Table \"WRONG\" not found");
+
+        checkSqlErrorMessage("delete from wrong where id = 2",
+            "Failed to parse query. Table \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong column name DML error message.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDmlWrongColumnName() throws Exception {
+        checkSqlErrorMessage("insert into test (id, wrong) values (3, 'val3')",
+            "Failed to parse query. Column \"WRONG\" not found");
+
+        checkSqlErrorMessage("merge into test (id, wrong) values (3, 'val3')",
+            "Failed to parse query. Column \"WRONG\" not found");
+
+        checkSqlErrorMessage("update test set wrong = 'val3' where id = 2",
+            "Failed to parse query. Column \"WRONG\" not found");
+
+        checkSqlErrorMessage("delete from test where wrong = 2",
+            "Failed to parse query. Column \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong syntax DML error message.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDmlWrongSyntax() throws Exception {
+        checkSqlErrorMessage("insert test (id, val) values (3, 'val3')",
+            "Failed to parse query. Syntax error in SQL statement \"INSERT TEST[*] (ID, VAL)");
+
+        checkSqlErrorMessage("merge test (id, val) values (3, 'val3')",
+            "Failed to parse query. Syntax error in SQL statement \"MERGE TEST[*] (ID, VAL)");
+
+        checkSqlErrorMessage("update test val = 'val3' where id = 2",
+            "Failed to parse query. Syntax error in SQL statement \"UPDATE TEST VAL =[*] 'val3' WHERE ID = 2");
+
+        checkSqlErrorMessage("delete from test 1where id = 2",
+            "Failed to parse query. Syntax error in SQL statement \"DELETE FROM TEST 1[*]WHERE ID = 2 ");
+    }
+
+    /**
+     * Checks wrong table name DDL error message.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDdlWrongTable() throws Exception {
+        checkSqlErrorMessage("create table test (id int primary key, val varchar)",
+            "Table already exists: TEST");
+
+        checkSqlErrorMessage("drop table wrong",
+            "Table doesn't exist: WRONG");
+
+        checkSqlErrorMessage("create index idx1 on wrong (val)",
+            "Table doesn't exist: WRONG");
+
+        checkSqlErrorMessage("drop index wrong",
+            "Index doesn't exist: WRONG");
+
+        checkSqlErrorMessage("alter table wrong drop column val",
+            "Failed to parse query. Table \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong column name DDL error message.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDdlWrongColumnName() throws Exception {
+        checkSqlErrorMessage("create index idx1 on test (wrong)",
+            "Column doesn't exist: WRONG");
+
+        checkSqlErrorMessage("alter table test drop column wrong",
+            "Failed to parse query. Column \"WRONG\" not found");
+    }
+
+    /**
+     * Checks wrong syntax DDL error message.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDdlWrongSyntax() throws Exception {
+        checkSqlErrorMessage("create table wrong (id int wrong key, val varchar)",
+            "Failed to parse query. Syntax error in SQL statement \"CREATE TABLE WRONG (ID INT WRONG[*]");
+
+        checkSqlErrorMessage("drop table test on",
+            "Failed to parse query. Syntax error in SQL statement \"DROP TABLE TEST ON[*]");
+
+        checkSqlErrorMessage("create index idx1 test (val)",
+            "Failed to parse query. Syntax error in SQL statement \"CREATE INDEX IDX1 TEST[*]");
+
+        checkSqlErrorMessage("drop index",
+            "Failed to parse query. Syntax error in SQL statement \"DROP INDEX [*]");
+
+        checkSqlErrorMessage("alter table test drop column",
+            "Failed to parse query. Syntax error in SQL statement \"ALTER TABLE TEST DROP COLUMN [*]");
+    }
+
+    /**
+     * Checks SQL error message.
+     *
+     * @param sql SQL command.
+     * @param expMsg Expected error message.
+     */
+    private void checkSqlErrorMessage(final String sql, String expMsg) {
+        execute("DROP TABLE IF EXISTS wrong");
+        execute("DROP TABLE IF EXISTS test");
+
+        execute("CREATE TABLE test (id INT PRIMARY KEY, val VARCHAR)");
+
+        execute("INSERT INTO test (id, val) VALUES (1, 'val1')");
+        execute("INSERT INTO test (id, val) VALUES (2, 'val2')");
+
+        GridTestUtils.assertThrows(null, new Callable() {
+
+            @Override public Object call() throws Exception {
+                execute(sql);
+
+                fail("Exception is expected");
+
+                return null;
+            }
+        }, CacheException.class, expMsg);
+    }
+
+    /**
+     * Executes SQL command.
+     *
+     * @param sql SQL command.
+     */
+    private void execute(String sql) {
+       jcache().query(new SqlFieldsQuery(sql).setSchema("PUBLIC")).getAll();
+    }
+
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexingComplexTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexingComplexTest.java
index 68df58b4336b5..02e48ed26451e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexingComplexTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexingComplexTest.java
@@ -199,7 +199,7 @@ public void testOperations() {
             @Override public Object call() throws Exception {
                 return executeSql("SELECT * from Person");
             }
-        }, IgniteSQLException.class, "Failed to parse query: SELECT * from Person");
+        }, IgniteSQLException.class, "Table \"PERSON\" not found");
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSchemaIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSchemaIndexingTest.java
index 570d2db605823..e375df2cf6e7e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSchemaIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSchemaIndexingTest.java
@@ -190,9 +190,11 @@ public void testSchemaEscapeAll() throws Exception {
             .setSqlSchema("\"SchemaName2\"")
             .setSqlEscapeAll(true);
 
-        escapeCheckSchemaName(ignite(0).createCache(cfg), log, cfg.getSqlSchema(), false);
+        escapeCheckSchemaName(ignite(0).createCache(cfg), log, cfg.getSqlSchema(), false,
+            "Table \"FACT\" not found");
 
-        escapeCheckSchemaName(ignite(0).createCache(cfgEsc), log, "SchemaName2", true);
+        escapeCheckSchemaName(ignite(0).createCache(cfgEsc), log, "SchemaName2", true,
+            "Schema \"SCHEMANAME2\" not found");
 
         ignite(0).destroyCache(cfg.getName());
         ignite(0).destroyCache(cfgEsc.getName());
@@ -204,9 +206,10 @@ public void testSchemaEscapeAll() throws Exception {
      * @param log logger for assertThrows
      * @param schemaName Schema name without quotes for testing
      * @param caseSensitive Whether schema name is case sensitive.
+     * @param msg Expected error message.
      */
     private static void escapeCheckSchemaName(final IgniteCache cache, IgniteLogger log,
-        String schemaName, boolean caseSensitive) {
+        String schemaName, boolean caseSensitive, String msg) {
         final SqlFieldsQuery qryWrong = new SqlFieldsQuery("select f.id, f.name " +
             "from " + schemaName.toUpperCase() + ".Fact f");
 
@@ -218,7 +221,7 @@ private static void escapeCheckSchemaName(final IgniteCache cache
 
                 return null;
             }
-        }, CacheException.class, "Failed to parse query");
+        }, CacheException.class, msg);
 
         if (caseSensitive)
             schemaName = "\"" + schemaName + "\"";
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 8cfa14e8b5672..f4eb393425ee1 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -70,6 +70,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheQueryH2IndexingLeakTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheQueryIndexSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheQueryLoadSelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheSqlQueryErrorSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheUpdateSqlQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCheckClusterStateBeforeExecuteQueryTest;
 import org.apache.ignite.internal.processors.cache.IgniteCrossCachesJoinsQueryTest;
@@ -213,6 +214,7 @@ public static TestSuite suite() throws Exception {
 
         // Parsing
         suite.addTestSuite(GridQueryParsingTest.class);
+        suite.addTestSuite(IgniteCacheSqlQueryErrorSelfTest.class);
 
         // Config.
         suite.addTestSuite(IgniteCacheDuplicateEntityConfigurationSelfTest.class);

From b54c0c8786bd74aa0abb208f537c29f0c4be4b1e Mon Sep 17 00:00:00 2001
From: tledkov-gridgain 
Date: Fri, 19 Jan 2018 12:09:34 +0300
Subject: [PATCH 20/65] IGNITE-7248: JDBC: fixed schema resolution for
 streaming mode. This closes #3384.

---
 .../jdbc2/JdbcStreamingToPublicCacheTest.java | 148 ++++++++++++++++++
 .../jdbc/suite/IgniteJdbcDriverTestSuite.java |   1 +
 .../processors/query/GridQueryProcessor.java  |   8 +-
 3 files changed, 152 insertions(+), 5 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingToPublicCacheTest.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingToPublicCacheTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingToPublicCacheTest.java
new file mode 100644
index 0000000000000..b973f10aa2b39
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingToPublicCacheTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Collections;
+import java.util.Properties;
+import org.apache.ignite.IgniteJdbcDriver;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Data streaming test.
+ */
+public class JdbcStreamingToPublicCacheTest extends GridCommonAbstractTest {
+    /** JDBC URL. */
+    private static final String BASE_URL = CFG_URL_PREFIX + "cache=%s@modules/clients/src/test/config/jdbc-config.xml";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration cache = defaultCacheConfiguration();
+
+        cache.setCacheMode(PARTITIONED);
+        cache.setBackups(1);
+        cache.setWriteSynchronizationMode(FULL_SYNC);
+        cache.setIndexedTypes(
+            Integer.class, Integer.class
+        );
+
+        cfg.setCacheConfiguration(cache);
+        cfg.setLocalHost("127.0.0.1");
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+        ipFinder.setAddresses(Collections.singleton("127.0.0.1:47500..47501"));
+
+        disco.setIpFinder(ipFinder);
+
+        cfg.setDiscoverySpi(disco);
+
+        cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @param cacheName Cache name.
+     * @param streaming Streaming mode flag.
+     * @return Connection to use for the test.
+     * @throws Exception if failed.
+     */
+    private Connection createConnection(String cacheName, boolean streaming) throws Exception {
+        Properties props = new Properties();
+
+        if (streaming) {
+            props.setProperty(IgniteJdbcDriver.PROP_STREAMING, "true");
+            props.setProperty(IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ, "500");
+        }
+
+        return DriverManager.getConnection(String.format(BASE_URL, cacheName), props);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        ignite(0).cache(DEFAULT_CACHE_NAME).clear();
+
+        super.afterTest();
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testStreamedInsert() throws Exception {
+        // Create table
+        try (Connection conn = createConnection(DEFAULT_CACHE_NAME, false)) {
+            Statement stmt = conn.createStatement();
+
+            stmt.execute("create table PUBLIC.STREAM_TEST (ID int primary key, str_val varchar)");
+        }
+
+        // Fill table with streaming
+        try (Connection conn = createConnection("SQL_PUBLIC_STREAM_TEST", true)) {
+            PreparedStatement pstmt = conn.prepareStatement("insert into STREAM_TEST(id, str_val) values (?, ?)");
+
+            for (int i = 1; i <= 100; i++) {
+                pstmt.setInt(1, i);
+                pstmt.setString(2, "val_" + i);
+
+                pstmt.executeUpdate();
+            }
+        }
+
+        // Check table's data
+        try (Connection conn = createConnection("SQL_PUBLIC_STREAM_TEST", false)) {
+            ResultSet rs = conn.createStatement().executeQuery("select id, str_val from STREAM_TEST");
+
+            int cnt = 0;
+
+            while (rs.next()) {
+                assertEquals("val_" + rs.getInt(1), rs.getString(2));
+
+                cnt++;
+            }
+
+            assertEquals(100, cnt);
+        }
+    }
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index bec388a0a9bb4..062de76efde95 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -110,6 +110,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDeleteStatementSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStatementBatchingSelfTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcErrorsSelfTest.class));
+        suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStreamingToPublicCacheTest.class));
 
         suite.addTest(new TestSuite(JdbcBlobTest.class));
         suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStreamingSelfTest.class));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 3bbe8f1e9ec6f..9184a497a09f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -2468,16 +2468,14 @@ private void processDynamicDropColumn(QueryTypeDescriptorImpl d, List co
     }
 
     /**
-     *
-     * @param cacheName Cache name.
+     * @param schemaName Schema name.
      * @param sql Query.
      * @return {@link PreparedStatement} from underlying engine to supply metadata to Prepared - most likely H2.
+     * @throws SQLException On error.
      */
-    public PreparedStatement prepareNativeStatement(String cacheName, String sql) throws SQLException {
+    public PreparedStatement prepareNativeStatement(String schemaName, String sql) throws SQLException {
         checkxEnabled();
 
-        String schemaName = idx.schema(cacheName);
-
         return idx.prepareNativeStatement(schemaName, sql);
     }
 

From 2f5997788ccff265a088921210f561985f640517 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin 
Date: Fri, 19 Jan 2018 14:46:38 +0300
Subject: [PATCH 21/65] IGNITE-7471 fix npe

---
 .../GridCacheDatabaseSharedManager.java       | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index e56b3521e8a64..580fb3a268f63 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1537,13 +1537,18 @@ private Map> searchLastCheckpointEntryPer
     ) {
         final Map> res = new HashMap<>();
 
+        if (F.isEmpty(part4reserve))
+            return res;
+
         for (Long cpTs : checkpointHist.checkpoints()) {
+            CheckpointEntry chpEntry = null;
+
             try {
-                final CheckpointEntry chpEntry = checkpointHist.entry(cpTs);
+                chpEntry = checkpointHist.entry(cpTs);
 
                 Map grpsState = chpEntry.groupState(cctx);
 
-                if (grpsState.isEmpty()){
+                if (F.isEmpty(grpsState)) {
                     res.clear();
 
                     continue;
@@ -1579,7 +1584,12 @@ private Map> searchLastCheckpointEntryPer
                     }
                 }
             }
-            catch (IgniteCheckedException ignore) {
+            catch (IgniteCheckedException ex) {
+                String msg = chpEntry != null ?
+                    ", chpId=" + chpEntry.cpId + " ptr=" + chpEntry.cpMark + " ts=" + chpEntry.cpTs : "";
+
+                U.error(log, "Failed to read checkpoint entry" + msg, ex);
+
                 // Treat exception the same way as a gap.
                 res.clear();
             }
@@ -3849,10 +3859,10 @@ private static class GroupStateLazyStore {
                 AtomicIntegerFieldUpdater.newUpdater(GroupStateLazyStore.class, "initGuard");
 
             /** Cache states. Initialized lazily. */
-            private Map grpStates;
+            private volatile Map grpStates;
 
             /** */
-            private volatile CountDownLatch latch;
+            private final CountDownLatch latch;
 
             /** */
             @SuppressWarnings("unused")
@@ -3920,7 +3930,7 @@ private Map remap(Map stateRec) {
             private Long partitionCounter(int grpId, int part) {
                 assert initGuard != 0 : initGuard;
 
-                if (initEx != null)
+                if (initEx != null || grpStates == null)
                     return null;
 
                 GroupState state = grpStates.get(grpId);
@@ -3963,6 +3973,8 @@ private void initIfNeeded(
                     }
                     catch (IgniteCheckedException e) {
                         initEx = e;
+
+                        throw e;
                     }
                     finally {
                         latch.countDown();

From 7adce10750704cc50cbf54fa7020aa3b20da87cb Mon Sep 17 00:00:00 2001
From: Oleg Ignatenko 
Date: Fri, 19 Jan 2018 14:59:01 +0300
Subject: [PATCH 22/65] IGNITE-7454 Wrong package in IgniteExamplesMLTestSuite

this closes #3393

(cherry picked from commit cb1233a)
---
 .../examples/datagrid/package-info.java       |  2 +-
 .../testsuites/IgniteExamplesMLTestSuite.java | 52 +++++++++++--------
 .../IgniteExamplesSelfTestSuite.java          |  5 +-
 ...zyCMeansDistributedClustererBenchmark.java |  0
 ...iteFuzzyCMeansLocalClustererBenchmark.java |  0
 .../ignite/yardstick/ml/knn/Datasets.java     |  0
 .../knn/IgniteKNNClassificationBenchmark.java |  0
 .../ml/knn/IgniteKNNRegressionBenchmark.java  |  0
 .../ignite/yardstick/ml/knn/package-info.java |  0
 ...eSparseDistributedMatrixMul2Benchmark.java |  0
 10 files changed, 36 insertions(+), 23 deletions(-)
 rename modules/yardstick/src/main/{ml => java}/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java (100%)
 rename modules/yardstick/src/main/{ml => java}/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java (100%)
 rename modules/yardstick/src/main/{ml => java}/org/apache/ignite/yardstick/ml/knn/Datasets.java (100%)
 rename modules/yardstick/src/main/{ml => java}/org/apache/ignite/yardstick/ml/knn/IgniteKNNClassificationBenchmark.java (100%)
 rename modules/yardstick/src/main/{ml => java}/org/apache/ignite/yardstick/ml/knn/IgniteKNNRegressionBenchmark.java (100%)
 rename modules/yardstick/src/main/{ml => java}/org/apache/ignite/yardstick/ml/knn/package-info.java (100%)
 rename modules/yardstick/src/main/{ml => java}/org/apache/ignite/yardstick/ml/math/IgniteSparseDistributedMatrixMul2Benchmark.java (100%)

diff --git a/examples/src/main/java/org/apache/ignite/examples/datagrid/package-info.java b/examples/src/main/java/org/apache/ignite/examples/datagrid/package-info.java
index e8c54ac64192e..cfa37a4fb33b1 100644
--- a/examples/src/main/java/org/apache/ignite/examples/datagrid/package-info.java
+++ b/examples/src/main/java/org/apache/ignite/examples/datagrid/package-info.java
@@ -19,4 +19,4 @@
  * 
  * Demonstrates data ignite cache usage.
  */
-package org.apache.ignite.examples.jav.datagrid;
\ No newline at end of file
+package org.apache.ignite.examples.datagrid;
diff --git a/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesMLTestSuite.java b/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesMLTestSuite.java
index bc917b901e6fd..d2f40e61148cd 100644
--- a/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesMLTestSuite.java
+++ b/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesMLTestSuite.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.testsuites;
+package org.apache.ignite.testsuites;
 
 import java.io.File;
 import java.io.IOException;
@@ -38,11 +38,19 @@
 /**
  * Examples test suite.
  * 

- * Contains only Spring ignite examples tests.
+ * Contains only ML Grid Ignite examples tests.

  */
 public class IgniteExamplesMLTestSuite extends TestSuite {
+    /** Base package to create test classes in. */
+    private static final String basePkgForTests = "org.apache.ignite.examples.ml";
+
+    /** Test class name pattern. */
+    private static final String clsNamePtrn = ".*Example$";
+
     /**
-     * @return Suite.
+     * Creates test suite for Ignite ML examples.
+     *
+     * @return Created suite.
      * @throws Exception If failed.
      */
     public static TestSuite suite() throws Exception {
@@ -51,8 +59,8 @@ public static TestSuite suite() throws Exception {
 
         TestSuite suite = new TestSuite("Ignite ML Examples Test Suite");
 
-        for (Class clazz : getClasses("org.apache.ignite.examples.ml", ".*Example$"))
-            suite.addTest(new TestSuite(makeTestClass(clazz, "org.apache.ignite.ml.examples")));
+        for (Class clazz : getClasses(basePkgForTests))
+            suite.addTest(new TestSuite(makeTestClass(clazz)));
 
         return suite;
     }
@@ -60,13 +68,12 @@ public static TestSuite suite() throws Exception {
     /**
      * Creates test class for given example.
      *
-     * @param exampleCls Class of the example to be tested
-     * @param basePkgForTests Base package to create test classes in
-     * @return Test class
-     * @throws NotFoundException if class not found
-     * @throws CannotCompileException if test class cannot be compiled
+     * @param exampleCls Class of the example to be tested.
+     * @return Test class.
+     * @throws NotFoundException If class not found.
+     * @throws CannotCompileException If test class cannot be compiled.
      */
-    private static Class makeTestClass(Class exampleCls, String basePkgForTests)
+    private static Class makeTestClass(Class exampleCls)
         throws NotFoundException, CannotCompileException {
         ClassPool cp = ClassPool.getDefault();
         cp.insertClassPath(new ClassClassPath(IgniteExamplesMLTestSuite.class));
@@ -87,12 +94,12 @@ private static Class makeTestClass(Class exampleCls, String basePkgForTests)
     /**
      * Scans all classes accessible from the context class loader which belong to the given package and subpackages.
      *
-     * @param pkgName The base package
-     * @return The classes
-     * @throws ClassNotFoundException if some classes not found
-     * @throws IOException if some resources unavailable
+     * @param pkgName The base package.
+     * @return The classes.
+     * @throws ClassNotFoundException If some classes not found.
+     * @throws IOException If some resources unavailable.
      */
-    private static List getClasses(String pkgName, String clsNamePtrn) throws ClassNotFoundException, IOException {
+    private static List getClasses(String pkgName) throws ClassNotFoundException, IOException {
         String path = pkgName.replace('.', '/');
 
         Enumeration resources = Thread.currentThread()
@@ -111,15 +118,17 @@ private static List getClasses(String pkgName, String clsNamePtrn) throws
     }
 
     /**
-     * Recursive method used to find all classes in a given directory and subdirs.
+     * Recursive method used to find all classes in a given directory and sub-dirs.
      *
-     * @param dir The base directory
-     * @param pkgName The package name for classes found inside the base directory
-     * @return The classes
-     * @throws ClassNotFoundException if class not found
+     * @param dir The base directory.
+     * @param pkgName The package name for classes found inside the base directory.
+     * @param clsNamePtrn Class name pattern.
+     * @return The classes.
+     * @throws ClassNotFoundException If class not found.
      */
     private static List findClasses(File dir, String pkgName, String clsNamePtrn) throws ClassNotFoundException {
         List classes = new ArrayList<>();
+
         if (!dir.exists())
             return classes;
 
@@ -130,6 +139,7 @@ private static List findClasses(File dir, String pkgName, String clsNameP
                     classes.addAll(findClasses(file, pkgName + "." + file.getName(), clsNamePtrn));
                 else if (file.getName().endsWith(".class")) {
                     String clsName = pkgName + '.' + file.getName().substring(0, file.getName().length() - 6);
+
                     if (clsName.matches(clsNamePtrn))
                         classes.add(Class.forName(clsName));
                 }
diff --git a/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesSelfTestSuite.java b/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesSelfTestSuite.java
index 0eea40ae6ec8d..1eeedf7ff40f2 100644
--- a/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesSelfTestSuite.java
+++ b/examples/src/test/java/org/apache/ignite/testsuites/IgniteExamplesSelfTestSuite.java
@@ -50,7 +50,7 @@
 /**
  * Examples test suite.
  * 

- * Contains only Spring ignite examples tests.
+ * Contains all Ignite examples tests.

  */
 public class IgniteExamplesSelfTestSuite extends TestSuite {
     /**
@@ -96,6 +96,9 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(CacheClientBinaryExampleTest.class));
         suite.addTest(new TestSuite(ComputeClientBinaryExampleTest.class));
 
+        // ML Grid.
+        suite.addTest(IgniteExamplesMLTestSuite.suite());
+
         return suite;
     }
 }
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java
similarity index 100%
rename from modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java
rename to modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java
similarity index 100%
rename from modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java
rename to modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/knn/Datasets.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/knn/Datasets.java
similarity index 100%
rename from modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/knn/Datasets.java
rename to modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/knn/Datasets.java
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/knn/IgniteKNNClassificationBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/knn/IgniteKNNClassificationBenchmark.java
similarity index 100%
rename from modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/knn/IgniteKNNClassificationBenchmark.java
rename to modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/knn/IgniteKNNClassificationBenchmark.java
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/knn/IgniteKNNRegressionBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/knn/IgniteKNNRegressionBenchmark.java
similarity index 100%
rename from modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/knn/IgniteKNNRegressionBenchmark.java
rename to modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/knn/IgniteKNNRegressionBenchmark.java
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/knn/package-info.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/knn/package-info.java
similarity index 100%
rename from modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/knn/package-info.java
rename to modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/knn/package-info.java
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/IgniteSparseDistributedMatrixMul2Benchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/math/IgniteSparseDistributedMatrixMul2Benchmark.java
similarity index 100%
rename from modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/math/IgniteSparseDistributedMatrixMul2Benchmark.java
rename to modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/math/IgniteSparseDistributedMatrixMul2Benchmark.java

From 25e91b60694c589d8bf50c63a0d898ca4180b428 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn 
Date: Fri, 19 Jan 2018 16:19:52 +0300
Subject: [PATCH 23/65] IGNITE-7117 .NET: Improve IgniteHome resolver

This closes #3402
---
 .../Impl/Common/IgniteHome.cs                 | 29 ++++++++++++-------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteHome.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteHome.cs
index d61c6b4c2ccfc..615798c8570e8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteHome.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/IgniteHome.cs
@@ -112,16 +112,25 @@ private static bool IsIgniteHome(DirectoryInfo dir)
         {
             try
             {
-                return dir.Exists &&
-                       (dir.EnumerateDirectories().Count(x => x.Name == "examples" || x.Name == "bin") == 2 &&
-                        dir.EnumerateDirectories().Count(x => x.Name == "modules" || x.Name == "platforms") == 1)
-                       || // NuGet home
-                       (dir.EnumerateDirectories().Any(x => x.Name == "libs") &&
-                        (dir.EnumerateFiles("Apache.Ignite.Core.dll").Any() ||
-                         dir.EnumerateFiles("Apache.Ignite.*.nupkg").Any() ||
-                         dir.EnumerateFiles("apache.ignite.*.nupkg").Any() ||  // Lowercase on Linux
-                         dir.EnumerateFiles("apache.ignite.nuspec").Any() ||  // Lowercase on Linux
-                         dir.EnumerateFiles("Apache.Ignite.nuspec").Any()));
+                if (!dir.Exists)
+                {
+                    return false;
+                }
+
+                // Binary release or NuGet home:
+                var libs = Path.Combine(dir.FullName, "libs");
+
+                if (Directory.Exists(libs) &&
+                    Directory.EnumerateFiles(libs, "ignite-core-*.jar", SearchOption.TopDirectoryOnly).Any())
+                {
+                    return true;
+                }
+
+                // Source release home:
+                var javaSrc = Path.Combine(dir.FullName,
+                    "modules", "core", "src", "main", "java", "org", "apache", "ignite");
+
+                return Directory.Exists(javaSrc);
             }
             catch (IOException)
             {

From bc6a08d94c26fd7ab12959aa27ca79fd818b65c7 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov 
Date: Fri, 19 Jan 2018 20:14:50 +0700
Subject: [PATCH 24/65] IGNITE-3143 Fixed execution of VisorCoordinatorNodeTask
 via VisorGatewayTask.

(cherry picked from commit 1ae1350)
---
 .../ignite/internal/visor/VisorMultiNodeTask.java      |  6 ++++++
 .../internal/visor/compute/VisorGatewayTask.java       | 10 +++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
index 89a5fcff03e66..29ad5195c59e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorMultiNodeTask.java
@@ -104,6 +104,12 @@ protected Map map0(List subgrid,
                 if (nodeIds.contains(node.id()))
                     map.put(job(taskArg), node);
 
+            if (map.isEmpty())
+                ignite.log().error("No mapped jobs: [task=" + getClass().getName() +
+                    ", topVer=" + ignite.cluster().topologyVersion() +
+                    ", jobNids=" + nodeIds +
+                    ", subGrid=" + U.toShortString(subgrid) + "]");
+
             return map;
         }
         finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index 4e415371f8608..ee0a54b51491b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.internal.util.lang.GridTuple3;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.visor.VisorCoordinatorNodeTask;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -416,7 +417,14 @@ else if (isBuildInObject(argCls))
 
             if (F.isEmpty(nidsArg) || "null".equals(nidsArg)) {
                 try {
-                    if (VisorOneNodeTask.class.isAssignableFrom(Class.forName(taskName)))
+                    Class taskCls = Class.forName(taskName);
+
+                    if (VisorCoordinatorNodeTask.class.isAssignableFrom(taskCls)) {
+                        ClusterNode crd = ignite.context().discovery().discoCache().oldestAliveServerNode();
+
+                        nids = Collections.singletonList(crd.id());
+                    }
+                    else if (VisorOneNodeTask.class.isAssignableFrom(taskCls))
                         nids = Collections.singletonList(ignite.localNode().id());
                     else {
                         Collection nodes = ignite.cluster().nodes();

From 022ffe3efa931e10ed01d57138bf262853582134 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Fri, 19 Jan 2018 16:56:52 +0300
Subject: [PATCH 25/65] master Using specific version of dependency plugin for
 compatibility tests

---
 .../ignite/compatibility/testframework/util/MavenUtils.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
index 14250c9ae0f92..7eb3131a3fa01 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
@@ -135,7 +135,7 @@ private static String defineMavenLocalRepositoryPath() throws Exception {
 
         assert endTagPos >= 0 : "Couldn't define path to Maven local repository";
 
-        return output.substring(output.lastIndexOf(">", endTagPos) + 1, endTagPos);
+        return output.substring(output.lastIndexOf('>', endTagPos) + 1, endTagPos);
     }
 
     /**
@@ -147,7 +147,7 @@ private static String defineMavenLocalRepositoryPath() throws Exception {
     private static void downloadArtifact(String artifact) throws Exception {
         X.println("Downloading artifact... Identifier: " + artifact);
 
-        exec(buildMvnCommand() + " dependency:get -Dartifact=" + artifact +
+        exec(buildMvnCommand() + " org.apache.maven.plugins:maven-dependency-plugin:3.0.2:get -Dartifact=" + artifact +
             (useGgRepo ? " -DremoteRepositories=" + GG_MVN_REPO : ""));
 
         X.println("Download is finished");

From 52efb3d0e4249629e3039067a7472172489a0c99 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Fri, 19 Jan 2018 19:23:13 +0300
Subject: [PATCH 26/65] master Fixed hanging test (missed notify)

---
 .../wal/SegmentedRingByteBufferTest.java      | 188 +++++++++---------
 1 file changed, 93 insertions(+), 95 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentedRingByteBufferTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentedRingByteBufferTest.java
index e74761ee13333..3a24738dd152c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentedRingByteBufferTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentedRingByteBufferTest.java
@@ -313,56 +313,57 @@ private void doTestNoOverflowMultiThreaded(SegmentedRingByteBuffer.BufferMode mo
 
         final Object mux = new Object();
 
-        IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
-            @Override public void run() {
-                try {
-                    barrier.await();
-                }
-                catch (InterruptedException | BrokenBarrierException e) {
-                    e.printStackTrace();
+        IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> {
+            try {
+                barrier.await();
+            }
+            catch (InterruptedException | BrokenBarrierException e) {
+                e.printStackTrace();
 
-                    fail();
-                }
+                fail();
+            }
+
+            while (!stop.get()) {
+                TestObject obj = new TestObject();
 
-                while (!stop.get()) {
-                    TestObject obj = new TestObject();
+                SegmentedRingByteBuffer.WriteSegment seg = buf.offer(obj.size());
+                ByteBuffer bbuf;
 
-                    SegmentedRingByteBuffer.WriteSegment seg = buf.offer(obj.size());
-                    ByteBuffer bbuf;
+                if (seg == null) {
+                    cnt.incrementAndGet();
 
-                    if (seg == null) {
-                        cnt.incrementAndGet();
+                    synchronized (mux) {
+                        try {
+                            if (stop.get())
+                                break;
 
-                        synchronized (mux) {
-                            try {
-                                mux.wait();
-                            }
-                            catch (InterruptedException e) {
-                                e.printStackTrace();
-                            }
+                            mux.wait();
+                        }
+                        catch (InterruptedException e) {
+                            e.printStackTrace();
                         }
+                    }
 
-                        cnt.decrementAndGet();
+                    cnt.decrementAndGet();
 
-                        continue;
-                    }
+                    continue;
+                }
 
-                    bbuf = seg.buffer();
+                bbuf = seg.buffer();
 
-                    assertEquals(obj.size(), bbuf.remaining());
+                assertEquals(obj.size(), bbuf.remaining());
 
-                    bbuf.putLong(obj.id);
-                    bbuf.putInt(obj.len);
-                    bbuf.put(obj.arr);
+                bbuf.putLong(obj.id);
+                bbuf.putInt(obj.len);
+                bbuf.put(obj.arr);
 
-                    assertEquals(0, bbuf.remaining());
+                assertEquals(0, bbuf.remaining());
 
-                    seg.release();
+                seg.release();
 
-                    long total = totalWritten.addAndGet(obj.size());
+                long total = totalWritten.addAndGet(obj.size());
 
-                    assertTrue(total <= cap);
-                }
+                assertTrue(total <= cap);
             }
         }, producerCnt, "producer-thread");
 
@@ -387,9 +388,9 @@ private void doTestNoOverflowMultiThreaded(SegmentedRingByteBuffer.BufferMode mo
             }
         }
 
-        stop.set(true);
-
         synchronized (mux) {
+            stop.set(true);
+
             mux.notifyAll();
         }
 
@@ -410,45 +411,43 @@ private void doTestMultiThreaded(SegmentedRingByteBuffer.BufferMode mode) throws
 
         final CyclicBarrier barrier = new CyclicBarrier(producerCnt);
 
-        IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
-            @Override public void run() {
-                try {
-                    barrier.await();
-                }
-                catch (InterruptedException | BrokenBarrierException e) {
-                    e.printStackTrace();
+        IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> {
+            try {
+                barrier.await();
+            }
+            catch (InterruptedException | BrokenBarrierException e) {
+                e.printStackTrace();
 
-                    fail();
-                }
+                fail();
+            }
 
-                while (!stop.get()) {
-                    TestObject obj = new TestObject();
+            while (!stop.get()) {
+                TestObject obj = new TestObject();
 
-                    SegmentedRingByteBuffer.WriteSegment seg;
-                    ByteBuffer bbuf;
+                SegmentedRingByteBuffer.WriteSegment seg;
+                ByteBuffer bbuf;
 
-                    for (;;) {
-                        if (stop.get())
-                            return;
+                for (;;) {
+                    if (stop.get())
+                        return;
 
-                        seg = buf.offer(obj.size());
+                    seg = buf.offer(obj.size());
 
-                        if (seg != null)
-                            break;
-                    }
+                    if (seg != null)
+                        break;
+                }
 
-                    bbuf = seg.buffer();
+                bbuf = seg.buffer();
 
-                    assertEquals(obj.size(), bbuf.remaining());
+                assertEquals(obj.size(), bbuf.remaining());
 
-                    bbuf.putLong(obj.id);
-                    bbuf.putInt(obj.len);
-                    bbuf.put(obj.arr);
+                bbuf.putLong(obj.id);
+                bbuf.putInt(obj.len);
+                bbuf.put(obj.arr);
 
-                    assertEquals(0, bbuf.remaining());
+                assertEquals(0, bbuf.remaining());
 
-                    seg.release();
-                }
+                seg.release();
             }
         }, producerCnt, "producer-thread");
 
@@ -496,47 +495,45 @@ private void doTestMultiThreaded2(SegmentedRingByteBuffer.BufferMode mode) throw
 
         final Set items = Collections.newSetFromMap(new ConcurrentHashMap());
 
-        IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
-            @Override public void run() {
-                try {
-                    barrier.await();
-                }
-                catch (InterruptedException | BrokenBarrierException e) {
-                    e.printStackTrace();
+        IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(() -> {
+            try {
+                barrier.await();
+            }
+            catch (InterruptedException | BrokenBarrierException e) {
+                e.printStackTrace();
 
-                    fail();
-                }
+                fail();
+            }
 
-                while (!stop.get()) {
-                    TestObject obj = new TestObject();
+            while (!stop.get()) {
+                TestObject obj = new TestObject();
 
-                    SegmentedRingByteBuffer.WriteSegment seg;
-                    ByteBuffer bbuf;
+                SegmentedRingByteBuffer.WriteSegment seg;
+                ByteBuffer bbuf;
 
-                    for (;;) {
-                        if (stop.get())
-                            return;
+                for (;;) {
+                    if (stop.get())
+                        return;
 
-                        seg = buf.offer(obj.size());
+                    seg = buf.offer(obj.size());
 
-                        if (seg != null)
-                            break;
-                    }
+                    if (seg != null)
+                        break;
+                }
 
-                    bbuf = seg.buffer();
+                bbuf = seg.buffer();
 
-                    assertEquals(obj.size(), bbuf.remaining());
+                assertEquals(obj.size(), bbuf.remaining());
 
-                    bbuf.putLong(obj.id);
-                    bbuf.putInt(obj.len);
-                    bbuf.put(obj.arr);
+                bbuf.putLong(obj.id);
+                bbuf.putInt(obj.len);
+                bbuf.put(obj.arr);
 
-                    assertEquals(0, bbuf.remaining());
+                assertEquals(0, bbuf.remaining());
 
-                    assertTrue("Ooops! The same value is already exist in Set! ", items.add(obj));
+                assertTrue("Ooops! The same value is already exist in Set! ", items.add(obj));
 
-                    seg.release();
-                }
+                seg.release();
             }
         }, producerCnt, "producer-thread");
 
@@ -699,8 +696,9 @@ public TestObject() {
          */
         public TestObject(long id, byte[] arr) {
             this.id = id;
-            this.len = arr.length;
             this.arr = arr;
+
+            len = arr.length;
         }
 
         /**

From ce2a30efa4216ab2efc28384a7bc72633f7a4e09 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn 
Date: Mon, 22 Jan 2018 16:04:18 +0300
Subject: [PATCH 27/65] IGNITE-7477 .NET: Java 9 support

This closes #3415
---
 .../Cache/DataStorageMetricsTest.cs           |  2 +-
 .../ExecutableTest.cs                         | 19 +++-
 .../Impl/Unmanaged/Jni/Jvm.cs                 | 93 ++++++++++++++++---
 .../Impl/Unmanaged/Jni/JvmDll.cs              | 33 ++++++-
 4 files changed, 128 insertions(+), 19 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs
index d98254d720d28..caac58a04604d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs
@@ -79,7 +79,7 @@ public void TestDataStorageMetrics()
                 Assert.AreEqual(0, metrics.WalArchiveSegments);
                 Assert.AreEqual(0, metrics.WalFsyncTimeAverage);
 
-                Assert.Greater(metrics.LastCheckpointTotalPagesNumber, 26);
+                Assert.GreaterOrEqual(metrics.LastCheckpointTotalPagesNumber, 26);
                 Assert.AreEqual(0, metrics.LastCheckpointDataPagesNumber);
                 Assert.AreEqual(0, metrics.LastCheckpointCopiedOnWritePagesNumber);
                 Assert.AreEqual(TimeSpan.Zero, metrics.LastCheckpointLockWaitDuration);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExecutableTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExecutableTest.cs
index eb1e68ed5710f..faf2119be7125 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExecutableTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExecutableTest.cs
@@ -321,6 +321,17 @@ public void TestXmlConfigurationCmd()
         [Test]
         public void TestInvalidCmdArgs()
         {
+            var ignoredWarns = new[]
+            {
+                "WARNING: An illegal reflective access operation has occurred",
+                "WARNING: Illegal reflective access by org.apache.ignite.internal.util.GridUnsafe$2 " +
+                "(file:/C:/w/incubator-ignite/modules/core/target/classes/) to field java.nio.Buffer.address",
+                "WARNING: Please consider reporting this to the maintainers of org.apache.ignite.internal.util." +
+                "GridUnsafe$2",
+                "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations",
+                "WARNING: All illegal access operations will be denied in a future release"
+            };
+
             Action checkError = (args, err) =>
             {
                 var reader = new ListDataReader();
@@ -330,7 +341,9 @@ public void TestInvalidCmdArgs()
                 Assert.IsTrue(proc.Join(30000, out exitCode));
                 Assert.AreEqual(-1, exitCode);
 
-                Assert.AreEqual(err, reader.GetOutput().FirstOrDefault(x => !string.IsNullOrWhiteSpace(x)));
+                Assert.AreEqual(err, reader.GetOutput()
+                    .Except(ignoredWarns)
+                    .FirstOrDefault(x => !string.IsNullOrWhiteSpace(x)));
             };
 
             checkError("blabla", "ERROR: Apache.Ignite.Core.Common.IgniteException: Missing argument value: " +
@@ -396,8 +409,8 @@ private static void GenerateDll(string outputPath)
         private static void AssertJvmMaxMemory(long expected, long actual)
         {
             // allow 20% tolerance because max memory in Java is not exactly equal to Xmx parameter value
-            Assert.LessOrEqual(actual, expected);
-            Assert.Greater(actual, expected/5*4);
+            Assert.LessOrEqual(actual, expected / 4 * 5);
+            Assert.Greater(actual, expected / 5 * 4);
         }
 
         /// 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
index 3d119cd1f48bb..6d706e8406fa3 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/Jvm.cs
@@ -21,6 +21,7 @@ namespace Apache.Ignite.Core.Impl.Unmanaged.Jni
     using System.Collections.Generic;
     using System.Diagnostics;
     using System.Diagnostics.CodeAnalysis;
+    using System.Linq;
     using System.Reflection;
     using System.Runtime.InteropServices;
     using System.Security;
@@ -35,7 +36,21 @@ internal sealed unsafe class Jvm
     {
         /** */
         // ReSharper disable once InconsistentNaming
-        private const int JNI_VERSION_1_6 = 0x00010006;
+        private const int JNI_VERSION_1_8 = 0x00010008;
+
+        /** */
+        // ReSharper disable once InconsistentNaming
+        private const int JNI_VERSION_9 = 0x00090000;
+
+        /** Options to enable startup on Java 9. */
+        private static readonly string[] Java9Options =
+        {
+            "--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED",
+            "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED",
+            "--add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED",
+            "--add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED",
+            "--illegal-access=permit"
+        };
 
         /** */
         private readonly IntPtr _jvmPtr;
@@ -220,35 +235,85 @@ private static IntPtr GetJvmPtr(IList options)
                 return jvm;
             }
 
+            return CreateJvm(options);
+        }
+
+        /// 
+        /// Determines whether we are on Java 9.
+        /// 
+        private static bool IsJava9()
+        {
+            var args = new JvmInitArgs
+            {
+                version = JNI_VERSION_9
+            };
+
+            // Returns error on Java 8 and lower.
+            var res = JvmDll.Instance.GetDefaultJvmInitArgs(&args);
+            return res == JniResult.Success;
+        }
+
+        /// 
+        /// Creates the JVM.
+        /// 
+        private static IntPtr CreateJvm(IList options)
+        {
+            if (IsJava9())
+            {
+                options = options == null
+                    ? Java9Options.ToList()
+                    : new List(options.Concat(Java9Options));
+            }
+
             var args = new JvmInitArgs
             {
-                version = JNI_VERSION_1_6
+                version = JNI_VERSION_1_8,
+                nOptions = options.Count
             };
 
-            if (options != null && options.Count > 0)
+            var opts = GetJvmOptions(options);
+
+            try
             {
-                args.nOptions = options.Count;
-                var opt = new JvmOption[options.Count];
+                JniResult res;
+                IntPtr jvm;
 
-                for (int i = 0; i < options.Count; i++)
+                fixed (JvmOption* optPtr = &opts[0])
+                {
+                    args.options = optPtr;
+                    IntPtr env;
+                    res = JvmDll.Instance.CreateJvm(out jvm, out env, &args);
+                }
+
+                if (res != JniResult.Success)
                 {
-                    opt[i].optionString = Marshal.StringToHGlobalAnsi(options[i]);
+                    throw new IgniteException("JNI_CreateJavaVM failed: " + res);
                 }
 
-                fixed (JvmOption* a = &opt[0])
+                return jvm;
+            }
+            finally
+            {
+                foreach (var opt in opts)
                 {
-                    args.options = a;
+                    Marshal.FreeHGlobal(opt.optionString);
                 }
             }
+        }
 
-            IntPtr env;
-            res = JvmDll.Instance.CreateJvm(out jvm, out env, &args);
-            if (res != JniResult.Success)
+        /// 
+        /// Gets the JVM options.
+        /// 
+        private static JvmOption[] GetJvmOptions(IList options)
+        {
+            var opt = new JvmOption[options.Count];
+
+            for (var i = 0; i < options.Count; i++)
             {
-                throw new IgniteException("JNI_CreateJavaVM failed: " + res);
+                opt[i].optionString = Marshal.StringToHGlobalAnsi(options[i]);
             }
 
-            return jvm;
+            return opt;
         }
 
         /// 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/JvmDll.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/JvmDll.cs
index 3e1d3a93dd88f..ef161f4604784 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/JvmDll.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Unmanaged/Jni/JvmDll.cs
@@ -84,6 +84,9 @@ internal class JvmDll
         /** */
         private unsafe delegate JniResult CreateJvmDel(out IntPtr pvm, out IntPtr penv, JvmInitArgs* args);
 
+        /** */
+        private unsafe delegate JniResult GetDefaultArgsDel(JvmInitArgs* args);
+
         /** */
         private delegate JniResult GetCreatedJvmsDel(out IntPtr pvm, int size, out int size2);
 
@@ -93,6 +96,9 @@ internal class JvmDll
         /** */
         private readonly GetCreatedJvmsDel _getCreatedJvms;
 
+        /** */
+        private readonly GetDefaultArgsDel _getDefaultArgs;
+
         /// 
         /// Initializes a new instance of the  class.
         /// 
@@ -121,16 +127,22 @@ private unsafe JvmDll(IntPtr ptr)
                 var getJvmsPtr = DllLoader.NativeMethodsMacOs.dlsym(ptr, "JNI_GetCreatedJavaVMs");
                 _getCreatedJvms = (GetCreatedJvmsDel) Marshal.GetDelegateForFunctionPointer(getJvmsPtr,
                     typeof(GetCreatedJvmsDel));
+
+                var getArgsPtr = DllLoader.NativeMethodsMacOs.dlsym(ptr, "JNI_GetDefaultJavaVMInitArgs");
+                _getDefaultArgs = (GetDefaultArgsDel) Marshal.GetDelegateForFunctionPointer(getArgsPtr,
+                    typeof(GetDefaultArgsDel));
             }
             else if (Os.IsWindows)
             {
                 _createJvm = JniNativeMethodsWindows.JNI_CreateJavaVM;
                 _getCreatedJvms = JniNativeMethodsWindows.JNI_GetCreatedJavaVMs;
+                _getDefaultArgs = JniNativeMethodsWindows.JNI_GetDefaultJavaVMInitArgs;
             }
             else
             {
                 _createJvm = JniNativeMethodsLinux.JNI_CreateJavaVM;
                 _getCreatedJvms = JniNativeMethodsLinux.JNI_GetCreatedJavaVMs;
+                _getDefaultArgs = JniNativeMethodsLinux.JNI_GetDefaultJavaVMInitArgs;
             }
         }
 
@@ -158,6 +170,17 @@ public JniResult GetCreatedJvms(out IntPtr pvm, int size, out int size2)
             return _getCreatedJvms(out pvm, size, out size2);
         }
 
+        /// 
+        /// Gets the default JVM init args.
+        /// Before calling this function, native code must set the vm_args->version field to the JNI version 
+        /// it expects the VM to support. After this function returns, vm_args->version will be set 
+        /// to the actual JNI version the VM supports.
+        /// 
+        public unsafe JniResult GetDefaultJvmInitArgs(JvmInitArgs* args)
+        {
+            return _getDefaultArgs(args);
+        }
+
         /// 
         /// Loads the JVM DLL into process memory.
         /// 
@@ -398,6 +421,10 @@ private static unsafe class JniNativeMethodsWindows
             [DllImport("jvm.dll", CallingConvention = CallingConvention.StdCall)]
             internal static extern JniResult JNI_GetCreatedJavaVMs(out IntPtr pvm, int size,
                 [Out] out int size2);
+
+            [SuppressMessage("Microsoft.Design", "CA1060:MovePInvokesToNativeMethodsClass")]
+            [DllImport("jvm.dll", CallingConvention = CallingConvention.StdCall)]
+            internal static extern JniResult JNI_GetDefaultJavaVMInitArgs(JvmInitArgs* args);
         }
 
         /// 
@@ -413,6 +440,10 @@ private static unsafe class JniNativeMethodsLinux
             [DllImport("libjvm.so", CallingConvention = CallingConvention.StdCall)]
             internal static extern JniResult JNI_GetCreatedJavaVMs(out IntPtr pvm, int size,
                 [Out] out int size2);
+
+            [SuppressMessage("Microsoft.Design", "CA1060:MovePInvokesToNativeMethodsClass")]
+            [DllImport("libjvm.so", CallingConvention = CallingConvention.StdCall)]
+            internal static extern JniResult JNI_GetDefaultJavaVMInitArgs(JvmInitArgs* args);
         }
-   }
+    }
 }
\ No newline at end of file

From 167ef1022d7f92843c06695c97020d32959df19e Mon Sep 17 00:00:00 2001
From: Anton Vinogradov 
Date: Mon, 22 Jan 2018 18:55:22 +0300
Subject: [PATCH 28/65] IGNITE-6711 DataRegionMetrics#totalAllocatedPages is
 not valid after node restart

Signed-off-by: Anton Vinogradov 

(cherry picked from commit 326c19b)
Signed-off-by: Anton Vinogradov 
---
 .../pagemem/store/IgnitePageStoreManager.java |   8 +
 .../persistence/AllocatedPageTracker.java     |  11 +
 .../persistence/DataRegionMetricsImpl.java    |   9 +-
 .../cache/persistence/file/FilePageStore.java |  64 +++--
 .../file/FilePageStoreFactory.java            |   3 +-
 .../file/FilePageStoreManager.java            |  52 +++-
 .../persistence/file/FilePageStoreV2.java     |  11 +-
 .../file/FileVersionCheckingFactory.java      |  23 +-
 .../persistence/pagemem/PageMemoryImpl.java   |   2 -
 .../db/IgnitePdsDataRegionMetricsTest.java    | 232 ++++++++++++++++++
 .../pagemem/NoOpPageStoreManager.java         |   5 +
 .../junits/common/GridCommonAbstractTest.java |   9 +
 .../ignite/testsuites/IgnitePdsTestSuite.java |   2 +
 13 files changed, 393 insertions(+), 38 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AllocatedPageTracker.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 6802a3fc26a0f..deb21a77f9554 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -198,4 +198,12 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac
      * @return {@code True} if index store for given cache group existed before node started.
      */
     public boolean hasIndexStore(int grpId);
+
+    /**
+     * Calculates number of pages currently allocated for given cache group.
+     *
+     * @param grpId cache group id.
+     * @return number of pages.
+     */
+    public long pagesAllocated(int grpId);
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AllocatedPageTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AllocatedPageTracker.java
new file mode 100644
index 0000000000000..bd2a13ef63b06
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AllocatedPageTracker.java
@@ -0,0 +1,11 @@
+package org.apache.ignite.internal.processors.cache.persistence;
+
+/**
+ * Tracks allocated pages.
+ */
+public interface AllocatedPageTracker {
+    /**
+     * Increments totalAllocatedPages counter.
+     */
+    public void updateTotalAllocatedPages(long delta);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
index 06fe9fe980398..46988b7d24151 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
@@ -28,7 +28,7 @@
 /**
  *
  */
-public class DataRegionMetricsImpl implements DataRegionMetrics {
+public class DataRegionMetricsImpl implements DataRegionMetrics, AllocatedPageTracker {
     /** */
     private final IgniteOutClosure fillFactorProvider;
 
@@ -210,8 +210,13 @@ public void resetDirtyPages() {
      * Increments totalAllocatedPages counter.
      */
     public void incrementTotalAllocatedPages() {
+        updateTotalAllocatedPages(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void updateTotalAllocatedPages(long delta) {
         if (metricsEnabled) {
-            totalAllocatedPages.increment();
+            totalAllocatedPages.add(delta);
 
             updateAllocationRateMetrics();
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 0ed9167bb8c4e..ddf6062c864f4 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
@@ -71,6 +72,9 @@ public class FilePageStore implements PageStore {
     /** */
     private final AtomicLong allocated;
 
+    /** Region metrics updater. */
+    private final AllocatedPageTracker allocatedTracker;
+
     /** */
     private final int pageSize;
 
@@ -92,16 +96,19 @@ public class FilePageStore implements PageStore {
     /**
      * @param file File.
      */
-    public FilePageStore(byte type, File file, FileIOFactory factory, DataStorageConfiguration cfg) {
+    public FilePageStore(
+        byte type,
+        File file,
+        FileIOFactory factory,
+        DataStorageConfiguration cfg,
+        AllocatedPageTracker allocatedTracker) {
         this.type = type;
-
-        cfgFile = file;
-        dbCfg = cfg;
-        ioFactory = factory;
-
-        allocated = new AtomicLong();
-
-        pageSize = dbCfg.getPageSize();
+        this.cfgFile = file;
+        this.dbCfg = cfg;
+        this.ioFactory = factory;
+        this.allocated = new AtomicLong();
+        this.pageSize = dbCfg.getPageSize();
+        this.allocatedTracker = allocatedTracker;
     }
 
     /** {@inheritDoc} */
@@ -261,7 +268,13 @@ public void truncate(int tag) throws IgniteCheckedException {
 
             fileIO.clear();
 
-            allocated.set(initFile());
+            long newAlloc = initFile();
+
+            long delta = newAlloc - allocated.getAndSet(newAlloc);
+
+            assert delta % pageSize == 0;
+
+            allocatedTracker.updateTotalAllocatedPages(delta / pageSize);
         }
         catch (IOException e) {
             throw new IgniteCheckedException(e);
@@ -294,8 +307,15 @@ public void finishRecover() {
         try {
             // Since we always have a meta-page in the store, never revert allocated counter to a value smaller than
             // header + page.
-            if (inited)
-                allocated.set(Math.max(headerSize() + pageSize, fileIO.size()));
+            if (inited) {
+                long newSize = Math.max(headerSize() + pageSize, fileIO.size());
+
+                long delta = newSize - allocated.getAndSet(newSize);
+
+                assert delta % pageSize == 0;
+
+                allocatedTracker.updateTotalAllocatedPages(delta / pageSize);
+            }
 
             recover = false;
         }
@@ -407,10 +427,17 @@ private void init() throws IgniteCheckedException {
                     try {
                         this.fileIO = fileIO = ioFactory.create(cfgFile, CREATE, READ, WRITE);
 
-                        if (cfgFile.length() == 0)
-                            allocated.set(initFile());
-                        else
-                            allocated.set(checkFile());
+                        long newSize = cfgFile.length() == 0 ? initFile() : checkFile();
+
+                        assert allocated.get() == 0;
+
+                        long delta = newSize - headerSize();
+
+                        assert delta % pageSize == 0;
+
+                        allocatedTracker.updateTotalAllocatedPages(delta / pageSize);
+
+                        allocated.set(newSize);
 
                         inited = true;
                     }
@@ -550,8 +577,11 @@ private long allocPage() {
         do {
             off = allocated.get();
 
-            if (allocated.compareAndSet(off, off + pageSize))
+            if (allocated.compareAndSet(off, off + pageSize)) {
+                allocatedTracker.updateTotalAllocatedPages(1);
+
                 break;
+            }
         }
         while (true);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
index d97ab26397a03..fe93d0743be07 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
@@ -20,6 +20,7 @@
 import java.io.File;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
 
 /**
  *
@@ -31,5 +32,5 @@ public interface FilePageStoreFactory {
      * @param type Data type, can be {@link PageIdAllocator#FLAG_IDX} or {@link PageIdAllocator#FLAG_DATA}.
      * @param file File Page store file.
      */
-    public FilePageStore createPageStore(byte type, File file) throws IgniteCheckedException;
+    public FilePageStore createPageStore(byte type, File file, AllocatedPageTracker allocatedTracker) throws IgniteCheckedException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index deacf79004aa3..c9b7278d88fac 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -48,8 +48,9 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
-import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
+import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
@@ -217,7 +218,12 @@ public FilePageStoreManager(GridKernalContext ctx) {
         int grpId = MetaStorage.METASTORAGE_CACHE_ID;
 
         if (!idxCacheStores.containsKey(grpId)) {
-            CacheStoreHolder holder = initDir(new File(storeWorkDir, "metastorage"), grpId, 1);
+            CacheStoreHolder holder = initDir(
+                new File(storeWorkDir,
+                    "metastorage"),
+                grpId,
+                1,
+                delta -> {/* No-op */} );
 
             CacheStoreHolder old = idxCacheStores.put(grpId, holder);
 
@@ -369,17 +375,27 @@ private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfigu
 
         File cacheWorkDir = cacheWorkDir(ccfg);
 
-        return initDir(cacheWorkDir, grpDesc.groupId(), grpDesc.config().getAffinity().partitions());
+        AllocatedPageTracker allocatedTracker =
+            cctx.database().dataRegion(grpDesc.config().getDataRegionName()).memoryMetrics();
+
+        return initDir(cacheWorkDir,
+            grpDesc.groupId(),
+            grpDesc.config().getAffinity().partitions(),
+            allocatedTracker);
     }
 
     /**
      * @param cacheWorkDir Work directory.
      * @param grpId Group ID.
      * @param partitions Number of partitions.
+     * @param allocatedTracker Metrics updater
      * @return Cache store holder.
      * @throws IgniteCheckedException If failed.
      */
-    private CacheStoreHolder initDir(File cacheWorkDir, int grpId, int partitions) throws IgniteCheckedException {
+    private CacheStoreHolder initDir(File cacheWorkDir,
+        int grpId,
+        int partitions,
+        AllocatedPageTracker allocatedTracker) throws IgniteCheckedException {
         boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir);
 
         File idxFile = new File(cacheWorkDir, INDEX_FILE_NAME);
@@ -390,13 +406,20 @@ private CacheStoreHolder initDir(File cacheWorkDir, int grpId, int partitions) t
         FilePageStoreFactory pageStoreFactory = new FileVersionCheckingFactory(
             pageStoreFileIoFactory, pageStoreV1FileIoFactory, igniteCfg.getDataStorageConfiguration());
 
-        FilePageStore idxStore = pageStoreFactory.createPageStore(PageMemory.FLAG_IDX, idxFile);
+        FilePageStore idxStore =
+            pageStoreFactory.createPageStore(
+                PageMemory.FLAG_IDX,
+                idxFile,
+                allocatedTracker);
 
         FilePageStore[] partStores = new FilePageStore[partitions];
 
         for (int partId = 0; partId < partStores.length; partId++) {
-            FilePageStore partStore = pageStoreFactory.createPageStore(
-                PageMemory.FLAG_DATA, getPartitionFile(cacheWorkDir, partId));
+            FilePageStore partStore =
+                pageStoreFactory.createPageStore(
+                    PageMemory.FLAG_DATA,
+                    getPartitionFile(cacheWorkDir, partId),
+                    allocatedTracker);
 
             partStores[partId] = partStore;
         }
@@ -588,6 +611,21 @@ private StoredCacheData readCacheData(File conf) throws IgniteCheckedException {
         return !grpsWithoutIdx.contains(grpId);
     }
 
+    /** {@inheritDoc} */
+    @Override public long pagesAllocated(int grpId) {
+        CacheStoreHolder holder = idxCacheStores.get(grpId);
+
+        if (holder == null)
+            return 0;
+
+        long pageCnt = holder.idxStore.pages();
+
+        for (int i = 0; i < holder.partStores.length; i++)
+            pageCnt += holder.partStores[i].pages();
+
+        return pageCnt;
+    }
+
     /**
      * @return Store work dir. Includes consistent-id based folder
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
index c2e2d36566b71..d8c800d39b9a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
@@ -18,6 +18,7 @@
 
 import java.io.File;
 import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
 
 /**
  *
@@ -34,9 +35,15 @@ public class FilePageStoreV2 extends FilePageStore {
      * @param file File.
      * @param factory Factory.
      * @param cfg Config.
+     * @param allocatedTracker Metrics updater
      */
-    public FilePageStoreV2(byte type, File file, FileIOFactory factory, DataStorageConfiguration cfg) {
-        super(type, file, factory, cfg);
+    public FilePageStoreV2(
+        byte type,
+        File file,
+        FileIOFactory factory,
+        DataStorageConfiguration cfg,
+        AllocatedPageTracker allocatedTracker) {
+        super(type, file, factory, cfg, allocatedTracker);
 
         hdrSize = cfg.getPageSize();
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
index bb25ab0b844ab..c7aaf1bb401fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
@@ -23,6 +23,7 @@
 import java.nio.ByteOrder;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
 
 /**
  * Checks version in files if it's present on the disk, creates store with latest version otherwise.
@@ -67,15 +68,18 @@ public FileVersionCheckingFactory(FileIOFactory fileIOFactory, DataStorageConfig
     }
 
     /** {@inheritDoc} */
-    @Override public FilePageStore createPageStore(byte type, File file) throws IgniteCheckedException {
+    @Override public FilePageStore createPageStore(
+        byte type,
+        File file,
+        AllocatedPageTracker allocatedTracker) throws IgniteCheckedException {
         if (!file.exists())
-            return createPageStore(type, file, latestVersion());
+            return createPageStore(type, file, latestVersion(), allocatedTracker);
 
         try (FileIO fileIO = fileIOFactoryStoreV1.create(file)) {
             int minHdr = FilePageStore.HEADER_SIZE;
 
             if (fileIO.size() < minHdr)
-                return createPageStore(type, file, latestVersion());
+                return createPageStore(type, file, latestVersion(), allocatedTracker);
 
             ByteBuffer hdr = ByteBuffer.allocate(minHdr).order(ByteOrder.LITTLE_ENDIAN);
 
@@ -88,7 +92,7 @@ public FileVersionCheckingFactory(FileIOFactory fileIOFactory, DataStorageConfig
 
             int ver = hdr.getInt();
 
-            return createPageStore(type, file, ver);
+            return createPageStore(type, file, ver, allocatedTracker);
         }
         catch (IOException e) {
             throw new IgniteCheckedException("Error while creating file page store [file=" + file + "]:", e);
@@ -116,14 +120,19 @@ public int latestVersion() {
      * @param type Type.
      * @param file File.
      * @param ver Version.
+     * @param allocatedTracker Metrics updater
      */
-    public FilePageStore createPageStore(byte type, File file, int ver) {
+    public FilePageStore createPageStore(
+        byte type,
+        File file,
+        int ver,
+        AllocatedPageTracker allocatedTracker) {
         switch (ver) {
             case FilePageStore.VERSION:
-                return new FilePageStore(type, file, fileIOFactoryStoreV1, memCfg);
+                return new FilePageStore(type, file, fileIOFactoryStoreV1, memCfg, allocatedTracker);
 
             case FilePageStoreV2.VERSION:
-                return new FilePageStoreV2(type, file, fileIOFactory, memCfg);
+                return new FilePageStoreV2(type, file, fileIOFactory, memCfg, allocatedTracker);
 
             default:
                 throw new IllegalArgumentException("Unknown version of file page store: " + ver + " for file [" + file.getAbsolutePath() + "]");
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 90fde31155f5d..2c24cea232cc3 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -430,8 +430,6 @@ private void initWriteThrottle() {
 
         long pageId = storeMgr.allocatePage(cacheId, partId, flags);
 
-        memMetrics.incrementTotalAllocatedPages();
-
         assert PageIdUtils.pageIndex(pageId) > 0; //it's crucial for tracking pages (zero page is super one)
 
         // We need to allocate page in memory for marking it dirty to save it in the next checkpoint.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
new file mode 100644
index 0000000000000..af65d1d614837
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+import org.apache.ignite.DataRegionMetrics;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
+
+/**
+ *
+ */
+public class IgnitePdsDataRegionMetricsTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final long INIT_REGION_SIZE = 10 << 20;
+
+    /** */
+    private static final int ITERATIONS = 3;
+
+    /** */
+    private static final int BATCHES = 5;
+
+    /** */
+    private static final int BATCH_SIZE_LOW = 100;
+
+    /** */
+    private static final int BATCH_SIZE_HIGH = 1000;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setInitialSize(INIT_REGION_SIZE)
+                    .setPersistenceEnabled(true)
+                    .setMetricsEnabled(true));
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        CacheConfiguration ccfg = new CacheConfiguration<>()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setBackups(1);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        GridTestUtils.deleteDbFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        GridTestUtils.deleteDbFiles();
+
+        super.afterTest();
+    }
+
+    /** */
+    public void testMemoryUsageSingleNode() throws Exception {
+        DataRegionMetrics initMetrics = null;
+
+        for (int iter = 0; iter < ITERATIONS; iter++) {
+            final IgniteEx node = startGrid(0);
+
+            node.cluster().active(true);
+
+            DataRegionMetrics currMetrics = getDfltRegionMetrics(node);
+
+            if (initMetrics == null)
+                initMetrics = currMetrics;
+
+            assertTrue(currMetrics.getTotalAllocatedPages() >= currMetrics.getPhysicalMemoryPages());
+
+            final IgniteCache cache = node.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+            final Set grpIds = node.context().cache().cacheGroups()
+                .stream().map(CacheGroupContext::groupId).collect(Collectors.toSet());
+
+            Map map = new HashMap<>();
+
+            for (int batch = 0; batch < BATCHES; batch++) {
+                int nPuts = BATCH_SIZE_LOW + ThreadLocalRandom.current().nextInt(BATCH_SIZE_HIGH - BATCH_SIZE_LOW);
+
+                for (int i = 0; i < nPuts; i++)
+                    map.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
+
+                cache.putAll(map);
+
+                checkMetricsConsistency(node, grpIds);
+            }
+
+            currMetrics = getDfltRegionMetrics(node);
+
+            // Make sure metrics are rising
+            assertTrue(currMetrics.getPhysicalMemoryPages() > initMetrics.getPhysicalMemoryPages());
+            assertTrue(currMetrics.getTotalAllocatedPages() > initMetrics.getTotalAllocatedPages());
+
+            stopGrid(0, true);
+        }
+    }
+
+    /** */
+    public void testMemoryUsageMultipleNodes() throws Exception {
+        IgniteEx node0 = startGrid(0);
+        IgniteEx node1 = startGrid(1);
+
+        node0.cluster().active(true);
+
+        final IgniteCache cache = node0.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        final Set grpIds = node0.context().cache().cacheGroups()
+            .stream().map(CacheGroupContext::groupId).collect(Collectors.toSet());
+
+        Map map = new HashMap<>();
+
+        for (int i = 0; i < 10_000; i++)
+            map.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
+
+        cache.putAll(map);
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkMetricsConsistency(node0, grpIds);
+        checkMetricsConsistency(node1, grpIds);
+
+        IgniteEx node2 = startGrid(2);
+
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkMetricsConsistency(node0, grpIds);
+        checkMetricsConsistency(node1, grpIds);
+        checkMetricsConsistency(node2, grpIds);
+
+        stopGrid(1, true);
+
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkMetricsConsistency(node0, grpIds);
+        checkMetricsConsistency(node2, grpIds);
+    }
+
+    /** */
+    private static DataRegionMetrics getDfltRegionMetrics(Ignite node) {
+        for (DataRegionMetrics m : node.dataRegionMetrics())
+            if (DFLT_DATA_REG_DEFAULT_NAME.equals(m.getName()))
+                return m;
+
+        throw new RuntimeException("No metrics found for default data region");
+    }
+
+    /** */
+    private static void checkMetricsConsistency(
+        final IgniteEx node,
+        final Set grpIds) throws Exception {
+        boolean storageMatches = GridTestUtils.waitForCondition((PA)() -> {
+            long pagesInStore = 0;
+            long allocated = 0;
+
+            for (int grpId : grpIds) {
+                DataRegion region = node.context().cache().cacheGroup(grpId).dataRegion();
+
+                if (!region.config().isMetricsEnabled())
+                    continue;
+
+                pagesInStore += node.context().cache().context().pageStore().pagesAllocated(grpId);
+                allocated += region.memoryMetrics().getTotalAllocatedPages();
+            }
+
+            assert 0 != pagesInStore;
+            assert 0 != allocated;
+
+            return allocated == pagesInStore;
+        }, 1000);
+
+        assertTrue(storageMatches);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index 8f72ed61f439d..39a794dc4b846 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -195,4 +195,9 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
     @Override public void onDeActivate(GridKernalContext kctx) {
         // No-op.
     }
+
+    /** {@inheritDoc} */
+    @Override public long pagesAllocated(int grpId) {
+        return 0;
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 74a0de91502a3..e39d482f12866 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -1775,6 +1775,15 @@ protected final void checkOnePhaseCommitReturnValuesCleaned(final int nodesCnt)
         }
     }
 
+    /**
+     * Sets baseline topology.
+     */
+    public void resetBaselineTopology() {
+        Ignite node = G.allGrids().get(0);
+
+        node.cluster().setBaselineTopology(node.cluster().topologyVersion());
+    }
+
     /**
      * @param ignite Node.
      * @return Completed txs map.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 8e022795b6626..383698f99d9f5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsDataRegionMetricsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.DefaultPageSizeBackwardsCompatibilityTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimulationWithRealCpDisabledTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsEvictionTest;
@@ -98,6 +99,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgniteClusterActivateDeactivateTestWithPersistence.class);
 
         suite.addTestSuite(IgnitePdsCacheRestoreTest.class);
+        suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class);
 
         suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
     }

From f385ecc0546ed1fa1361afeb7ac22ced709f5627 Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov 
Date: Mon, 22 Jan 2018 19:16:51 +0300
Subject: [PATCH 29/65] IGNITE-6902 Implement new JMX metrics for Memory
 Regions

Signed-off-by: Anton Vinogradov 

(cherry picked from commit fae726a)
Signed-off-by: Anton Vinogradov 
---
 .../org/apache/ignite/DataRegionMetrics.java  | 38 ++++++++++++++++
 .../ignite/internal/pagemem/PageMemory.java   |  5 +++
 .../pagemem/impl/PageMemoryNoStoreImpl.java   |  5 +++
 .../persistence/DataRegionMetricsImpl.java    | 42 ++++++++++++++++-
 .../DataRegionMetricsMXBeanImpl.java          | 25 +++++++++++
 .../DataRegionMetricsSnapshot.java            | 45 +++++++++++++++++++
 .../IgniteCacheDatabaseSharedManager.java     |  6 ++-
 .../persistence/pagemem/PageMemoryImpl.java   |  6 +--
 .../cluster/PlatformClusterGroup.java         | 16 +++++--
 .../ApiParity/DataRegionMetricsParityTest.cs  | 10 ++---
 .../Cache/DataRegionMetricsTest.cs            |  6 +++
 .../Apache.Ignite.Core/IDataRegionMetrics.cs  | 45 +++++++++++++++++++
 .../Impl/DataRegionMetrics.cs                 | 36 +++++++++++++++
 13 files changed, 269 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
index 25c0db97d02c8..f5ae96b153377 100644
--- a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
@@ -63,6 +63,15 @@ public interface DataRegionMetrics {
      */
     public long getTotalAllocatedPages();
 
+    /**
+     * Gets a total size of memory allocated in the data region. When persistence is disabled, this
+     * metric shows the total size of pages in memory. When persistence is enabled, this metric shows the
+     * total size of pages in memory and on disk.
+     *
+     * @return Total size of memory allocated, in bytes.
+     */
+    public long getTotalAllocatedSize();
+
     /**
      * Gets pages allocation rate of a memory region.
      *
@@ -127,4 +136,33 @@ public interface DataRegionMetrics {
      * @return Total number of pages loaded to RAM.
      */
     public long getPhysicalMemoryPages();
+
+    /**
+     * Gets total size of pages loaded to the RAM. When persistence is disabled, this metric is equal
+     * to {@link #getTotalAllocatedSize()}.
+     *
+     * @return Total size of pages loaded to RAM in bytes.
+     */
+    public long getPhysicalMemorySize();
+
+    /**
+     * Gets checkpoint buffer size in pages.
+     *
+     * @return Checkpoint buffer size in pages.
+     */
+    public long getCheckpointBufferPages();
+
+    /**
+     * Gets checkpoint buffer size in bytes.
+     *
+     * @return Checkpoint buffer size in bytes.
+     */
+    public long getCheckpointBufferSize();
+
+    /**
+     * Gets memory page size.
+     *
+     * @return page size in bytes.
+     */
+    public int getPageSize();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
index c20e1a776a1d5..6f2e2c9ec36ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
@@ -43,4 +43,9 @@ public interface PageMemory extends LifecycleAware, PageIdAllocator, PageSupport
      * @return Total number of loaded pages in memory.
      */
     public long loadedPages();
+
+    /**
+     * Number of pages used in checkpoint buffer.
+     */
+    public int checkpointBufferPagesCount();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index 88237eee2cf5f..af1555e583fe4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -821,4 +821,9 @@ public int pageIndex(int seqNo) {
             return PageIdUtils.pageIndex(fromSegmentIndex(idx, seqNo - pagesInPrevSegments));
         }
     }
+
+    /** {@inheritDoc} */
+    public int checkpointBufferPagesCount() {
+        return 0;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
index 46988b7d24151..3760cc4ded5ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
@@ -98,7 +98,17 @@ public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg, @Nullable Ignite
 
     /** {@inheritDoc} */
     @Override public long getTotalAllocatedPages() {
-        return metricsEnabled ? totalAllocatedPages.longValue() : 0;
+        if (!metricsEnabled)
+            return 0;
+
+        return totalAllocatedPages.longValue();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTotalAllocatedSize() {
+        assert pageMem != null;
+
+        return getTotalAllocatedPages() * pageMem.pageSize();
     }
 
     /** {@inheritDoc} */
@@ -171,6 +181,36 @@ public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg, @Nullable Ignite
         return pageMem.loadedPages();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getPhysicalMemorySize() {
+        return getPhysicalMemoryPages() * pageMem.pageSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointBufferPages() {
+        if (!metricsEnabled)
+            return 0;
+
+        assert pageMem != null;
+
+        return pageMem.checkpointBufferPagesCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointBufferSize() {
+        return getCheckpointBufferPages() * pageMem.pageSize();
+    }
+
+    /** {@inheritDoc} */
+    public int getPageSize() {
+        if (!metricsEnabled)
+            return 0;
+
+        assert pageMem != null;
+
+        return pageMem.pageSize();
+    }
+
     /**
      * Updates pageReplaceRate metric.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
index 86ea91875d38d..fc746fe3be3ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
@@ -66,6 +66,11 @@ class DataRegionMetricsMXBeanImpl implements DataRegionMetricsMXBean {
         return memMetrics.getTotalAllocatedPages();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getTotalAllocatedSize() {
+        return memMetrics.getTotalAllocatedSize();
+    }
+
     /** {@inheritDoc} */
     @Override public long getDirtyPages() {
         return memMetrics.getDirtyPages();
@@ -86,6 +91,26 @@ class DataRegionMetricsMXBeanImpl implements DataRegionMetricsMXBean {
         return memMetrics.getPhysicalMemoryPages();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getPhysicalMemorySize() {
+        return memMetrics.getPhysicalMemorySize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointBufferPages() {
+        return memMetrics.getCheckpointBufferPages();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointBufferSize() {
+        return memMetrics.getCheckpointBufferSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getPageSize() {
+        return memMetrics.getPageSize();
+    }
+
     /** {@inheritDoc} */
     @Override public void rateTimeInterval(long rateTimeInterval) {
         if (rateTimeInterval < 1000)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
index c9e3d088be6f3..d715e82caea63 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
@@ -29,6 +29,9 @@ public class DataRegionMetricsSnapshot implements DataRegionMetrics {
     /** */
     private long totalAllocatedPages;
 
+    /** */
+    private long totalAllocatedSize;
+
     /** */
     private float allocationRate;
 
@@ -53,12 +56,25 @@ public class DataRegionMetricsSnapshot implements DataRegionMetrics {
     /** */
     private long physicalMemoryPages;
 
+    /** */
+    private long physicalMemorySize;
+
+    /** */
+    private long checkpointBufferPages;
+
+    /** */
+    private long checkpointBufferSize;
+
+    /** */
+    private int pageSize;
+
     /**
      * @param metrics Metrics instance to take a copy.
      */
     public DataRegionMetricsSnapshot(DataRegionMetrics metrics) {
         name = metrics.getName();
         totalAllocatedPages = metrics.getTotalAllocatedPages();
+        totalAllocatedSize = metrics.getTotalAllocatedSize();
         allocationRate = metrics.getAllocationRate();
         evictionRate = metrics.getEvictionRate();
         largeEntriesPagesPercentage = metrics.getLargeEntriesPagesPercentage();
@@ -67,6 +83,10 @@ public DataRegionMetricsSnapshot(DataRegionMetrics metrics) {
         pageReplaceRate = metrics.getPagesReplaceRate();
         pageReplaceAge = metrics.getPagesReplaceAge();
         physicalMemoryPages = metrics.getPhysicalMemoryPages();
+        physicalMemorySize = metrics.getPhysicalMemorySize();
+        checkpointBufferPages = metrics.getCheckpointBufferPages();
+        checkpointBufferSize = metrics.getCheckpointBufferSize();
+        pageSize = metrics.getPageSize();
     }
 
     /** {@inheritDoc} */
@@ -79,6 +99,11 @@ public DataRegionMetricsSnapshot(DataRegionMetrics metrics) {
         return totalAllocatedPages;
     }
 
+    /** {@inheritDoc} */
+    @Override public long getTotalAllocatedSize() {
+        return totalAllocatedSize;
+    }
+
     /** {@inheritDoc} */
     @Override public float getAllocationRate() {
         return allocationRate;
@@ -118,4 +143,24 @@ public DataRegionMetricsSnapshot(DataRegionMetrics metrics) {
     @Override public long getPhysicalMemoryPages() {
         return physicalMemoryPages;
     }
+
+    /** {@inheritDoc} */
+    @Override public long getPhysicalMemorySize() {
+        return physicalMemorySize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointBufferPages() {
+        return checkpointBufferPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointBufferSize() {
+        return checkpointBufferSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getPageSize() {
+        return pageSize;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 1260147c41d89..8658c97c652bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -927,7 +927,7 @@ protected PageMemory createPageMemory(
     ) {
         memMetrics.persistenceEnabled(false);
 
-        return new PageMemoryNoStoreImpl(
+        PageMemory pageMem = new PageMemoryNoStoreImpl(
             log,
             memProvider,
             cctx,
@@ -936,6 +936,10 @@ protected PageMemory createPageMemory(
             memMetrics,
             false
         );
+
+        memMetrics.pageMemory(pageMem);
+
+        return pageMem;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 2c24cea232cc3..f3924002af0e1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1435,10 +1435,8 @@ public int activePagesCount() {
         return total;
     }
 
-    /**
-     * Number of used pages in checkpoint buffer.
-     */
-    public int checkpointBufferPagesCount() {
+    /** {@inheritDoc} */
+    @Override public int checkpointBufferPagesCount() {
         return cpBufPagesCntr.get();
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java
index e0fff66d9457e..f95e69b985e1c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java
@@ -17,6 +17,9 @@
 
 package org.apache.ignite.internal.processors.platform.cluster;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.UUID;
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.DataStorageMetrics;
 import org.apache.ignite.IgniteCache;
@@ -42,10 +45,6 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.UUID;
-
 /**
  * Interop projection.
  */
@@ -551,10 +550,19 @@ private static void writeDataRegionMetrics(BinaryRawWriter writer, DataRegionMet
 
         writer.writeString(metrics.getName());
         writer.writeLong(metrics.getTotalAllocatedPages());
+        writer.writeLong(metrics.getTotalAllocatedSize());
         writer.writeFloat(metrics.getAllocationRate());
         writer.writeFloat(metrics.getEvictionRate());
         writer.writeFloat(metrics.getLargeEntriesPagesPercentage());
         writer.writeFloat(metrics.getPagesFillFactor());
+        writer.writeLong(metrics.getDirtyPages());
+        writer.writeFloat(metrics.getPagesReplaceRate());
+        writer.writeFloat(metrics.getPagesReplaceAge());
+        writer.writeLong(metrics.getPhysicalMemoryPages());
+        writer.writeLong(metrics.getPhysicalMemorySize());
+        writer.writeLong(metrics.getCheckpointBufferPages());
+        writer.writeLong(metrics.getCheckpointBufferSize());
+        writer.writeInt(metrics.getPageSize());
     }
 
     /**
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
index d85d391f16e6d..bea31b12c62dd 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
@@ -28,15 +28,13 @@ public class DataRegionMetricsParityTest
         /** Known name mappings. */
         private static readonly Dictionary KnownMappings = new Dictionary
         {
-            {"PagesFillFactor", "PageFillFactor"}
+            {"PagesFillFactor", "PageFillFactor"},
+            {"PagesReplaceRate", "PageReplaceRate"},
+            {"PagesReplaceAge", "PageReplaceAge"}
         };
 
         /** Properties that are missing on .NET side. */
-        private static readonly string[] MissingProperties =
-        {
-            // IGNITE-7128
-            "DirtyPages", "PagesReplaceRate", "PagesReplaceAge", "PhysicalMemoryPages"
-        };
+        private static readonly string[] MissingProperties = {};
 
         /// 
         /// Tests the API parity.
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataRegionMetricsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataRegionMetricsTest.cs
index dd1cf5375d3b5..cf306e583c7e9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataRegionMetricsTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataRegionMetricsTest.cs
@@ -56,6 +56,9 @@ public void TestMemoryMetrics()
             Assert.AreEqual(0, memMetrics.LargeEntriesPagesPercentage);
             Assert.Greater(memMetrics.PageFillFactor, 0);
             Assert.Greater(memMetrics.TotalAllocatedPages, 1000);
+            Assert.Greater(memMetrics.PhysicalMemoryPages, 1000);
+            Assert.AreEqual(memMetrics.TotalAllocatedSize, memMetrics.TotalAllocatedPages * memMetrics.PageSize);
+            Assert.AreEqual(memMetrics.PhysicalMemorySize, memMetrics.PhysicalMemoryPages * memMetrics.PageSize);
 
             var sysMetrics = metrics[2];
             Assert.AreEqual("sysMemPlc", sysMetrics.Name);
@@ -73,6 +76,9 @@ public void TestMemoryMetrics()
             Assert.AreEqual(0, memMetrics.LargeEntriesPagesPercentage);
             Assert.Greater(memMetrics.PageFillFactor, 0);
             Assert.Greater(memMetrics.TotalAllocatedPages, 1000);
+            Assert.Greater(memMetrics.PhysicalMemoryPages, 1000);
+            Assert.AreEqual(memMetrics.TotalAllocatedSize, memMetrics.TotalAllocatedPages * memMetrics.PageSize);
+            Assert.AreEqual(memMetrics.PhysicalMemorySize, memMetrics.PhysicalMemoryPages * memMetrics.PageSize);
 
             sysMetrics = ignite.GetDataRegionMetrics("sysMemPlc");
             Assert.AreEqual("sysMemPlc", sysMetrics.Name);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IDataRegionMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/IDataRegionMetrics.cs
index 0cb619258a1bf..7be35e05f5eea 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/IDataRegionMetrics.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/IDataRegionMetrics.cs
@@ -32,6 +32,11 @@ public interface IDataRegionMetrics
         /// 
         long TotalAllocatedPages { get; }
 
+        /// 
+        /// Gets the size of allocated pages in bytes.
+        /// 
+        long TotalAllocatedSize { get; }
+
         /// 
         /// Gets the allocation rate, in pages per second.
         /// 
@@ -51,5 +56,45 @@ public interface IDataRegionMetrics
         /// Gets the page fill factor: free space to overall size ratio across all pages.
         /// 
         float PageFillFactor { get; }
+
+        /// 
+        /// Gets the number of dirty RAM pages.
+        /// 
+        long DirtyPages { get; }
+
+        /// 
+        /// Gets the rate (pages per second) at which pages get replaced with other pages from persistent storage.
+        /// 
+        float PageReplaceRate { get; }
+
+        /// 
+        /// Gets the average age (in milliseconds) for pages being replaced from persistent storage.
+        /// 
+        float PageReplaceAge { get; }
+
+        /// 
+        /// Gets the count of pages loaded to RAM.
+        /// 
+        long PhysicalMemoryPages { get; }
+
+        /// 
+        /// Gets the size of pages loaded to RAM in bytes.
+        /// 
+        long PhysicalMemorySize { get; }
+
+        /// 
+        /// Gets checkpointing buffer size in pages.
+        /// 
+        long CheckpointBufferPages { get; }
+
+        /// 
+        /// Gets checkpointing buffer size in bytes.
+        /// 
+        long CheckpointBufferSize { get; }
+
+        /// 
+        /// Gets memory page size in bytes.
+        /// 
+        int PageSize { get; }
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataRegionMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataRegionMetrics.cs
index 7b174a69a1e48..c5428367cc694 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataRegionMetrics.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/DataRegionMetrics.cs
@@ -34,10 +34,19 @@ public DataRegionMetrics(IBinaryRawReader reader)
 
             Name = reader.ReadString();
             TotalAllocatedPages = reader.ReadLong();
+            TotalAllocatedSize = reader.ReadLong();
             AllocationRate = reader.ReadFloat();
             EvictionRate = reader.ReadFloat();
             LargeEntriesPagesPercentage = reader.ReadFloat();
             PageFillFactor = reader.ReadFloat();
+            DirtyPages = reader.ReadLong();
+            PageReplaceRate = reader.ReadFloat();
+            PageReplaceAge = reader.ReadFloat();
+            PhysicalMemoryPages = reader.ReadLong();
+            PhysicalMemorySize = reader.ReadLong();
+            CheckpointBufferPages = reader.ReadLong();
+            CheckpointBufferSize = reader.ReadLong();
+            PageSize = reader.ReadInt();
         }
 
         /**  */
@@ -46,16 +55,43 @@ public DataRegionMetrics(IBinaryRawReader reader)
         /**  */
         public long TotalAllocatedPages { get; private set; }
 
+        /**  */
+        public long TotalAllocatedSize { get; private set; }
+
         /**  */
         public float AllocationRate { get; private set; }
         
         /**  */
         public float EvictionRate { get; private set; }
 
+        /**  */
+        public long DirtyPages { get; private set; }
+
+        /**  */
+        public float PageReplaceRate { get; private set; }
+
+        /**  */
+        public float PageReplaceAge { get; private set; }
+
         /**  */
         public float LargeEntriesPagesPercentage { get; private set; }
 
         /**  */
         public float PageFillFactor { get; private set; }
+
+        /**  */
+        public long PhysicalMemoryPages { get; private set; }
+
+        /**  */
+        public long PhysicalMemorySize { get; private set; }
+
+        /**  */
+        public long CheckpointBufferPages { get; private set; }
+
+        /**  */
+        public long CheckpointBufferSize { get; private set; }
+
+        /**  */
+        public int PageSize { get; private set; }
     }
 }

From 95d38643cfdefefb64f0eb221761027b6f7ef6d6 Mon Sep 17 00:00:00 2001
From: Anton Vinogradov 
Date: Mon, 22 Jan 2018 22:05:17 +0300
Subject: [PATCH 30/65] IGNITE-7003: Ability to change WAL mode in runtime.
 This closes #3335.

---
 .../jdbc/suite/IgniteJdbcDriverTestSuite.java |   3 +
 .../thin/JdbcThinWalModeChangeSelfTest.java   | 113 +++
 .../java/org/apache/ignite/IgniteCluster.java |  47 +
 .../org/apache/ignite/internal/GridTopic.java |   5 +-
 .../cluster/IgniteClusterAsyncImpl.java       |  14 +
 .../internal/cluster/IgniteClusterImpl.java   |  45 +
 .../communication/GridIoMessageFactory.java   |   7 +-
 .../pagemem/store/IgnitePageStoreManager.java |   5 +
 .../wal/IgniteWriteAheadLogManager.java       |   7 +
 .../pagemem/wal/record/PageSnapshot.java      |   7 +-
 .../wal/record/WalRecordCacheGroupAware.java  |  28 +
 .../wal/record/delta/PageDeltaRecord.java     |   9 +-
 .../record/delta/PartitionDestroyRecord.java  |   9 +-
 .../delta/PartitionMetaStateRecord.java       |   9 +-
 .../processors/cache/CacheGroupContext.java   |  33 +-
 .../processors/cache/CacheGroupData.java      |  36 +-
 .../cache/CacheGroupDescriptor.java           |  75 +-
 .../processors/cache/ClusterCachesInfo.java   |  17 +-
 .../processors/cache/ExchangeActions.java     |   2 +-
 .../processors/cache/GridCacheMapEntry.java   |   8 +-
 .../GridCachePartitionExchangeManager.java    |  10 +-
 .../processors/cache/GridCacheProcessor.java  |  53 +-
 .../cache/GridCacheSharedContext.java         |  22 +-
 .../cache/GridCacheSharedManager.java         |   5 +
 .../cache/GridCacheSharedManagerAdapter.java  |   5 +
 .../cache/WalStateAbstractMessage.java        | 135 +++
 .../processors/cache/WalStateAckMessage.java  | 221 +++++
 .../cache/WalStateDistributedProcess.java     | 172 ++++
 .../cache/WalStateFinishMessage.java          |  73 ++
 .../processors/cache/WalStateManager.java     | 914 ++++++++++++++++++
 .../cache/WalStateNodeLeaveExchangeTask.java  |  57 ++
 .../cache/WalStateProposeMessage.java         | 104 ++
 .../processors/cache/WalStateResult.java      |  94 ++
 .../GridDistributedTxRemoteAdapter.java       |   2 +-
 .../dht/GridDhtLocalPartition.java            |   2 +-
 .../GridDhtPartitionsExchangeFuture.java      |  36 +
 .../cache/persistence/CheckpointFuture.java   |  35 +
 .../GridCacheDatabaseSharedManager.java       | 219 ++++-
 .../persistence/GridCacheOffheapManager.java  |   3 +-
 .../IgniteCacheDatabaseSharedManager.java     |  27 +
 .../file/FilePageStoreManager.java            |  17 +-
 .../persistence/metastorage/MetaStorage.java  |  72 +-
 .../metastorage/ReadOnlyMetastorage.java      |  12 +
 .../persistence/pagemem/PageMemoryImpl.java   |  25 +-
 .../persistence/tree/util/PageHandler.java    |   2 +-
 .../wal/FileWriteAheadLogManager.java         |   8 +
 .../wal/reader/IgniteWalIteratorFactory.java  |   2 +-
 .../transactions/IgniteTxLocalAdapter.java    |   6 +-
 .../odbc/ClientListenerProcessor.java         |   7 +
 .../ignite/internal/sql/SqlKeyword.java       |   9 +
 .../apache/ignite/internal/sql/SqlParser.java |  31 +-
 .../sql/command/SqlAlterTableCommand.java     | 114 +++
 .../ignite/internal/util/GridIntList.java     |   2 +-
 .../cache/WalModeChangeAbstractSelfTest.java  | 282 ++++++
 .../cache/WalModeChangeAdvancedSelfTest.java  | 535 ++++++++++
 .../WalModeChangeCommonAbstractSelfTest.java  | 338 +++++++
 ...ngeCoordinatorNotAffinityNodeSelfTest.java |  30 +
 .../cache/WalModeChangeSelfTest.java          |  30 +
 .../pagemem/BPlusTreePageMemoryImplTest.java  |   1 +
 .../BPlusTreeReuseListPageMemoryImplTest.java |   1 +
 .../IndexStoragePageMemoryImplTest.java       |   1 +
 .../pagemem/NoOpPageStoreManager.java         |  10 +
 .../persistence/pagemem/NoOpWALManager.java   |  10 +
 .../pagemem/PageMemoryImplNoLoadTest.java     |   1 +
 .../pagemem/PageMemoryImplTest.java           |   1 +
 .../hashmap/GridCacheTestContext.java         |   2 +
 .../multijvm/IgniteClusterProcessProxy.java   |  15 +
 .../testsuites/IgniteCacheTestSuite6.java     |   7 +
 .../processors/query/h2/IgniteH2Indexing.java |  10 +-
 .../query/h2/ddl/DdlStatementsProcessor.java  |  35 +-
 70 files changed, 4172 insertions(+), 112 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WalRecordCacheGroupAware.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAbstractMessage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateDistributedProcess.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateNodeLeaveExchangeTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointFuture.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlAlterTableCommand.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAbstractSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAdvancedSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCoordinatorNotAffinityNodeSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeSelfTest.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index 062de76efde95..87eeb84d7556f 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -62,6 +62,7 @@
 import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSkipReducerOnUpdateSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSkipReducerOnUpdateSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinWalModeChangeSelfTest;
 
 /**
  * JDBC driver test suite.
@@ -163,6 +164,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.class));
         suite.addTest(new TestSuite(JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.class));
 
+        // Various commands.
+        suite.addTest(new TestSuite(JdbcThinWalModeChangeSelfTest.class));
 
         return suite;
     }
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java
new file mode 100644
index 0000000000000..6a3ac52986c1a
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc.thin;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.WalModeChangeAbstractSelfTest;
+import org.apache.ignite.internal.processors.query.QueryUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+
+/**
+ * Tests for WAL mode change from within JDBC driver.
+ */
+public class JdbcThinWalModeChangeSelfTest extends WalModeChangeAbstractSelfTest {
+    /**
+     * Constructor.
+     */
+    public JdbcThinWalModeChangeSelfTest() {
+        super(false, true);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void createCache(Ignite node, CacheConfiguration ccfg) {
+        String template = ccfg.getCacheMode() == CacheMode.PARTITIONED ?
+            QueryUtils.TEMPLATE_PARTITIONED : QueryUtils.TEMPLATE_REPLICATED;
+
+        String cmd = "CREATE TABLE " + ccfg.getName() + " (k BIGINT PRIMARY KEY, v BIGINT) WITH \"" +
+            "TEMPLATE=" + template + ", " +
+            "CACHE_NAME=" + ccfg.getName() + ", " +
+            "ATOMICITY=" + ccfg.getAtomicityMode() +
+            (ccfg.getGroupName() != null ? ", CACHE_GROUP=" + ccfg.getGroupName() : "") +
+            (ccfg.getDataRegionName() != null ? ", DATA_REGION=" + ccfg.getDataRegionName() : "") +
+            "\"";
+
+        execute(node, cmd);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void destroyCache(Ignite node, String cacheName) {
+        String cmd = "DROP TABLE IF EXISTS " + cacheName;
+
+        execute(node, cmd);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean walEnable(Ignite node, String cacheName) {
+        String cmd = "ALTER TABLE " + cacheName + " LOGGING";
+
+        execute(node, cmd);
+
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean walDisable(Ignite node, String cacheName) {
+        String cmd = "ALTER TABLE " + cacheName + " NOLOGGING";
+
+        execute(node, cmd);
+
+        return false;
+    }
+
+    /**
+     * Execute single command.
+     *
+     * @param node Node.
+     * @param cmd Command.
+     */
+    private static void execute(Ignite node, String cmd) {
+        try (Connection conn = connect(node)) {
+            try (Statement stmt = conn.createStatement()) {
+                stmt.executeUpdate(cmd);
+            }
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Get connection for node.
+     *
+     * @param node Node.
+     * @return Connection.
+     */
+    private static Connection connect(Ignite node) throws Exception {
+        IgniteKernal node0 = (IgniteKernal)node;
+
+        int port = node0.context().sqlListener().port();
+
+        return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1:" + port);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java b/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
index b80de0c1b9556..7329d682ebea4 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
@@ -469,4 +469,51 @@ public IgniteFuture> startNodesAsync(Collecti
     /** {@inheritDoc} */
     @Deprecated
     @Override public IgniteCluster withAsync();
+
+    /**
+     * Disables write-ahead logging for specified cache. When WAL is disabled, changes are not logged to disk.
+     * This significantly improves cache update speed. The drawback is absence of local crash-recovery guarantees.
+     * If node is crashed, local content of WAL-disabled cache will be cleared on restart to avoid data corruption.
+     * 

+     * Internally this method will wait for all current cache operations to finish and prevent new cache operations
+     * from being executed. Then checkpoint is initiated to flush all data to disk. Control is returned to the callee
+     * when all dirty pages are prepared for checkpoint, but not necessarily flushed to disk.
+     * 

+     * WAL state can be changed only for persistent caches.
+     *
+     * @param cacheName Cache name.
+     * @return Whether WAL disabled by this call.
+     * @throws IgniteException If error occurs.
+     * @see #enableWal(String)
+     * @see #isWalEnabled(String)
+     */
+    public boolean disableWal(String cacheName) throws IgniteException;
+
+    /**
+     * Enables write-ahead logging for specified cache. Restoring crash-recovery guarantees of a previous call to
+     * {@link #disableWal(String)}.
+     * 

+     * Internally this method will wait for all current cache operations to finish and prevent new cache operations
+     * from being executed. Then checkpoint is initiated to flush all data to disk. Control is returned to the callee
+     * when all data is persisted to disk.
+     * 

+     * WAL state can be changed only for persistent caches.
+     *
+     * @param cacheName Cache name.
+     * @return Whether WAL enabled by this call.
+     * @throws IgniteException If error occurs.
+     * @see #disableWal(String)
+     * @see #isWalEnabled(String)
+     */
+    public boolean enableWal(String cacheName) throws IgniteException;
+
+    /**
+     * Checks if write-ahead logging is enabled for specified cache.
+     *
+     * @param cacheName Cache name.
+     * @return {@code True} if WAL is enabled for cache.
+     * @see #disableWal(String)
+     * @see #enableWal(String)
+     */
+    public boolean isWalEnabled(String cacheName);
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index abdbf956017f4..7522881417cee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -115,7 +115,10 @@ public enum GridTopic {
     TOPIC_SCHEMA,
 
     /** */
-    TOPIC_INTERNAL_DIAGNOSTIC;
+    TOPIC_INTERNAL_DIAGNOSTIC,
+
+    /** */
+    TOPIC_WAL;
 
     /** Enum values. */
     private static final GridTopic[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
index e238cb72ba5c3..43e97b5e4ac94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
@@ -336,6 +336,20 @@ public IgniteClusterAsyncImpl(IgniteClusterImpl cluster) {
         return cluster.clientReconnectFuture();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean enableWal(String cacheName) throws IgniteException {
+        return cluster.enableWal(cacheName);
+    }
+    /** {@inheritDoc} */
+    @Override public boolean disableWal(String cacheName) throws IgniteException {
+        return cluster.disableWal(cacheName);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isWalEnabled(String cacheName) {
+        return cluster.isWalEnabled(cacheName);
+    }
+
     /** {@inheritDoc} */
     @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
         cluster = (IgniteClusterImpl)in.readObject();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
index 392b43ca3e0e7..8cdb5502318e3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
@@ -475,6 +475,51 @@ private Collection getConsistentIds(Collection n
         return new IgniteClusterAsyncImpl(this);
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean enableWal(String cacheName) throws IgniteException {
+        return changeWalMode(cacheName, true);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean disableWal(String cacheName) throws IgniteException {
+        return changeWalMode(cacheName, false);
+    }
+
+    /**
+     * Change WAL mode.
+     *
+     * @param cacheName Cache name.
+     * @param enabled Enabled flag.
+     * @return {@code True} if WAL mode was changed as a result of this call.
+     */
+    private boolean changeWalMode(String cacheName, boolean enabled) {
+        A.notNull(cacheName, "cacheName");
+
+        guard();
+
+        try {
+            return ctx.cache().changeWalMode(Collections.singleton(cacheName), enabled).get();
+        }
+        catch (IgniteCheckedException e) {
+            throw U.convertException(e);
+        }
+        finally {
+            unguard();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isWalEnabled(String cacheName) {
+        guard();
+
+        try {
+            return ctx.cache().walEnabled(cacheName);
+        }
+        finally {
+            unguard();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isAsync() {
         return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 78cb7a8948479..b220040d23747 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -46,6 +46,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
 import org.apache.ignite.internal.processors.cache.GridChangeGlobalStateMessageResponse;
 import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
+import org.apache.ignite.internal.processors.cache.WalStateAckMessage;
 import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage;
 import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTtlUpdateRequest;
@@ -879,8 +880,12 @@ public GridIoMessageFactory(MessageFactory[] ext) {
 
                 break;
 
+            case 129:
+                msg = new WalStateAckMessage();
 
-            // [-3..119] [124..128] [-23..-27] [-36..-55]- this
+                break;
+
+            // [-3..119] [124..129] [-23..-27] [-36..-55]- this
             // [120..123] - DR
             // [-4..-22, -30..-35] - SQL
             // [2048..2053] - Snapshots
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index deb21a77f9554..2707a5e0db2c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -199,6 +199,11 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac
      */
     public boolean hasIndexStore(int grpId);
 
+    /**
+     * @param grpDesc Cache group descriptor.
+     */
+    public void beforeCacheGroupStart(CacheGroupDescriptor grpDesc);
+
     /**
      * Calculates number of pages currently allocated for given cache group.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 19b47e641e56e..6c3c36e43656e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -121,4 +121,11 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
      * @return True if given pointer is located in reserved segment.
      */
     public boolean reserved(WALPointer ptr);
+
+    /**
+     * Checks WAL disabled for cache group.
+     *
+     * @param grpId Group id.
+     */
+    public boolean disabled(int grpId);
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
index 126b84510ad9a..1aa065e10df40 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
@@ -29,7 +29,7 @@
 /**
  *
  */
-public class PageSnapshot extends WALRecord {
+public class PageSnapshot extends WALRecord implements WalRecordCacheGroupAware{
     /** */
     @GridToStringExclude
     private byte[] pageData;
@@ -100,4 +100,9 @@ public FullPageId fullPageId() {
             GridUnsafe.cleanDirectBuffer(buf);
         }
     }
+
+    /** {@inheritDoc} */
+    @Override public int groupId() {
+        return fullPageId.groupId();
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WalRecordCacheGroupAware.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WalRecordCacheGroupAware.java
new file mode 100644
index 0000000000000..0964721da93eb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WalRecordCacheGroupAware.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+/**
+ * WAL record related to specific cache group.
+ */
+public interface WalRecordCacheGroupAware {
+    /**
+     * @return Cache group ID.
+     */
+    public int groupId();
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageDeltaRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageDeltaRecord.java
index 260c2c84cf6df..a70907a053a10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageDeltaRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageDeltaRecord.java
@@ -20,6 +20,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WalRecordCacheGroupAware;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -27,7 +28,7 @@
 /**
  * Abstract page delta record.
  */
-public abstract class PageDeltaRecord extends WALRecord {
+public abstract class PageDeltaRecord extends WALRecord implements WalRecordCacheGroupAware {
     /** */
     private int grpId;
 
@@ -51,10 +52,8 @@ public long pageId() {
         return pageId;
     }
 
-    /**
-     * @return Cache group ID.
-     */
-    public int groupId() {
+    /** {@inheritDoc} */
+    @Override public int groupId() {
         return grpId;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionDestroyRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionDestroyRecord.java
index 9daa0eace9133..8a2b4f7e13fe6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionDestroyRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionDestroyRecord.java
@@ -18,12 +18,13 @@
 package org.apache.ignite.internal.pagemem.wal.record.delta;
 
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WalRecordCacheGroupAware;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
  *
  */
-public class PartitionDestroyRecord extends WALRecord {
+public class PartitionDestroyRecord extends WALRecord implements WalRecordCacheGroupAware {
     /** */
     private int grpId;
 
@@ -44,10 +45,8 @@ public PartitionDestroyRecord(int grpId, int partId) {
         return RecordType.PARTITION_DESTROY;
     }
 
-    /**
-     * @return Cache group ID.
-     */
-    public int groupId() {
+    /** {@inheritDoc} */
+    @Override public int groupId() {
         return grpId;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java
index f4dab907b6f75..a89f7be04b406 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java
@@ -18,13 +18,14 @@
 package org.apache.ignite.internal.pagemem.wal.record.delta;
 
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WalRecordCacheGroupAware;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
  *
  */
-public class PartitionMetaStateRecord extends WALRecord {
+public class PartitionMetaStateRecord extends WALRecord implements WalRecordCacheGroupAware {
     /** State. */
     private final byte state;
 
@@ -60,10 +61,8 @@ public byte state() {
         return state;
     }
 
-    /**
-     * @return Cache group ID.
-     */
-    public int groupId() {
+    /** {@inheritDoc} */
+    @Override public int groupId() {
         return grpId;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 6cd39edd8d06e..246f298f5d046 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -147,6 +147,9 @@ public class CacheGroupContext {
     /** MXBean. */
     private CacheGroupMetricsMXBean mxBean;
 
+    /** */
+    private volatile boolean walEnabled;
+
     /**
      * @param grpId Group ID.
      * @param ctx Context.
@@ -159,6 +162,7 @@ public class CacheGroupContext {
      * @param freeList Free list.
      * @param reuseList Reuse list.
      * @param locStartVer Topology version when group was started on local node.
+     * @param walEnabled Wal enabled flag.
      */
     CacheGroupContext(
         GridCacheSharedContext ctx,
@@ -171,7 +175,8 @@ public class CacheGroupContext {
         CacheObjectContext cacheObjCtx,
         FreeList freeList,
         ReuseList reuseList,
-        AffinityTopologyVersion locStartVer) {
+        AffinityTopologyVersion locStartVer,
+        boolean walEnabled) {
         assert ccfg != null;
         assert dataRegion != null || !affNode;
         assert grpId != 0 : "Invalid group ID [cache=" + ccfg.getName() + ", grpName=" + ccfg.getGroupName() + ']';
@@ -187,6 +192,9 @@ public class CacheGroupContext {
         this.reuseList = reuseList;
         this.locStartVer = locStartVer;
         this.cacheType = cacheType;
+        this.walEnabled = walEnabled;
+
+        persistWalState(walEnabled);
 
         ioPlc = cacheType.ioPolicy();
 
@@ -992,4 +1000,27 @@ public CacheGroupMetricsMXBean mxBean() {
     @Override public String toString() {
         return "CacheGroupContext [grp=" + cacheOrGroupName() + ']';
     }
+
+    /**
+     * WAL enabled flag.
+     */
+    public boolean walEnabled() {
+        return walEnabled;
+    }
+
+    /**
+     * @param enabled WAL enabled flag.
+     */
+    public void walEnabled(boolean enabled) {
+        persistWalState(enabled);
+
+        this.walEnabled = enabled;
+    }
+
+    /**
+     * @param enabled Enabled flag..
+     */
+    private void persistWalState(boolean enabled) {
+        shared().database().walEnabled(grpId, enabled);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupData.java
index 617db567d8b0c..5588cfb31d4a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupData.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupData.java
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import java.io.Serializable;
-import java.util.Map;
-import java.util.UUID;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -27,6 +24,11 @@
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
 
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
 /**
  *
  */
@@ -62,6 +64,12 @@ public class CacheGroupData implements Serializable {
     /** Persistence enabled flag. */
     private final boolean persistenceEnabled;
 
+    /** WAL state. */
+    private final boolean walEnabled;
+
+    /** WAL change requests. */
+    private final List walChangeReqs;
+
     /**
      * @param cacheCfg Cache configuration.
      * @param grpName Group name.
@@ -71,6 +79,8 @@ public class CacheGroupData implements Serializable {
      * @param deploymentId Deployment ID.
      * @param caches Cache group caches.
      * @param persistenceEnabled Persistence enabled flag.
+     * @param walEnabled WAL state.
+     * @param walChangeReqs WAL change requests.
      */
     CacheGroupData(
         CacheConfiguration cacheCfg,
@@ -81,7 +91,9 @@ public class CacheGroupData implements Serializable {
         IgniteUuid deploymentId,
         Map caches,
         long flags,
-        boolean persistenceEnabled) {
+        boolean persistenceEnabled,
+        boolean walEnabled,
+        List walChangeReqs) {
         assert cacheCfg != null;
         assert grpId != 0 : cacheCfg.getName();
         assert deploymentId != null : cacheCfg.getName();
@@ -95,6 +107,8 @@ public class CacheGroupData implements Serializable {
         this.caches = caches;
         this.flags = flags;
         this.persistenceEnabled = persistenceEnabled;
+        this.walEnabled = walEnabled;
+        this.walChangeReqs = walChangeReqs;
     }
 
     /**
@@ -153,6 +167,20 @@ public boolean persistenceEnabled() {
         return persistenceEnabled;
     }
 
+    /**
+     * @return {@code True} if WAL is enabled.
+     */
+    public boolean walEnabled() {
+        return walEnabled;
+    }
+
+    /**
+     * @return WAL mode change requests.
+     */
+    public List walChangeRequests() {
+        return walChangeReqs;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(CacheGroupData.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java
index 86e330e47a6dd..70cdcc735af62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java
@@ -17,7 +17,11 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -61,6 +65,12 @@ public class CacheGroupDescriptor {
     /** Persistence enabled flag. */
     private final boolean persistenceEnabled;
 
+    /** WAL enabled state. */
+    private volatile boolean walEnabled;
+
+    /** Pending WAL change requests. */
+    private final LinkedList walChangeReqs;
+
     /**
      * @param cacheCfg Cache configuration.
      * @param grpName Group name.
@@ -70,7 +80,10 @@ public class CacheGroupDescriptor {
      * @param deploymentId Deployment ID.
      * @param caches Cache group caches.
      * @param persistenceEnabled Persistence enabled flag.
+     * @param walEnabled Whether WAL is enabled.
+     * @param walChangeReqs Pending WAL change requests.
      */
+    @SuppressWarnings("unchecked")
     CacheGroupDescriptor(
         CacheConfiguration cacheCfg,
         @Nullable String grpName,
@@ -79,7 +92,9 @@ public class CacheGroupDescriptor {
         @Nullable AffinityTopologyVersion startTopVer,
         IgniteUuid deploymentId,
         Map caches,
-        boolean persistenceEnabled) {
+        boolean persistenceEnabled,
+        boolean walEnabled,
+        @Nullable Collection walChangeReqs) {
         assert cacheCfg != null;
         assert grpId != 0;
 
@@ -91,6 +106,8 @@ public class CacheGroupDescriptor {
         this.cacheCfg = new CacheConfiguration<>(cacheCfg);
         this.caches = caches;
         this.persistenceEnabled = persistenceEnabled;
+        this.walEnabled = walEnabled;
+        this.walChangeReqs = walChangeReqs == null ? new LinkedList<>() : new LinkedList<>(walChangeReqs);
     }
 
     /**
@@ -107,6 +124,62 @@ public IgniteUuid deploymentId() {
         return deploymentId;
     }
 
+    /**
+     * @return {@code True} if WAL is enabled for cache group.
+     */
+    public boolean walEnabled() {
+        return walEnabled;
+    }
+
+    /**
+     * @param walEnabled {@code True} if WAL is enabled for cache group.
+     */
+    public void walEnabled(boolean walEnabled) {
+        this.walEnabled = walEnabled;
+    }
+
+    /**
+     * @return Pending WAL change requests.
+     */
+    public List walChangeRequests() {
+        return new ArrayList<>(walChangeReqs);
+    }
+
+    /**
+     * @return {@code True} whether there are pending WAL change requests.
+     */
+    public boolean hasWalChangeRequests() {
+        return !walChangeReqs.isEmpty();
+    }
+
+    /**
+     * @return Next pending WAL change request or {@code null} if none available.
+     */
+    @Nullable public WalStateProposeMessage nextWalChangeRequest() {
+        return walChangeReqs.isEmpty() ? null : walChangeReqs.getFirst();
+    }
+
+    /**
+     * Add pending WAL change request.
+     *
+     * @param msg Message.
+     * @return {@code True} if this is the very first enlisted message.
+     */
+    public boolean addWalChangeRequest(WalStateProposeMessage msg) {
+        boolean first = !hasWalChangeRequests();
+
+        walChangeReqs.addLast(msg);
+
+        return first;
+    }
+
+    /**
+     * Remove pending WAL change request.
+     */
+    public void removeWalChangeRequest() {
+        walChangeReqs.removeFirst();
+    }
+
     /**
      * @param cacheName Cache name
      * @param cacheId Cache ID.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 3b5a3a59a1f23..08a910b81603d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -920,7 +920,9 @@ private CacheNodeCommonDiscoveryData collectCommonDiscoveryData() {
                 grpDesc.deploymentId(),
                 grpDesc.caches(),
                 0,
-                grpDesc.persistenceEnabled());
+                grpDesc.persistenceEnabled(),
+                grpDesc.walEnabled(),
+                grpDesc.walChangeRequests());
 
             cacheGrps.put(grpDesc.groupId(), grpData);
         }
@@ -999,7 +1001,9 @@ public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) {
                 grpData.startTopologyVersion(),
                 grpData.deploymentId(),
                 grpData.caches(),
-                grpData.persistenceEnabled());
+                grpData.persistenceEnabled(),
+                grpData.walEnabled(),
+                grpData.walChangeRequests());
 
             if (locCacheGrps.containsKey(grpDesc.groupId())) {
                 CacheGroupDescriptor locGrpCfg = locCacheGrps.get(grpDesc.groupId());
@@ -1521,6 +1525,8 @@ private CacheGroupDescriptor registerCacheGroup(
 
         Map caches = Collections.singletonMap(startedCacheCfg.getName(), cacheId);
 
+        boolean persistent = CU.isPersistentCache(startedCacheCfg, ctx.config().getDataStorageConfiguration());
+
         CacheGroupDescriptor grpDesc = new CacheGroupDescriptor(
             startedCacheCfg,
             startedCacheCfg.getGroupName(),
@@ -1529,7 +1535,12 @@ private CacheGroupDescriptor registerCacheGroup(
             curTopVer != null ? curTopVer.nextMinorVersion() : null,
             deploymentId,
             caches,
-            CU.isPersistentCache(startedCacheCfg, ctx.config().getDataStorageConfiguration()));
+            persistent,
+            persistent,
+            null);
+
+        if (ctx.cache().context().pageStore() != null)
+            ctx.cache().context().pageStore().beforeCacheGroupStart(grpDesc);
 
         CacheGroupDescriptor old = registeredCacheGrps.put(grpId, grpDesc);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
index d84f8d8eff0c4..bcf3f408c222f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
@@ -149,7 +149,7 @@ public boolean hasStop() {
      */
     public Set cachesToResetLostPartitions() {
         Set caches = null;
-        
+
         if (cachesToResetLostParts != null)
             caches = new HashSet<>(cachesToResetLostParts.keySet());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index cb36acd803742..26e2b18581202 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -1046,7 +1046,7 @@ else if (interceptorVal != val0)
             if (updateCntr != null && updateCntr != 0)
                 updateCntr0 = updateCntr;
 
-            if (tx != null && cctx.group().persistenceEnabled())
+            if (tx != null && cctx.group().persistenceEnabled() && cctx.group().walEnabled())
                 logPtr = logTxUpdate(tx, val, expireTime, updateCntr0);
 
             update(val, expireTime, ttl, newVer, true);
@@ -1235,7 +1235,7 @@ protected Object keyValue(boolean cpy) {
             if (updateCntr != null && updateCntr != 0)
                 updateCntr0 = updateCntr;
 
-            if (tx != null && cctx.group().persistenceEnabled())
+            if (tx != null && cctx.group().persistenceEnabled() && cctx.group().walEnabled())
                 logPtr = logTxUpdate(tx, null, 0, updateCntr0);
 
             drReplicate(drType, null, newVer, topVer);
@@ -2707,7 +2707,7 @@ protected final boolean hasValueUnlocked() {
 
             boolean update;
 
-            boolean walEnabled = !cctx.isNear() && cctx.group().persistenceEnabled();
+            boolean walEnabled = !cctx.isNear() && cctx.group().persistenceEnabled() && cctx.group().walEnabled();
 
             if (cctx.group().persistenceEnabled()) {
                 unswap(false);
@@ -3540,7 +3540,7 @@ protected void logUpdate(GridCacheOperation op, CacheObject val, GridCacheVersio
         assert cctx.atomic();
 
         try {
-            if (cctx.group().persistenceEnabled())
+            if (cctx.group().persistenceEnabled() && cctx.group().walEnabled())
                 cctx.shared().wal().log(new DataRecord(new DataEntry(
                     cctx.cacheId(),
                     key,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index ef972389fda8e..9b9284f04b10a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -481,6 +481,12 @@ else if (customMsg instanceof SnapshotDiscoveryMessage
 
                 exchFut = exchangeFuture(exchId, evt, null, null, null);
             }
+            else if (customMsg instanceof WalStateAbstractMessage
+                && ((WalStateAbstractMessage)customMsg).needExchange()) {
+                exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
+
+                exchFut = exchangeFuture(exchId, evt, null, null, null);
+            }
             else {
                 // Process event as custom discovery task if needed.
                 CachePartitionExchangeWorkerTask task =
@@ -509,8 +515,10 @@ else if (customMsg instanceof SnapshotDiscoveryMessage
         notifyNodeFail(evt);
 
         // Notify indexing engine about node leave so that we can re-map coordinator accordingly.
-        if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
+        if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) {
             exchWorker.addCustomTask(new SchemaNodeLeaveExchangeWorkerTask(evt.eventNode()));
+            exchWorker.addCustomTask(new WalStateNodeLeaveExchangeTask(evt.eventNode()));
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index f234ecc8db4cc..1561f25f8e0e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -369,6 +369,11 @@ else if (task instanceof StopCachesOnClientReconnectExchangeTask) {
 
             task0.onDone();
         }
+        else if (task instanceof WalStateNodeLeaveExchangeTask) {
+            WalStateNodeLeaveExchangeTask task0 = (WalStateNodeLeaveExchangeTask)task;
+
+            sharedCtx.walState().onNodeLeft(task0.node().id());
+        }
         else
             U.warn(log, "Unsupported custom exchange task: " + task);
     }
@@ -807,6 +812,8 @@ public Collection cacheGroups() {
 
             cachesInfo.onKernalStart(checkConsistency);
 
+            sharedCtx.walState().onKernalStart();
+
             ctx.query().onCacheKernalStart();
 
             sharedCtx.exchange().onKernalStart(active, false);
@@ -1960,7 +1967,9 @@ private CacheGroupContext startCacheGroup(
             cacheObjCtx,
             freeList,
             reuseList,
-            exchTopVer);
+            exchTopVer,
+            desc.walEnabled()
+        );
 
         for (Object obj : grp.configuredUserObjects())
             prepare(cfg, obj, false);
@@ -2337,6 +2346,8 @@ private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx,
         else
             dbMgr = new IgniteCacheDatabaseSharedManager();
 
+        WalStateManager walStateMgr = new WalStateManager(ctx);
+
         IgniteCacheSnapshotManager snpMgr = ctx.plugins().createComponent(IgniteCacheSnapshotManager.class);
 
         if (snpMgr == null)
@@ -2355,6 +2366,7 @@ private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx,
             mvccMgr,
             pageStoreMgr,
             walMgr,
+            walStateMgr,
             dbMgr,
             snpMgr,
             depMgr,
@@ -2390,6 +2402,8 @@ private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx,
     /** {@inheritDoc} */
     @Override public void onGridDataReceived(GridDiscoveryData data) {
         cachesInfo.onGridDataReceived(data);
+
+        sharedCtx.walState().onCachesInfoCollected();
     }
 
     /**
@@ -2958,6 +2972,32 @@ public IgniteInternalFuture dynamicDestroyCaches(Collection cacheName
         return compoundFut;
     }
 
+    /**
+     * Change WAL mode.
+     *
+     * @param cacheNames Cache names.
+     * @param enabled Enabled flag.
+     * @return Future completed when operation finished.
+     */
+    public IgniteInternalFuture changeWalMode(Collection cacheNames, boolean enabled) {
+        if (transactions().tx() != null || sharedCtx.lockedTopologyVersion(null) != null)
+            throw new IgniteException("Cache WAL mode cannot be changed within lock or transaction.");
+
+        return sharedCtx.walState().init(cacheNames, enabled);
+    }
+
+    /**
+     * @param cacheName Cache name.
+     */
+    public boolean walEnabled(String cacheName) {
+        DynamicCacheDescriptor desc = ctx.cache().cacheDescriptor(cacheName);
+
+        if (desc == null)
+            throw new IgniteException("Cache not found: " + cacheName);
+
+        return desc.groupDescriptor().walEnabled();
+    }
+
     /**
      * @param cacheName Cache name to close.
      * @return Future that will be completed when cache is closed.
@@ -3194,6 +3234,17 @@ public boolean onCustomEvent(DiscoveryCustomMessage msg, AffinityTopologyVersion
             ((SnapshotDiscoveryMessage)msg).needExchange())
             return true;
 
+        if (msg instanceof WalStateAbstractMessage) {
+            WalStateAbstractMessage msg0 = (WalStateAbstractMessage)msg;
+
+            if (msg0 instanceof WalStateProposeMessage)
+                sharedCtx.walState().onProposeDiscovery((WalStateProposeMessage)msg);
+            else if (msg0 instanceof WalStateFinishMessage)
+                sharedCtx.walState().onFinishDiscovery((WalStateFinishMessage)msg);
+
+            return msg0.needExchange();
+        }
+
         if (msg instanceof DynamicCacheChangeBatch)
             return cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index 306723bb2f4a2..cb01532ded5e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -107,6 +107,9 @@ public class GridCacheSharedContext {
     /** Write ahead log manager. {@code Null} if persistence is not enabled. */
     @Nullable private IgniteWriteAheadLogManager walMgr;
 
+    /** Write ahead log state manager. */
+    private WalStateManager walStateMgr;
+
     /** Database manager. */
     private IgniteCacheDatabaseSharedManager dbMgr;
 
@@ -171,6 +174,7 @@ public class GridCacheSharedContext {
      * @param mvccMgr MVCC manager.
      * @param pageStoreMgr Page store manager. {@code Null} if persistence is not enabled.
      * @param walMgr WAL manager. {@code Null} if persistence is not enabled.
+     * @param walStateMgr WAL state manager.
      * @param depMgr Deployment manager.
      * @param exchMgr Exchange manager.
      * @param affMgr Affinity manager.
@@ -186,6 +190,7 @@ public GridCacheSharedContext(
         GridCacheMvccManager mvccMgr,
         @Nullable IgnitePageStoreManager pageStoreMgr,
         @Nullable IgniteWriteAheadLogManager walMgr,
+        WalStateManager walStateMgr,
         IgniteCacheDatabaseSharedManager dbMgr,
         IgniteCacheSnapshotManager snpMgr,
         GridCacheDeploymentManager depMgr,
@@ -198,7 +203,8 @@ public GridCacheSharedContext(
     ) {
         this.kernalCtx = kernalCtx;
 
-        setManagers(mgrs, txMgr, jtaMgr, verMgr, mvccMgr, pageStoreMgr, walMgr, dbMgr, snpMgr, depMgr, exchMgr, affMgr, ioMgr, ttlMgr);
+        setManagers(mgrs, txMgr, jtaMgr, verMgr, mvccMgr, pageStoreMgr, walMgr, walStateMgr, dbMgr, snpMgr, depMgr,
+            exchMgr, affMgr, ioMgr, ttlMgr);
 
         this.storeSesLsnrs = storeSesLsnrs;
 
@@ -363,6 +369,7 @@ void onReconnected(boolean active) throws IgniteCheckedException {
             mvccMgr,
             pageStoreMgr,
             walMgr,
+            walStateMgr,
             dbMgr,
             snpMgr,
             new GridCacheDeploymentManager(),
@@ -376,6 +383,8 @@ void onReconnected(boolean active) throws IgniteCheckedException {
         for (GridCacheSharedManager mgr : mgrs) {
             if (restartOnDisconnect(mgr))
                 mgr.start(this);
+
+            mgr.onReconnected(active);
         }
 
         kernalCtx.query().onCacheReconnect();
@@ -401,12 +410,14 @@ private boolean restartOnDisconnect(GridCacheSharedManager mgr) {
      * @param verMgr Version manager.
      * @param mvccMgr MVCC manager.
      * @param pageStoreMgr Page store manager. {@code Null} if persistence is not enabled.
+     * @param walStateMgr WAL state manager.
      * @param depMgr Deployment manager.
      * @param exchMgr Exchange manager.
      * @param affMgr Affinity manager.
      * @param ioMgr IO manager.
      * @param ttlMgr Ttl cleanup manager.
      */
+    @SuppressWarnings("unchecked")
     private void setManagers(List> mgrs,
         IgniteTxManager txMgr,
         CacheJtaManagerAdapter jtaMgr,
@@ -414,6 +425,7 @@ private void setManagers(List> mgrs,
         GridCacheMvccManager mvccMgr,
         @Nullable IgnitePageStoreManager pageStoreMgr,
         IgniteWriteAheadLogManager walMgr,
+        WalStateManager walStateMgr,
         IgniteCacheDatabaseSharedManager dbMgr,
         IgniteCacheSnapshotManager snpMgr,
         GridCacheDeploymentManager depMgr,
@@ -426,6 +438,7 @@ private void setManagers(List> mgrs,
         this.txMgr = add(mgrs, txMgr);
         this.pageStoreMgr = add(mgrs, pageStoreMgr);
         this.walMgr = add(mgrs, walMgr);
+        this.walStateMgr = add(mgrs, walStateMgr);
         this.dbMgr = add(mgrs, dbMgr);
         this.snpMgr = add(mgrs, snpMgr);
         this.jtaMgr = add(mgrs, jtaMgr);
@@ -683,6 +696,13 @@ public IgniteWriteAheadLogManager wal() {
         return walMgr;
     }
 
+    /**
+     * @return WAL state manager.
+     */
+    public WalStateManager walState() {
+       return walStateMgr;
+    }
+
     /**
      * @return IO manager.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManager.java
index bc1bbb97b9e43..214c5a18d6b62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManager.java
@@ -49,6 +49,11 @@ public interface GridCacheSharedManager {
      */
     public void onDisconnected(IgniteFuture reconnectFut);
 
+    /**
+     * @param active Active flag.
+     */
+    public void onReconnected(boolean active);
+
     /**
      * Prints memory statistics (sizes of internal data structures, etc.).
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManagerAdapter.java
index 90ae6702b703d..693d26fc46ebc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManagerAdapter.java
@@ -135,6 +135,11 @@ protected void onKernalStop0(boolean cancel) {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void onReconnected(boolean active) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void printMemoryStats() {
         // No-op.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAbstractMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAbstractMessage.java
new file mode 100644
index 0000000000000..752184a50d19a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAbstractMessage.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.UUID;
+
+/**
+ * WAL state change abstract message.
+ */
+public abstract class WalStateAbstractMessage implements DiscoveryCustomMessage {
+    /** Message ID */
+    private final IgniteUuid id = IgniteUuid.randomUuid();
+
+    /** Unique operation ID. */
+    private final UUID opId;
+
+    /** Group ID. */
+    private int grpId;
+
+    /** Group deployment ID. */
+    private IgniteUuid grpDepId;
+
+    /** Message that should be processed through exchange thread. */
+    @GridToStringExclude
+    private transient WalStateProposeMessage exchangeMsg;
+
+    /**
+     * Constructor.
+     *
+     * @param opId Unique operation ID.
+     * @param grpId Group ID.
+     * @param grpDepId Group deployment ID.
+     */
+    protected WalStateAbstractMessage(UUID opId, int grpId, IgniteUuid grpDepId) {
+        this.opId = opId;
+        this.grpId = grpId;
+        this.grpDepId = grpDepId;
+    }
+
+    /**
+     * @return Unique operation ID.
+     */
+    public UUID operationId() {
+        return opId;
+    }
+
+    /**
+     * @return Group ID.
+     */
+    public int groupId() {
+        return grpId;
+    }
+
+    /**
+     * @return Group deployment ID.
+     */
+    public IgniteUuid groupDeploymentId() {
+        return grpDepId;
+    }
+
+    /**
+     * @return {@code True} if exchange is needed.
+     */
+    public boolean needExchange() {
+        return exchangeMsg != null;
+    }
+
+    /**
+     * Get exchange message.
+     *
+     * @return Massage or {@code null} if no processing is required.
+     */
+    @Nullable public WalStateProposeMessage exchangeMessage() {
+        return exchangeMsg;
+    }
+
+    /**
+     * Set message that will be processed through exchange thread later on.
+     *
+     * @param exchangeMsg Message.
+     */
+    public void exchangeMessage(WalStateProposeMessage exchangeMsg) {
+        this.exchangeMsg = exchangeMsg;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteUuid id() {
+        return id;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public DiscoveryCustomMessage ackMessage() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMutable() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
+        DiscoCache discoCache) {
+        return discoCache.copy(topVer, null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(WalStateAbstractMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java
new file mode 100644
index 0000000000000..7c241068231bb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.query.schema.message.SchemaOperationStatusMessage;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+/**
+ * WAL state ack message (sent from participant node to coordinator).
+ */
+public class WalStateAckMessage implements Message {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Operation ID. */
+    private UUID opId;
+
+    /** Affinity node flag. */
+    private boolean affNode;
+
+    /** Operation result. */
+    private boolean changed;
+
+    /** Error message. */
+    private String errMsg;
+
+    /** Sender node ID. */
+    @GridDirectTransient
+    private UUID sndNodeId;
+
+    /**
+     * Default constructor.
+     */
+    public WalStateAckMessage() {
+        // No-op.
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param opId Operation ID.
+     * @param affNode Affinity node.
+     * @param changed Operation result.
+     * @param errMsg Error message.
+     */
+    public WalStateAckMessage(UUID opId, boolean affNode, boolean changed, @Nullable String errMsg) {
+        this.opId = opId;
+        this.affNode = affNode;
+        this.changed = changed;
+        this.errMsg = errMsg;
+    }
+
+    /**
+     * @return Operation ID.
+     */
+    public UUID operationId() {
+        return opId;
+    }
+
+    /**
+     * @return Affinity node flag.
+     */
+    public boolean affNode() {
+        return affNode;
+    }
+
+    /**
+     * @return Result.
+     */
+    public boolean changed() {
+        return changed;
+    }
+
+    /**
+     * @return Error message.
+     */
+    @Nullable public String errorMessage() {
+        return errMsg;
+    }
+
+    /**
+     * @return Sender node ID.
+     */
+    public UUID senderNodeId() {
+        return sndNodeId;
+    }
+
+    /**
+     * @param sndNodeId Sender node ID.
+     */
+    public void senderNodeId(UUID sndNodeId) {
+        this.sndNodeId = sndNodeId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeUuid("opId", opId))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeBoolean("affNode", affNode))
+                    return false;
+
+                writer.incrementState();
+
+            case 2:
+                if (!writer.writeBoolean("changed", changed))
+                    return false;
+
+                writer.incrementState();
+
+            case 3:
+                if (!writer.writeString("errMsg", errMsg))
+                    return false;
+
+                writer.incrementState();
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                opId = reader.readUuid("opId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                affNode = reader.readBoolean("affNode");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 2:
+                changed = reader.readBoolean("changed");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 3:
+                errMsg = reader.readString("errMsg");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+        }
+
+        return reader.afterMessageRead(SchemaOperationStatusMessage.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public short directType() {
+        return 129;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 4;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(WalStateAckMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateDistributedProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateDistributedProcess.java
new file mode 100644
index 0000000000000..db0dbb54625af
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateDistributedProcess.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Distributed process governing WAL state change.
+ */
+public class WalStateDistributedProcess {
+    /** Original propose message. */
+    private final WalStateProposeMessage msg;
+
+    /** Remaining nodes. */
+    @GridToStringInclude
+    private final Collection remainingNodes;
+
+    /** Acks. */
+    private final Map acks;
+
+    /**
+     * Constructor.
+     *
+     * @param msg Original propose message.
+     * @param remainingNodes Remaining nodes.
+     */
+    public WalStateDistributedProcess(WalStateProposeMessage msg, Collection remainingNodes) {
+        assert !F.isEmpty(remainingNodes);
+
+        this.msg = msg;
+        this.remainingNodes = remainingNodes;
+
+        acks = new HashMap<>(remainingNodes.size());
+    }
+
+    /**
+     * Handle node finish.
+     *
+     * @param nodeId Node ID.
+     * @param ack Ack message.
+     */
+    public void onNodeFinished(UUID nodeId, WalStateAckMessage ack) {
+        remainingNodes.remove(nodeId);
+
+        // Log only messages from affinity nodes. Non-affinity nodes are of no interest for us since they do not have
+        // group context in general case and hence we cannot tell fo sure whether anything was changed or not.
+        if (ack.affNode())
+            acks.put(nodeId, ack);
+    }
+
+    /**
+     * Handle node leave.
+     *
+     * @param nodeId Node ID.
+     */
+    public void onNodeLeft(UUID nodeId) {
+        remainingNodes.remove(nodeId);
+    }
+
+    /**
+     * @return {@code True} if process is completed.
+     */
+    public boolean completed() {
+        return remainingNodes.isEmpty();
+    }
+
+    /**
+     * @return Operation ID.
+     */
+    public UUID operationId() {
+        return msg.operationId();
+    }
+
+    /**
+     * Prepare finish message based on obtained results.
+     *
+     * @return Message.
+     */
+    public WalStateFinishMessage prepareFinishMessage() {
+        assert completed();
+
+        if (acks.isEmpty()) {
+            // We haven't received any response from affinity nodes. Result is unknown, so throw an exception.
+            return new WalStateFinishMessage(msg.operationId(), msg.groupId(), msg.groupDeploymentId(), false,
+                "Operation result is unknown because all affinity nodes have left the grid.");
+        }
+
+        Map errs = new HashMap<>();
+
+        // Look for errors first.
+        for (Map.Entry ackEntry : acks.entrySet()) {
+            UUID nodeId = ackEntry.getKey();
+            WalStateAckMessage ackMsg = ackEntry.getValue();
+
+            assert ackMsg.affNode();
+
+            if (ackMsg.errorMessage() != null)
+                errs.put(nodeId, ackMsg.errorMessage());
+        }
+
+        if (!errs.isEmpty()) {
+            SB errMsg =
+                new SB("Operation failed on some nodes (please consult to node logs for more information) [");
+
+            boolean first = true;
+
+            for (Map.Entry err : errs.entrySet()) {
+                if (first)
+                    first = false;
+                else
+                    errMsg.a(", ");
+
+                errMsg.a("[nodeId=" + err.getKey() + ", err=" + err.getValue() + ']');
+            }
+
+            errMsg.a(']');
+
+            return new WalStateFinishMessage(msg.operationId(), msg.groupId(), msg.groupDeploymentId(), false,
+                errMsg.toString());
+        }
+
+        // Verify results consistency. Note that non-affinity node are not present at this stage.
+        Boolean changed = null;
+
+        for (WalStateAckMessage ackMsg : acks.values()) {
+            boolean curChanged = ackMsg.changed();
+
+            if (changed == null)
+                changed = curChanged;
+            else {
+                if (!F.eq(curChanged, changed)) {
+                    return new WalStateFinishMessage(msg.operationId(), msg.groupId(), msg.groupDeploymentId(),
+                        false, "Operation result is unknown because nodes reported different results (please " +
+                        "re-try operation).");
+                }
+            }
+        }
+
+        // All nodes completed operation with the same result, complete with success.
+        assert changed != null;
+
+        return new WalStateFinishMessage(msg.operationId(), msg.groupId(), msg.groupDeploymentId(), changed, null);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(WalStateDistributedProcess.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
new file mode 100644
index 0000000000000..57f25d0e80ca1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.UUID;
+
+/**
+ * WAL state finish message.
+ */
+public class WalStateFinishMessage extends WalStateAbstractMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Whether WAL state was changed as a result of this call. */
+    private final boolean changed;
+
+    /** Error message. */
+    private final String errMsg;
+
+    /**
+     * Constructor.
+     *
+     * @param opId Unique operation ID.
+     * @param grpId Group ID.
+     * @param grpDepId Group deployment ID.
+     * @param changed Result.
+     * @param errMsg Error message.
+     */
+    public WalStateFinishMessage(UUID opId, int grpId, IgniteUuid grpDepId, boolean changed, @Nullable String errMsg) {
+        super(opId, grpId, grpDepId);
+
+        this.changed = changed;
+        this.errMsg = errMsg;
+    }
+
+    /**
+     * @return Result.
+     */
+    public boolean changed() {
+        return changed;
+    }
+
+    /**
+     * @return Error message.
+     */
+    @Nullable public String errorMessage() {
+        return errMsg;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(WalStateFinishMessage.class, this, "super", super.toString());
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
new file mode 100644
index 0000000000000..0ac699f5a8fee
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -0,0 +1,914 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.communication.GridMessageListener;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointFuture;
+import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.thread.IgniteThread;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.ignite.internal.GridTopic.TOPIC_WAL;
+import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+
+/**
+ * Write-ahead log state manager. Manages WAL enable and disable.
+ */
+public class WalStateManager extends GridCacheSharedManagerAdapter {
+    /** History size for to track stale messages. */
+    private static final int HIST_SIZE = 1000;
+
+    /** ID history for discovery messages. */
+    private final GridBoundedConcurrentLinkedHashSet> discoMsgIdHist =
+        new GridBoundedConcurrentLinkedHashSet<>(HIST_SIZE);
+
+    /** History of already completed operations. */
+    private final GridBoundedConcurrentLinkedHashSet completedOpIds =
+        new GridBoundedConcurrentLinkedHashSet<>(HIST_SIZE);
+
+    /** Client futures. */
+    private final Map> userFuts = new HashMap<>();
+
+    /** Finished results awaiting discovery finish message. */
+    private final Map ress = new HashMap<>();
+
+    /** Active distributed processes. */
+    private final Map procs = new HashMap<>();
+
+    /** Pending results created on cache processor start based on available discovery data. */
+    private final Collection initialRess = new LinkedList<>();
+
+    /** Pending acknowledge messages (i.e. received before node completed it's local part). */
+    private final Collection pendingAcks = new HashSet<>();
+
+    /** Whether this is a server node. */
+    private final boolean srv;
+
+    /** IO message listener. */
+    private final GridMessageListener ioLsnr;
+
+    /** Operation mutex. */
+    private final Object mux = new Object();
+
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /** Current coordinator node. */
+    private ClusterNode crdNode;
+
+    /** Disconnected flag. */
+    private boolean disconnected;
+
+    /**
+     * Constructor.
+     *
+     * @param kernalCtx Kernal context.
+     */
+    public WalStateManager(GridKernalContext kernalCtx) {
+        if (kernalCtx != null) {
+            IgniteConfiguration cfg = kernalCtx.config();
+
+            srv = !cfg.isClientMode() && !cfg.isDaemon();
+
+            log = kernalCtx.log(WalStateManager.class);
+        }
+        else {
+            srv = false;
+
+            log = null;
+        }
+
+        if (srv) {
+            ioLsnr = new GridMessageListener() {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
+                    if (msg instanceof WalStateAckMessage) {
+                        WalStateAckMessage msg0 = (WalStateAckMessage) msg;
+
+                        msg0.senderNodeId(nodeId);
+
+                        onAck(msg0);
+                    }
+                    else
+                        U.warn(log, "Unexpected IO message (will ignore): " + msg);
+                }
+            };
+        }
+        else
+            ioLsnr = null;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void start0() throws IgniteCheckedException {
+        if (srv)
+            cctx.kernalContext().io().addMessageListener(TOPIC_WAL, ioLsnr);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void stop0(boolean cancel) {
+        if (srv)
+            cctx.kernalContext().io().removeMessageListener(TOPIC_WAL, ioLsnr);
+    }
+
+    /**
+     * Callback invoked when caches info is collected inside cache processor start routine. Discovery is not
+     * active at this point.
+     */
+    public void onCachesInfoCollected() {
+        if (!srv)
+            return;
+
+        synchronized (mux) {
+            // Process top pending requests.
+            for (CacheGroupDescriptor grpDesc : cacheProcessor().cacheGroupDescriptors().values()) {
+                WalStateProposeMessage msg = grpDesc.nextWalChangeRequest();
+
+                if (msg != null) {
+                    if (log.isDebugEnabled())
+                        log.debug("Processing WAL state message on start: " + msg);
+
+                    boolean enabled = grpDesc.walEnabled();
+
+                    WalStateResult res;
+
+                    if (F.eq(enabled, msg.enable()))
+                        res = new WalStateResult(msg, false);
+                    else {
+                        res = new WalStateResult(msg, true);
+
+                        grpDesc.walEnabled(!enabled);
+                    }
+
+                    initialRess.add(res);
+
+                    addResult(res);
+                }
+            }
+        }
+    }
+
+    /**
+     * Handle cache processor kernal start. At this point we already collected discovery data from other nodes
+     * (discovery already active), but exchange worker is not active yet. We need to iterate over available group
+     * descriptors and perform top operations, taking in count that no cache operations are possible at this point,
+     * so checkpoint is not needed.
+     */
+    public void onKernalStart() {
+        if (!srv)
+            return;
+
+        synchronized (mux) {
+            for (WalStateResult res : initialRess)
+                onCompletedLocally(res);
+
+            initialRess.clear();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onDisconnected(IgniteFuture reconnectFut) {
+        Collection> userFuts0;
+
+        synchronized (mux) {
+            assert !disconnected;
+
+            disconnected = true;
+
+            userFuts0 = new ArrayList<>(userFuts.values());
+
+            userFuts.clear();
+        }
+
+        for (GridFutureAdapter userFut : userFuts0)
+            completeWithError(userFut, "Client node was disconnected from topology (operation result is unknown).");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onReconnected(boolean active) {
+        synchronized (mux) {
+            assert disconnected;
+
+            disconnected = false;
+        }
+    }
+
+    /**
+     * Initiate WAL mode change operation.
+     *
+     * @param cacheNames Cache names.
+     * @param enabled Enabled flag.
+     * @return Future completed when operation finished.
+     */
+    public IgniteInternalFuture init(Collection cacheNames, boolean enabled) {
+        if (F.isEmpty(cacheNames))
+            return errorFuture("Cache names cannot be empty.");
+
+        synchronized (mux) {
+            if (disconnected)
+                return errorFuture("Failed to initiate WAL mode change because client node is disconnected.");
+
+            // Prepare cache and group infos.
+            Map caches = new HashMap<>(cacheNames.size());
+
+            CacheGroupDescriptor grpDesc = null;
+
+            for (String cacheName : cacheNames) {
+                DynamicCacheDescriptor cacheDesc = cacheProcessor().cacheDescriptor(cacheName);
+
+                if (cacheDesc == null)
+                    return errorFuture("Cache doesn't exist: " + cacheName);
+
+                caches.put(cacheName, cacheDesc.deploymentId());
+
+                CacheGroupDescriptor curGrpDesc = cacheDesc.groupDescriptor();
+
+                if (grpDesc == null)
+                    grpDesc = curGrpDesc;
+                else if (!F.eq(grpDesc.deploymentId(), curGrpDesc.deploymentId())) {
+                    return errorFuture("Cannot change WAL mode for caches from different cache groups [" +
+                        "cache1=" + cacheNames.iterator().next() + ", grp1=" + grpDesc.groupName() +
+                        ", cache2=" + cacheName + ", grp2=" + curGrpDesc.groupName() + ']');
+                }
+            }
+
+            assert grpDesc != null;
+
+            HashSet grpCaches = new HashSet<>(grpDesc.caches().keySet());
+
+            grpCaches.removeAll(cacheNames);
+
+            if (!grpCaches.isEmpty()) {
+                return errorFuture("Cannot change WAL mode because not all cache names belonging to the group are " +
+                    "provided [group=" + grpDesc.groupName() + ", missingCaches=" + grpCaches + ']');
+            }
+
+            if (grpDesc.config().getCacheMode() == CacheMode.LOCAL)
+                return errorFuture("WAL mode cannot be changed for LOCAL cache(s): " + cacheNames);
+
+            // WAL mode change makes sense only for persistent groups.
+            if (!grpDesc.persistenceEnabled())
+                return errorFuture("Cannot change WAL mode because persistence is not enabled for cache(s) [" +
+                    "caches=" + cacheNames + ", dataRegion=" + grpDesc.config().getDataRegionName() + ']');
+
+            // Send request.
+            final UUID opId = UUID.randomUUID();
+
+            GridFutureAdapter fut = new GridFutureAdapter<>();
+
+            fut.listen(new IgniteInClosure>() {
+                @Override public void apply(IgniteInternalFuture fut) {
+                    synchronized (mux) {
+                        userFuts.remove(opId);
+                    }
+                }
+            });
+
+            WalStateProposeMessage msg = new WalStateProposeMessage(opId, grpDesc.groupId(), grpDesc.deploymentId(),
+                cctx.localNodeId(), caches, enabled);
+
+            userFuts.put(opId, fut);
+
+            try {
+                cctx.discovery().sendCustomEvent(msg);
+
+                if (log.isDebugEnabled())
+                    log.debug("Initiated WAL state change operation: " + msg);
+            }
+            catch (Exception e) {
+                IgniteCheckedException e0 =
+                    new IgniteCheckedException("Failed to initiate WAL mode change due to unexpected exception.", e);
+
+                fut.onDone(e0);
+            }
+
+            return fut;
+        }
+    }
+
+    /**
+     * Handle propose message in discovery thread.
+     *
+     * @param msg Message.
+     */
+    public void onProposeDiscovery(WalStateProposeMessage msg) {
+        if (isDuplicate(msg))
+            return;
+
+        synchronized (mux) {
+            if (disconnected)
+                return;
+
+            // Validate current caches state before deciding whether to process message further.
+            if (validateProposeDiscovery(msg)) {
+                if (log.isDebugEnabled())
+                    log.debug("WAL state change message is valid (will continue processing): " + msg);
+
+                CacheGroupDescriptor grpDesc = cacheProcessor().cacheGroupDescriptors().get(msg.groupId());
+
+                assert grpDesc != null;
+
+                IgnitePredicate nodeFilter = grpDesc.config().getNodeFilter();
+
+                boolean affNode = srv && (nodeFilter == null || nodeFilter.apply(cctx.localNode()));
+
+                msg.affinityNode(affNode);
+
+                if (grpDesc.addWalChangeRequest(msg)) {
+                    msg.exchangeMessage(msg);
+
+                    if (log.isDebugEnabled())
+                        log.debug("WAL state change message will be processed in exchange thread: " + msg);
+                }
+                else {
+                    if (log.isDebugEnabled())
+                        log.debug("WAL state change message is added to pending set and will be processed later: " +
+                            msg);
+                }
+            }
+            else {
+                if (log.isDebugEnabled())
+                    log.debug("WAL state change message is invalid (will ignore): " + msg);
+            }
+        }
+    }
+
+    /**
+     * Validate propose message.
+     *
+     * @param msg Message.
+     * @return {@code True} if message should be processed further, {@code false} if no further processing is needed.
+     */
+    private boolean validateProposeDiscovery(WalStateProposeMessage msg) {
+        GridFutureAdapter userFut = userFuts.get(msg.operationId());
+
+        String errMsg = validate(msg);
+
+        if (errMsg != null) {
+            completeWithError(userFut, errMsg);
+
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Validate propose message.
+     *
+     * @param msg Message.
+     * @return Error message or {@code null} if everything is OK.
+     */
+    @Nullable private String validate(WalStateProposeMessage msg) {
+        // Is group still there?
+        CacheGroupDescriptor grpDesc = cacheProcessor().cacheGroupDescriptors().get(msg.groupId());
+
+        if (grpDesc == null)
+            return "Failed to change WAL mode because some caches no longer exist: " + msg.caches().keySet();
+
+        // Are specified caches still there?
+        for (Map.Entry cache : msg.caches().entrySet()) {
+            String cacheName = cache.getKey();
+
+            DynamicCacheDescriptor cacheDesc = cacheProcessor().cacheDescriptor(cacheName);
+
+            if (cacheDesc == null || !F.eq(cacheDesc.deploymentId(), cache.getValue()))
+                return "Cache doesn't exist: " + cacheName;
+        }
+
+        // Are there any new caches in the group?
+        HashSet grpCacheNames = new HashSet<>(grpDesc.caches().keySet());
+
+        grpCacheNames.removeAll(msg.caches().keySet());
+
+        if (!grpCacheNames.isEmpty()) {
+            return "Cannot change WAL mode because not all cache names belonging to the " +
+                "group are provided [group=" + grpDesc.groupName() + ", missingCaches=" + grpCacheNames + ']';
+        }
+
+        return null;
+    }
+
+    /**
+     * Handle propose message which is synchronized with other cache state actions through exchange thread.
+     * If operation is no-op (i.e. state is not changed), then no additional processing is needed, and coordinator will
+     * trigger finish request right away. Otherwise all nodes start asynchronous checkpoint flush, and send responses
+     * to coordinator. Once all responses are received, coordinator node will trigger finish message.
+     *
+     * @param msg Message.
+     */
+    public void onProposeExchange(WalStateProposeMessage msg) {
+        if (!srv)
+            return;
+
+        synchronized (mux) {
+            WalStateResult res = null;
+
+            if (msg.affinityNode()) {
+                // Affinity node, normal processing.
+                CacheGroupContext grpCtx = cacheProcessor().cacheGroup(msg.groupId());
+
+                if (grpCtx == null) {
+                    // Related caches were destroyed concurrently.
+                    res = new WalStateResult(msg, "Failed to change WAL mode because some caches " +
+                        "no longer exist: " + msg.caches().keySet());
+                }
+                else {
+                    if (F.eq(msg.enable(), grpCtx.walEnabled()))
+                        // Nothing changed -> no-op.
+                        res = new WalStateResult(msg, false);
+                    else {
+                        // Initiate a checkpoint.
+                        CheckpointFuture cpFut = triggerCheckpoint(msg.groupId());
+
+                        if (cpFut != null) {
+                            try {
+                                // Wait for checkpoint mark synchronously before releasing the control.
+                                cpFut.beginFuture().get();
+
+                                if (msg.enable()) {
+                                    grpCtx.walEnabled(true);
+
+                                    // Enable: it is enough to release cache operations once mark is finished because
+                                    // not-yet-flushed dirty pages have been logged.
+                                    WalStateChangeWorker worker = new WalStateChangeWorker(msg, cpFut);
+
+                                    new IgniteThread(worker).start();
+                                }
+                                else {
+                                    // Disable: not-yet-flushed operations are not logged, so wait for them
+                                    // synchronously in exchange thread. Otherwise, we cannot define a point in
+                                    // when it is safe to continue cache operations.
+                                    res = awaitCheckpoint(cpFut, msg);
+
+                                    // WAL state is persisted after checkpoint if finished. Otherwise in case of crash
+                                    // and restart we will think that WAL is enabled, but data might be corrupted.
+                                    grpCtx.walEnabled(false);
+                                }
+                            }
+                            catch (Exception e) {
+                                U.warn(log, "Failed to change WAL mode due to unexpected exception [" +
+                                    "msg=" + msg + ']', e);
+
+                                res = new WalStateResult(msg, "Failed to change WAL mode due to unexpected " +
+                                    "exception (see server logs for more information): " + e.getMessage());
+                            }
+                        }
+                        else {
+                            res = new WalStateResult(msg, "Failed to initiate a checkpoint (checkpoint thread " +
+                                "is not available).");
+                        }
+                    }
+                }
+            }
+            else {
+                // We cannot know result on non-affinity server node, so just complete operation with "false" flag,
+                // which will be ignored anyway.
+                res = new WalStateResult(msg, false);
+            }
+
+            if (res != null) {
+                addResult(res);
+
+                onCompletedLocally(res);
+            }
+        }
+    }
+
+    /**
+     * Handle local operation completion.
+     *
+     * @param res Result.
+     */
+    private void onCompletedLocally(WalStateResult res) {
+        assert res != null;
+
+        synchronized (mux) {
+            ClusterNode crdNode = coordinator();
+
+            UUID opId = res.message().operationId();
+
+            WalStateAckMessage msg = new WalStateAckMessage(opId, res.message().affinityNode(),
+                res.changed(), res.errorMessage());
+
+            // Handle distributed completion.
+            if (crdNode.isLocal()) {
+                Collection srvNodes = cctx.discovery().aliveServerNodes();
+
+                Collection srvNodeIds = new ArrayList<>(srvNodes.size());
+
+                for (ClusterNode srvNode : srvNodes) {
+                    if (cctx.discovery().alive(srvNode))
+                        srvNodeIds.add(srvNode.id());
+                }
+
+                WalStateDistributedProcess proc = new WalStateDistributedProcess(res.message(), srvNodeIds);
+
+                procs.put(res.message().operationId(), proc);
+
+                unwindPendingAcks(proc);
+
+                proc.onNodeFinished(cctx.localNodeId(), msg);
+
+                sendFinishMessageIfNeeded(proc);
+            }
+            else {
+                // Just send message to coordinator.
+                try {
+                    cctx.kernalContext().io().sendToGridTopic(crdNode, TOPIC_WAL, msg, SYSTEM_POOL);
+                }
+                catch (IgniteCheckedException e) {
+                    U.warn(log, "Failed to send ack message to coordinator node [opId=" + opId +
+                        ", node=" + crdNode.id() + ']');
+                }
+            }
+        }
+    }
+
+    /**
+     * Unwind pending ack messages for the given distributed process.
+     *
+     * @param proc Process.
+     */
+    private void unwindPendingAcks(WalStateDistributedProcess proc) {
+        assert Thread.holdsLock(mux);
+
+        Iterator iter = pendingAcks.iterator();
+
+        while (iter.hasNext()) {
+            WalStateAckMessage ackMsg = iter.next();
+
+            if (F.eq(proc.operationId(), ackMsg.operationId())) {
+                proc.onNodeFinished(ackMsg.senderNodeId(), ackMsg);
+
+                iter.remove();
+            }
+        }
+    }
+
+    /**
+     * Handle ack message.
+     *
+     * @param msg Ack message.
+     */
+    public void onAck(WalStateAckMessage msg) {
+        synchronized (mux) {
+            if (completedOpIds.contains(msg.operationId()))
+                // Skip stale messages.
+                return;
+
+            WalStateDistributedProcess proc = procs.get(msg.operationId());
+
+            if (proc == null)
+                // If process if not initialized yet, add to pending set.
+                pendingAcks.add(msg);
+            else {
+                // Notify process on node completion.
+                proc.onNodeFinished(msg.senderNodeId(), msg);
+
+                sendFinishMessageIfNeeded(proc);
+            }
+        }
+    }
+
+    /**
+     * Send finish message for the given distributed process if needed.
+     *
+     * @param proc Process.
+     */
+    private void sendFinishMessageIfNeeded(WalStateDistributedProcess proc) {
+        if (proc.completed())
+            sendFinishMessage(proc.prepareFinishMessage());
+    }
+
+    /**
+     * Send finish message.
+     *
+     * @param finishMsg Finish message.
+     */
+    private void sendFinishMessage(WalStateFinishMessage finishMsg) {
+        try {
+            cctx.discovery().sendCustomEvent(finishMsg);
+        }
+        catch (Exception e) {
+            U.error(log, "Failed to send WAL mode change finish message due to unexpected exception: " + finishMsg, e);
+        }
+    }
+
+    /**
+     * Handle finish message in discovery thread.
+     *
+     * @param msg Message.
+     */
+    public void onFinishDiscovery(WalStateFinishMessage msg) {
+        if (isDuplicate(msg))
+            return;
+
+        synchronized (mux) {
+            if (disconnected)
+                return;
+
+            // Complete user future, if any.
+            GridFutureAdapter userFut = userFuts.get(msg.operationId());
+
+            if (userFut != null) {
+                if (msg.errorMessage() != null)
+                    completeWithError(userFut, msg.errorMessage());
+                else
+                    complete(userFut, msg.changed());
+            }
+
+            // Clear pending data.
+            WalStateResult res = ress.remove(msg.operationId());
+
+            if (res == null && srv)
+                U.warn(log, "Received finish message for unknown operation (will ignore): " + msg.operationId());
+
+            procs.remove(msg.operationId());
+
+            CacheGroupDescriptor grpDesc = cacheProcessor().cacheGroupDescriptors().get(msg.groupId());
+
+            if (grpDesc != null && F.eq(grpDesc.deploymentId(), msg.groupDeploymentId())) {
+                // Toggle WAL mode in descriptor.
+                if (msg.changed())
+                    grpDesc.walEnabled(!grpDesc.walEnabled());
+
+                // Remove now-outdated message from the queue.
+                WalStateProposeMessage oldProposeMsg = grpDesc.nextWalChangeRequest();
+
+                assert oldProposeMsg != null;
+                assert F.eq(oldProposeMsg.operationId(), msg.operationId());
+
+                grpDesc.removeWalChangeRequest();
+
+                // Move next message to exchange thread.
+                WalStateProposeMessage nextProposeMsg = grpDesc.nextWalChangeRequest();
+
+                if (nextProposeMsg != null)
+                    msg.exchangeMessage(nextProposeMsg);
+            }
+
+            if (srv) {
+                // Remember operation ID to handle duplicates.
+                completedOpIds.add(msg.operationId());
+
+                // Remove possible stale messages.
+                Iterator ackIter = pendingAcks.iterator();
+
+                while (ackIter.hasNext()) {
+                    WalStateAckMessage ackMsg = ackIter.next();
+
+                    if (F.eq(ackMsg.operationId(), msg.operationId()))
+                        ackIter.remove();
+                }
+            }
+        }
+    }
+
+    /**
+     * Handle node leave event.
+     *
+     * @param nodeId Node ID.
+     */
+    public void onNodeLeft(UUID nodeId) {
+        if (!srv)
+            return;
+
+        synchronized (mux) {
+            if (crdNode == null) {
+                assert ress.isEmpty();
+                assert procs.isEmpty();
+
+                return;
+            }
+
+            if (F.eq(crdNode.id(), nodeId)) {
+                // Coordinator exited, re-send to new, or initialize new distirbuted processes.
+                crdNode = null;
+
+                for (WalStateResult res : ress.values())
+                    onCompletedLocally(res);
+            }
+            else if (F.eq(cctx.localNodeId(), crdNode.id())) {
+                // Notify distributed processes on node leave.
+                for (Map.Entry procEntry : procs.entrySet()) {
+                    WalStateDistributedProcess proc = procEntry.getValue();
+
+                    proc.onNodeLeft(nodeId);
+
+                    sendFinishMessageIfNeeded(proc);
+                }
+            }
+        }
+    }
+
+    /**
+     * Create future with error.
+     *
+     * @param errMsg Error message.
+     * @return Future.
+     */
+    @SuppressWarnings("Convert2Diamond")
+    private static IgniteInternalFuture errorFuture(String errMsg) {
+        return new GridFinishedFuture(new IgniteCheckedException(errMsg));
+    }
+
+    /**
+     * Complete user future with normal result.
+     *
+     * @param userFut User future.
+     * @param res Result.
+     */
+    private static void complete(@Nullable GridFutureAdapter userFut, boolean res) {
+        if (userFut != null)
+            userFut.onDone(res);
+    }
+
+    /**
+     * Complete user future with error.
+     *
+     * @param errMsg Error message.
+     */
+    private static void completeWithError(@Nullable GridFutureAdapter userFut, String errMsg) {
+        if (userFut != null)
+            userFut.onDone(new IgniteCheckedException(errMsg));
+    }
+
+    /**
+     * @return Cache processor.
+     */
+    private GridCacheProcessor cacheProcessor() {
+        return cctx.cache();
+    }
+
+    /**
+     * Get current coordinator node.
+     *
+     * @return Coordinator node.
+     */
+    private ClusterNode coordinator() {
+        assert Thread.holdsLock(mux);
+
+        if (crdNode != null)
+            return crdNode;
+        else {
+            ClusterNode res = null;
+
+            for (ClusterNode node : cctx.discovery().aliveServerNodes()) {
+                if (res == null || res.order() > node.order())
+                    res = node;
+            }
+
+            assert res != null;
+
+            crdNode = res;
+
+            return res;
+        }
+    }
+
+    /**
+     * Check if discovery message has already been received.
+     *
+     * @param msg Message.
+     * @return {@code True} if this is a duplicate.
+     */
+    private boolean isDuplicate(WalStateAbstractMessage msg) {
+        T2 key;
+
+        if (msg instanceof WalStateProposeMessage)
+            key = new T2<>(msg.operationId(), true);
+        else {
+            assert msg instanceof WalStateFinishMessage;
+
+            key = new T2<>(msg.operationId(), false);
+        }
+
+        if (!discoMsgIdHist.add(key)) {
+            U.warn(log, "Received duplicate WAL mode change discovery message (will ignore): " + msg);
+
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Add locally result to pending map.
+     *
+     * @param res Result.
+     */
+    private void addResult(WalStateResult res) {
+        ress.put(res.message().operationId(), res);
+    }
+
+    /**
+     * Force checkpoint.
+     *
+     * @param grpId Group ID.
+     * @return Checkpoint future or {@code null} if failed to get checkpointer.
+     */
+    @Nullable private CheckpointFuture triggerCheckpoint(int grpId) {
+        return cctx.database().forceCheckpoint("wal-state-change-grp-" + grpId);
+    }
+
+    /**
+     * Await for the checkpoint to finish.
+     *
+     * @param cpFut Checkpoint future.
+     * @param msg Orignial message which triggered the process.
+     * @return Result.
+     */
+    private WalStateResult awaitCheckpoint(CheckpointFuture cpFut, WalStateProposeMessage msg) {
+        WalStateResult res;
+
+        try {
+            assert msg.affinityNode();
+
+            if (cpFut != null)
+                cpFut.finishFuture().get();
+
+            res = new WalStateResult(msg, true);
+        }
+        catch (Exception e) {
+            U.warn(log, "Failed to change WAL mode due to unexpected exception [msg=" + msg + ']', e);
+
+            res = new WalStateResult(msg, "Failed to change WAL mode due to unexpected exception " +
+                "(see server logs for more information): " + e.getMessage());
+        }
+
+        return res;
+    }
+
+    /**
+     * WAL state change worker.
+     */
+    private class WalStateChangeWorker extends GridWorker {
+        /** Message. */
+        private final WalStateProposeMessage msg;
+
+        /** Checkpoint future. */
+        private final CheckpointFuture cpFut;
+
+        /**
+         * Constructor.
+         *
+         * @param msg Propose message.
+         */
+        private WalStateChangeWorker(WalStateProposeMessage msg, CheckpointFuture cpFut) {
+            super(cctx.igniteInstanceName(), "wal-state-change-worker-" + msg.groupId(), WalStateManager.this.log);
+
+            this.msg = msg;
+            this.cpFut = cpFut;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+            WalStateResult res = awaitCheckpoint(cpFut, msg);
+
+            addResult(res);
+
+            onCompletedLocally(res);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateNodeLeaveExchangeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateNodeLeaveExchangeTask.java
new file mode 100644
index 0000000000000..3ac12fc3b2ee8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateNodeLeaveExchangeTask.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Exchange task to handle node leave for WAL state manager.
+ */
+public class WalStateNodeLeaveExchangeTask implements CachePartitionExchangeWorkerTask {
+    /** Node that has left the grid. */
+    private final ClusterNode node;
+
+    /**
+     * Constructor.
+     *
+     * @param node Node that has left the grid.
+     */
+    public WalStateNodeLeaveExchangeTask(ClusterNode node) {
+        assert node != null;
+
+        this.node = node;
+    }
+
+    /**
+     * @return Node that has left the grid.
+     */
+    public ClusterNode node() {
+        return node;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean skipForExchangeMerge() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(WalStateNodeLeaveExchangeTask.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
new file mode 100644
index 0000000000000..747fd6af0a3af
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * WAL state propose message.
+ */
+public class WalStateProposeMessage extends WalStateAbstractMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Node ID. */
+    private final UUID nodeId;
+
+    /** Cache names which are expected to be in the group along with their deployment IDs. */
+    private Map caches;
+
+    /** Whether WAL should be enabled or disabled. */
+    private final boolean enable;
+
+    /** Whether message is being handled on cache affinity node. */
+    private transient boolean affNode;
+
+    /**
+     * Constructor.
+     *
+     * @param opId Operation IDs.
+     * @param grpId Expected group ID.
+     * @param grpDepId Expected group deployment ID.
+     * @param nodeId Node ID.
+     * @param caches Expected cache names and their relevant deployment IDs.
+     *
+     * @param enable WAL state flag.
+     */
+    public WalStateProposeMessage(UUID opId, int grpId, IgniteUuid grpDepId, UUID nodeId,
+        Map caches, boolean enable) {
+        super(opId, grpId, grpDepId);
+
+        this.nodeId = nodeId;
+        this.caches = caches;
+        this.enable = enable;
+    }
+
+    /**
+     * @return Node ID.
+     */
+    public UUID nodeId() {
+        return nodeId;
+    }
+
+    /**
+     * @return Caches.
+     */
+    public Map caches() {
+        return caches;
+    }
+
+    /**
+     * @return WAL state flag.
+     */
+    public boolean enable() {
+        return enable;
+    }
+
+    /**
+     * @return Whether message is being handled on cache affintiy node.
+     */
+    public boolean affinityNode() {
+        return affNode;
+    }
+
+    /**
+     * @param affNode Whether message is being handled on cache affintiy node.
+     */
+    public void affinityNode(boolean affNode) {
+        this.affNode = affNode;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(WalStateProposeMessage.class, this, "super", super.toString());
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateResult.java
new file mode 100644
index 0000000000000..a2bd0afe9497d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateResult.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Local WAL state change result.
+ */
+public class WalStateResult {
+    /** Original message. */
+    private final WalStateProposeMessage msg;
+
+    /** Whether mode was changed. */
+    private final boolean changed;
+
+    /** Error message (if any). */
+    private final String errMsg;
+
+    /**
+     * Constructor.
+     *
+     * @param msg Original message.
+     * @param changed Whether mode was changed.
+     */
+    public WalStateResult(WalStateProposeMessage msg, boolean changed) {
+        this(msg, changed, null);
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param msg Original message.
+     * @param errMsg Error message (if any).
+     */
+    public WalStateResult(WalStateProposeMessage msg, String errMsg) {
+        this(msg, false, errMsg);
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param msg Original message.
+     * @param changed Whether mode was changed.
+     * @param errMsg Error message (if any).
+     */
+    private WalStateResult(WalStateProposeMessage msg, boolean changed, String errMsg) {
+        this.msg = msg;
+        this.changed = changed;
+        this.errMsg = errMsg;
+    }
+
+    /**
+     * @return Original message.
+     */
+    public WalStateProposeMessage message() {
+        return msg;
+    }
+
+    /**
+     * @return Whether mode was changed.
+     */
+    public boolean changed() {
+        return changed;
+    }
+
+    /**
+     * @return Error message (if any).
+     */
+    @Nullable public String errorMessage() {
+        return errMsg;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(WalStateResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index 57330398780fb..7407f6c8eee2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -563,7 +563,7 @@ else if (conflictCtx.isMerge()) {
 
                                             GridCacheVersion dhtVer = cached.isNear() ? writeVersion() : null;
 
-                                            if (!near() && cacheCtx.group().persistenceEnabled() &&
+                                            if (!near() && cacheCtx.group().persistenceEnabled() && cacheCtx.group().walEnabled() &&
                                                 op != NOOP && op != RELOAD && (op != READ || cctx.snapshot().needTxReadLogging())) {
                                                 if (dataEntries == null)
                                                     dataEntries = new ArrayList<>(entries.size());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index cd5a9510dc2de..e1f1d6f43ce1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -514,7 +514,7 @@ public void restoreState(GridDhtPartitionState stateToRestore) {
      * @return {@code true} if cas succeeds.
      */
     private boolean casState(long state, GridDhtPartitionState toState) {
-        if (grp.persistenceEnabled()) {
+        if (grp.persistenceEnabled() && grp.walEnabled()) {
             synchronized (this) {
                 boolean update = this.state.compareAndSet(state, setPartState(state, toState));
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index cf2c92538f1ad..883cac69f88f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
+import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
@@ -602,6 +603,8 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                 else if (msg instanceof SnapshotDiscoveryMessage) {
                     exchange = onCustomMessageNoAffinityChange(crdNode);
                 }
+                else if (msg instanceof WalStateAbstractMessage)
+                    exchange = onCustomMessageNoAffinityChange(crdNode);
                 else {
                     assert affChangeMsg != null : this;
 
@@ -1073,6 +1076,8 @@ private void distributedExchange() throws IgniteCheckedException {
             }
         }
 
+        changeWalModeIfNeeded();
+
         if (crd.isLocal()) {
             if (remaining.isEmpty())
                 onAllReceived(null);
@@ -1107,6 +1112,37 @@ private void tryToPerformLocalSnapshotOperation() {
         }
     }
 
+    /**
+     * Change WAL mode if needed.
+     */
+    private void changeWalModeIfNeeded() {
+        WalStateAbstractMessage msg = firstWalMessage();
+
+        if (msg != null)
+            cctx.walState().onProposeExchange(msg.exchangeMessage());
+    }
+
+    /**
+     * Get first message if and only if this is WAL message.
+     *
+     * @return WAL message or {@code null}.
+     */
+    @Nullable private WalStateAbstractMessage firstWalMessage() {
+        if (firstDiscoEvt != null && firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
+            DiscoveryCustomMessage customMsg = ((DiscoveryCustomEvent)firstDiscoEvt).customMessage();
+
+            if (customMsg instanceof WalStateAbstractMessage) {
+                WalStateAbstractMessage msg0 = (WalStateAbstractMessage)customMsg;
+
+                assert msg0.needExchange();
+
+                return msg0;
+            }
+        }
+
+        return null;
+    }
+
     /**
      * The main purpose of this method is to wait for all ongoing updates (transactional and atomic), initiated on
      * the previous topology version, to finish to prevent inconsistencies during rebalancing and to prevent two
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointFuture.java
new file mode 100644
index 0000000000000..1c77013d63333
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointFuture.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+
+/**
+ * Checkpoint futures.
+ */
+public interface CheckpointFuture  {
+    /**
+     * @return Begin future.
+     */
+    public GridFutureAdapter beginFuture();
+
+    /**
+     * @return Finish future.
+     */
+    public GridFutureAdapter finishFuture();
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 580fb3a268f63..85c40055d92ee 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -267,6 +267,16 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** */
     private static final String MBEAN_GROUP = "Persistent Store";
 
+    /** WAL marker prefix for meta store. */
+    private static final String WAL_KEY_PREFIX = "grp-wal-disabled-";
+
+    /** WAL marker predicate for meta store. */
+    private static final IgnitePredicate WAL_KEY_PREFIX_PRED = new IgnitePredicate() {
+        @Override public boolean apply(String key) {
+            return key.startsWith(WAL_KEY_PREFIX);
+        }
+    };
+
     /** Checkpoint thread. Needs to be volatile because it is created in exchange worker. */
     private volatile Checkpointer checkpointer;
 
@@ -351,6 +361,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** */
     private List metastorageLifecycleLsnrs;
 
+    /** Initially disabled cache groups. */
+    public Collection initiallyWalDisabledGrps;
+
     /**
      * @param ctx Kernal context.
      */
@@ -533,6 +546,8 @@ private void readMetastore() throws IgniteCheckedException {
 
                 applyLastUpdates(status, true);
 
+                initiallyWalDisabledGrps = walDisabledGroups();
+
                 notifyMetastorageReadyForRead();
             }
             finally {
@@ -1527,6 +1542,16 @@ public Map, T2> reservedForPreloading() {
         fut2.get();
     }
 
+    /** {@inheritDoc} */
+    @Override public CheckpointFuture forceCheckpoint(String reason) {
+        Checkpointer cp = checkpointer;
+
+        if (cp == null)
+            return null;
+
+        return cp.wakeupForCheckpoint(0, reason);
+    }
+
     /**
      * Tries to search for a WAL pointer for the given partition counter start.
      *
@@ -1825,6 +1850,8 @@ private WALPointer restoreMemory(CheckpointStatus status, boolean storeOnly,
         int applied = 0;
         WALPointer lastRead = null;
 
+        Collection ignoreGrps = storeOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
+
         try (WALIterator it = cctx.wal().replay(status.endPtr)) {
             while (it.hasNextX()) {
                 IgniteBiTuple tup = it.nextX();
@@ -1861,27 +1888,29 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
                             if (storeOnly && grpId != METASTORAGE_CACHE_ID)
                                 continue;
 
-                            long pageId = pageRec.fullPageId().pageId();
+                            if (!ignoreGrps.contains(grpId)) {
+                                long pageId = pageRec.fullPageId().pageId();
 
-                            PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
+                                PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
 
-                            long page = pageMem.acquirePage(grpId, pageId, true);
-
-                            try {
-                                long pageAddr = pageMem.writeLock(grpId, pageId, page);
+                                long page = pageMem.acquirePage(grpId, pageId, true);
 
                                 try {
-                                    PageUtils.putBytes(pageAddr, 0, pageRec.pageData());
+                                    long pageAddr = pageMem.writeLock(grpId, pageId, page);
+
+                                    try {
+                                        PageUtils.putBytes(pageAddr, 0, pageRec.pageData());
+                                    }
+                                    finally {
+                                        pageMem.writeUnlock(grpId, pageId, page, null, true, true);
+                                    }
                                 }
                                 finally {
-                                    pageMem.writeUnlock(grpId, pageId, page, null, true, true);
+                                    pageMem.releasePage(grpId, pageId, page);
                                 }
-                            }
-                            finally {
-                                pageMem.releasePage(grpId, pageId, page);
-                            }
 
-                            applied++;
+                                applied++;
+                            }
                         }
 
                         break;
@@ -1895,15 +1924,18 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
                             if (storeOnly && gId != METASTORAGE_CACHE_ID)
                                 continue;
 
-                            final int pId = destroyRec.partitionId();
+                            if (!ignoreGrps.contains(gId)) {
+                                final int pId = destroyRec.partitionId();
 
-                            PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
+                                PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
 
-                            pageMem.clearAsync(new P3() {
-                                @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
-                                    return cacheId == gId && PageIdUtils.partId(pageId) == pId;
-                                }
-                            }, true).get();
+                                pageMem.clearAsync(new P3() {
+                                    @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
+                                        return cacheId == gId && PageIdUtils.partId(pageId) == pId;
+                                    }
+                                }, true).get();
+
+                            }
                         }
 
                         break;
@@ -1917,29 +1949,31 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
                             if (storeOnly && grpId != METASTORAGE_CACHE_ID)
                                 continue;
 
-                            long pageId = r.pageId();
-
-                            PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
+                            if (!ignoreGrps.contains(grpId)) {
+                                long pageId = r.pageId();
 
-                            // Here we do not require tag check because we may be applying memory changes after
-                            // several repetitive restarts and the same pages may have changed several times.
-                            long page = pageMem.acquirePage(grpId, pageId, true);
+                                PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
 
-                            try {
-                                long pageAddr = pageMem.writeLock(grpId, pageId, page);
+                                // Here we do not require tag check because we may be applying memory changes after
+                                // several repetitive restarts and the same pages may have changed several times.
+                                long page = pageMem.acquirePage(grpId, pageId, true);
 
                                 try {
-                                    r.applyDelta(pageMem, pageAddr);
+                                    long pageAddr = pageMem.writeLock(grpId, pageId, page);
+
+                                    try {
+                                        r.applyDelta(pageMem, pageAddr);
+                                    }
+                                    finally {
+                                        pageMem.writeUnlock(grpId, pageId, page, null, true, true);
+                                    }
                                 }
                                 finally {
-                                    pageMem.writeUnlock(grpId, pageId, page, null, true, true);
+                                    pageMem.releasePage(grpId, pageId, page);
                                 }
-                            }
-                            finally {
-                                pageMem.releasePage(grpId, pageId, page);
-                            }
 
-                            applied++;
+                                applied++;
+                            }
                         }
                 }
             }
@@ -2050,7 +2084,7 @@ else if (log != null)
         checkpointReadLock();
 
         try {
-            restorePartitionState(partStates);
+            restorePartitionState(partStates, Collections.emptySet());
         }
         finally {
             checkpointReadUnlock();
@@ -2073,6 +2107,8 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
         long start = U.currentTimeMillis();
         int applied = 0;
 
+        Collection ignoreGrps = metastoreOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
+
         try (WALIterator it = cctx.wal().replay(status.startPtr)) {
             Map, T2> partStates = new HashMap<>();
 
@@ -2091,11 +2127,15 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
                         for (DataEntry dataEntry : dataRec.writeEntries()) {
                             int cacheId = dataEntry.cacheId();
 
-                            GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+                            int grpId = cctx.cache().cacheDescriptor(cacheId).groupId();
+
+                            if (!ignoreGrps.contains(grpId)) {
+                                GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
-                            applyUpdate(cacheCtx, dataEntry);
+                                applyUpdate(cacheCtx, dataEntry);
 
-                            applied++;
+                                applied++;
+                            }
                         }
 
                         break;
@@ -2106,8 +2146,10 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
 
                         PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec;
 
-                        partStates.put(new T2<>(metaStateRecord.groupId(), metaStateRecord.partitionId()),
-                            new T2<>((int)metaStateRecord.state(), metaStateRecord.updateCounter()));
+                        if (!ignoreGrps.contains(metaStateRecord.groupId())) {
+                            partStates.put(new T2<>(metaStateRecord.groupId(), metaStateRecord.partitionId()),
+                                new T2<>((int)metaStateRecord.state(), metaStateRecord.updateCounter()));
+                        }
 
                         break;
 
@@ -2152,7 +2194,7 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
             }
 
             if (!metastoreOnly)
-                restorePartitionState(partStates);
+                restorePartitionState(partStates, ignoreGrps);
         }
         finally {
             if (!metastoreOnly)
@@ -2169,10 +2211,11 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
      * @throws IgniteCheckedException If failed to restore.
      */
     private void restorePartitionState(
-        Map, T2> partStates
+        Map, T2> partStates,
+        Collection ignoreGrps
     ) throws IgniteCheckedException {
         for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
-            if (grp.isLocal() || !grp.affinityNode()) {
+            if (grp.isLocal() || !grp.affinityNode() || ignoreGrps.contains(grp.groupId())) {
                 // Local cache has no partitions and its states.
                 continue;
             }
@@ -2935,7 +2978,7 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
                     snapFut = snapshotMgr.onMarkCheckPointBegin(curr.snapshotOperation, map);
 
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
-                    if (grp.isLocal())
+                    if (grp.isLocal() || !grp.walEnabled())
                         continue;
 
                     List locParts = new ArrayList<>();
@@ -3458,7 +3501,7 @@ private CheckpointProgress(long nextCpTs) {
     /**
      *
      */
-    private static class CheckpointProgressSnapshot {
+    private static class CheckpointProgressSnapshot implements CheckpointFuture {
         /** */
         private final boolean started;
 
@@ -3474,6 +3517,16 @@ private static class CheckpointProgressSnapshot {
             cpBeginFut = cpProgress.cpBeginFut;
             cpFinishFut = cpProgress.cpFinishFut;
         }
+
+        /** {@inheritDoc} */
+        @Override public GridFutureAdapter beginFuture() {
+            return cpBeginFut;
+        }
+
+        /** {@inheritDoc} */
+        @Override public GridFutureAdapter finishFuture() {
+            return cpFinishFut;
+        }
     }
 
     /**
@@ -4181,4 +4234,78 @@ public DataStorageMetricsImpl persistentStoreMetricsImpl() {
     @Override public MetaStorage metaStorage() {
         return metaStorage;
     }
+
+    /** {@inheritDoc} */
+    @Override public boolean walEnabled(int grpId) {
+        return !initiallyWalDisabledGrps.contains(grpId);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void walEnabled(int grpId, boolean enabled) {
+        String key = walGroupIdToKey(grpId);
+
+        checkpointReadLock();
+
+        try {
+            if (enabled)
+                metaStorage.remove(key);
+            else
+                metaStorage.write(key, true);
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException("Failed to write cache group WAL state [grpId=" + grpId +
+                ", enabled=" + enabled + ']', e);
+        }
+        finally {
+            checkpointReadUnlock();
+        }
+    }
+
+    /**
+     * @return List of initially WAL-disabled groups.
+     */
+    private Collection walDisabledGroups() {
+        MetaStorage meta = cctx.database().metaStorage();
+
+        try {
+            Set keys = meta.readForPredicate(WAL_KEY_PREFIX_PRED).keySet();
+
+            if (keys.isEmpty())
+                return Collections.emptySet();
+
+            HashSet res = new HashSet<>(keys.size());
+
+            for (String key : keys) {
+                int grpId = walKeyToGroupId(key);
+
+                res.add(grpId);
+            }
+
+            return res;
+
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException("Failed to read cache groups WAL state.", e);
+        }
+    }
+
+    /**
+     * Convert cache group ID to WAL state key.
+     *
+     * @param grpId Group ID.
+     * @return Key.
+     */
+    private static String walGroupIdToKey(int grpId) {
+        return WAL_KEY_PREFIX + grpId;
+    }
+
+    /**
+     * Convert WAL state key to cache group ID.
+     *
+     * @param key Key.
+     * @return Group ID.
+     */
+    private static int walKeyToGroupId(String key) {
+        return Integer.parseInt(key.substring(WAL_KEY_PREFIX.length()));
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index f8fd86c3c93fd..4837f6e56b61d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -559,7 +559,8 @@ private static boolean addPartition(
 
             int tag = pageMemory.invalidate(grp.groupId(), p);
 
-            ctx.wal().log(new PartitionDestroyRecord(grp.groupId(), p));
+            if (grp.walEnabled())
+                ctx.wal().log(new PartitionDestroyRecord(grp.groupId(), p));
 
             ctx.pageStore().onPartitionDestroyed(grp.groupId(), p, tag);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 8658c97c652bb..237dacc31dfc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -727,6 +727,15 @@ public void checkpointReadUnlock() {
         return null;
     }
 
+    /**
+     * Allows to wait checkpoint finished.
+     *
+     * @param reason Reason.
+     */
+    @Nullable public CheckpointFuture forceCheckpoint(String reason) {
+        return null;
+    }
+
     /**
      * Waits until current state is checkpointed.
      *
@@ -1002,4 +1011,22 @@ protected void setPageSize(int pageSize) {
     public MetaStorage metaStorage() {
         return null;
     }
+
+    /**
+     * @param grpId Group ID.
+     * @return WAL enabled flag.
+     */
+    public boolean walEnabled(int grpId) {
+        return false;
+    }
+
+    /**
+     * Marks cache group as with disabled WAL.
+     *
+     * @param grpId Group id.
+     * @param enabled flag.
+     */
+    public void walEnabled(int grpId, boolean enabled) {
+        // No-op.
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index c9b7278d88fac..26e46b214848f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -52,6 +52,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
+import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
@@ -234,7 +235,6 @@ public FilePageStoreManager(GridKernalContext ctx) {
     /** {@inheritDoc} */
     @Override public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException {
         File cacheWorkDir = cacheWorkDir(cacheData.config());
-
         File file;
 
         checkAndInitCacheWorkDir(cacheWorkDir);
@@ -738,6 +738,21 @@ public PageStore getStore(int grpId, int partId) throws IgniteCheckedException {
         return store;
     }
 
+    /** {@inheritDoc} */
+    @Override public void beforeCacheGroupStart(CacheGroupDescriptor grpDesc) {
+        if (grpDesc.persistenceEnabled() && !cctx.database().walEnabled(grpDesc.groupId())) {
+            File dir = cacheWorkDir(grpDesc.config());
+
+            assert dir.exists();
+
+            boolean res = IgniteUtils.delete(dir);
+
+            assert res;
+
+            grpDesc.walEnabled(false);
+        }
+    }
+
     /**
      * @param pageStoreFileIoFactory File IO factory to override default, may be used for blocked read-write.
      * @param pageStoreV1FileIoFactory File IO factory for reading V1 page store and for fast touching page files
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index f0b2e165c1fd4..d57b9396bfcef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -19,6 +19,7 @@
 
 import java.io.Serializable;
 import java.nio.ByteBuffer;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
@@ -47,8 +48,10 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.SimpleDataPageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.lang.GridCursor;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.jetbrains.annotations.NotNull;
@@ -66,6 +69,9 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     /** */
     public static final int METASTORAGE_CACHE_ID = CU.cacheId(METASTORAGE_CACHE_NAME);
 
+    /** Marker for removed entry. */
+    private static final byte[] TOMBSTONE = new byte[0];
+
     /** */
     private final IgniteWriteAheadLogManager wal;
 
@@ -144,7 +150,63 @@ public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedExcept
         if (data != null)
             result = marshaller.unmarshal(data, getClass().getClassLoader());
 
-        return (Serializable) result;
+        return (Serializable)result;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map readForPredicate(IgnitePredicate keyPred)
+        throws IgniteCheckedException {
+        Map res = null;
+
+        if (readOnly) {
+            if (empty)
+                return Collections.emptyMap();
+
+            if (lastUpdates != null) {
+                for (Map.Entry lastUpdate : lastUpdates.entrySet()) {
+                    if (keyPred.apply(lastUpdate.getKey())) {
+                        byte[] valBytes = lastUpdate.getValue();
+
+                        if (valBytes == TOMBSTONE)
+                            continue;
+
+                        if (res == null)
+                            res = new HashMap<>();
+
+                        Serializable val = marshaller.unmarshal(valBytes, getClass().getClassLoader());
+
+                        res.put(lastUpdate.getKey(), val);
+                    }
+                }
+            }
+        }
+
+        GridCursor cur = tree.find(null, null);
+
+        while (cur.next()) {
+            MetastorageDataRow row = cur.get();
+
+            String key = row.key();
+            byte[] valBytes = row.value();
+
+            if (keyPred.apply(key)) {
+                // Either already added it, or this is a tombstone -> ignore.
+                if (lastUpdates != null && lastUpdates.containsKey(key))
+                    continue;
+
+                if (res == null)
+                    res = new HashMap<>();
+
+                Serializable val = marshaller.unmarshal(valBytes, getClass().getClassLoader());
+
+                res.put(key, val);
+            }
+        }
+
+        if (res == null)
+            res = Collections.emptyMap();
+
+        return res;
     }
 
     /** {@inheritDoc} */
@@ -187,10 +249,10 @@ public void putData(String key, byte[] data) throws IgniteCheckedException {
     public byte[] getData(String key) throws IgniteCheckedException {
         if (readOnly) {
             if (lastUpdates != null) {
-                byte[] lastUpdate = lastUpdates.get(key);
+                byte[] res = lastUpdates.get(key);
 
-                if (lastUpdate != null)
-                    return lastUpdate;
+                if (res != null)
+                    return res != TOMBSTONE ? res : null;
             }
 
             if (empty)
@@ -342,7 +404,7 @@ public void applyUpdate(String key, byte[] value) throws IgniteCheckedException
             if (lastUpdates == null)
                 lastUpdates = new HashMap<>();
 
-            lastUpdates.put(key, value);
+            lastUpdates.put(key, value != null ? value : TOMBSTONE);
         }
         else {
             if (value != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java
index 9b7367659cdce..390c2e40c77a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java
@@ -17,7 +17,10 @@
 package org.apache.ignite.internal.processors.cache.persistence.metastorage;
 
 import java.io.Serializable;
+import java.util.Map;
+
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.lang.IgnitePredicate;
 
 /**
  *
@@ -25,4 +28,13 @@
 public interface ReadOnlyMetastorage {
     /** */
     Serializable read(String key) throws IgniteCheckedException;
+
+    /**
+     * Read all keys matching provided predicate.
+     *
+     * @param keyPred Key predicate.
+     * @return Matched key-value pairs.
+     * @throws IgniteCheckedException If failed.
+     */
+    Map readForPredicate(IgnitePredicate keyPred) throws IgniteCheckedException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index f3924002af0e1..bcc8939c013ba 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -485,17 +485,18 @@ private void initWriteThrottle() {
                 if (PageIO.getType(pageAddr) == 0) {
                     trackingIO.initNewPage(pageAddr, pageId, pageSize());
 
-                    if (!ctx.wal().isAlwaysWriteFullPages())
-                        ctx.wal().log(
-                            new InitNewPageRecord(
-                                cacheId,
-                                pageId,
-                                trackingIO.getType(),
-                                trackingIO.getVersion(), pageId
-                            )
-                        );
-                    else
-                        ctx.wal().log(new PageSnapshot(fullId, absPtr + PAGE_OVERHEAD, pageSize()));
+                    if (!ctx.wal().disabled(fullId.groupId()))
+                        if (!ctx.wal().isAlwaysWriteFullPages())
+                            ctx.wal().log(
+                                new InitNewPageRecord(
+                                    cacheId,
+                                    pageId,
+                                    trackingIO.getType(),
+                                    trackingIO.getVersion(), pageId
+                                )
+                            );
+                        else
+                            ctx.wal().log(new PageSnapshot(fullId, absPtr + PAGE_OVERHEAD, pageSize()));
                 }
             }
 
@@ -1480,7 +1481,7 @@ void setDirty(FullPageId pageId, long absPtr, boolean dirty, boolean forceAdd) {
      *
      */
     void beforeReleaseWrite(FullPageId pageId, long ptr, boolean pageWalRec) {
-        if (walMgr != null && (pageWalRec || walMgr.isAlwaysWriteFullPages())) {
+        if (walMgr != null && (pageWalRec || walMgr.isAlwaysWriteFullPages()) && !walMgr.disabled(pageId.groupId())) {
             try {
                 walMgr.log(new PageSnapshot(pageId, ptr, pageSize()));
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
index 33169800c7067..8f854cffeea9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
@@ -449,7 +449,7 @@ public static boolean isWalDeltaRecordNeeded(
         Boolean walPlc) {
         // If the page is clean, then it is either newly allocated or just after checkpoint.
         // In both cases we have to write full page contents to WAL.
-        return wal != null && !wal.isAlwaysWriteFullPages() && walPlc != TRUE &&
+        return wal != null && !wal.isAlwaysWriteFullPages() && walPlc != TRUE && !wal.disabled(cacheId) &&
             (walPlc == FALSE || pageMem.isDirty(cacheId, pageId, page));
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 444a207ff4b66..7b3e93871d2c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -79,6 +79,7 @@
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
 import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.DataStorageMetricsImpl;
@@ -909,6 +910,13 @@ private boolean hasIndex(long absIdx) {
         return archiver0 != null && archiver0.reserved(fPtr.index());
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean disabled(int grpId) {
+        CacheGroupContext ctx = cctx.cache().cacheGroup(grpId);
+
+        return ctx != null && !ctx.walEnabled();
+    }
+
     /**
      * Lists files in archive directory and returns the index of last archived file.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index d30dddb71e214..0c7bbb323656f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -202,7 +202,7 @@ public WALIterator iteratorWorkFiles(@NotNull final File... files) throws Ignite
 
         return new GridCacheSharedContext<>(
             kernalCtx, null, null, null,
-            null, null, dbMgr, null,
+            null, null, null, dbMgr, null,
             null, null, null, null,
             null, null, null);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 2a0b1766b3cc0..8dec59a276961 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.CacheInvokeEntry;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.EntryProcessorResourceInjectorProxy;
@@ -759,7 +760,10 @@ else if (op == RELOAD) {
                                             nearCached.innerReload();
                                     }
                                     else if (op == READ) {
-                                        if (cacheCtx.group().persistenceEnabled() && cctx.snapshot().needTxReadLogging()) {
+                                        CacheGroupContext grp = cacheCtx.group();
+
+                                        if (grp.persistenceEnabled() && grp.walEnabled() &&
+                                            cctx.snapshot().needTxReadLogging()) {
                                             ptr = cctx.wal().log(new DataRecord(new DataEntry(
                                                 cacheCtx.cacheId(),
                                                 txEntry.key(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java
index 344e6fe2761e1..a490cfcb33b09 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerProcessor.java
@@ -193,6 +193,13 @@ public ClientListenerProcessor(GridKernalContext ctx) {
         }
     }
 
+    /**
+     * @return Server port.
+     */
+    public int port() {
+        return srv.port();
+    }
+
     /**
      * Prepare connector configuration.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
index 021dfb95cd24d..f5b8c3cedeb48 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlKeyword.java
@@ -27,6 +27,9 @@
  * SQL keyword constants.
  */
 public class SqlKeyword {
+    /** Keyword: ALTER. */
+    public static final String ALTER = "ALTER";
+
     /** Keyword: ASC. */
     public static final String ASC = "ASC";
 
@@ -120,6 +123,9 @@ public class SqlKeyword {
     /** Keyword: KEY. */
     public static final String KEY = "KEY";
 
+    /** Keyword: LOGGING. */
+    public static final String LOGGING = "LOGGING";
+
     /** Keyword: LONGVARCHAR. */
     public static final String LONGVARCHAR = "LONGVARCHAR";
 
@@ -129,6 +135,9 @@ public class SqlKeyword {
     /** Keyword: NCHAR. */
     public static final String NCHAR = "NCHAR";
 
+    /** Keyword: NOLOGGING. */
+    public static final String NOLOGGING = "NOLOGGING";
+
     /** Keyword: NOT. */
     public static final String NOT = "NOT";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
index 19f526d146ea6..401ee98923c27 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlParser.java
@@ -17,17 +17,20 @@
 
 package org.apache.ignite.internal.sql;
 
+import org.apache.ignite.internal.sql.command.SqlAlterTableCommand;
 import org.apache.ignite.internal.sql.command.SqlCommand;
 import org.apache.ignite.internal.sql.command.SqlCreateIndexCommand;
 import org.apache.ignite.internal.sql.command.SqlDropIndexCommand;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.sql.SqlKeyword.ALTER;
 import static org.apache.ignite.internal.sql.SqlKeyword.CREATE;
 import static org.apache.ignite.internal.sql.SqlKeyword.DROP;
 import static org.apache.ignite.internal.sql.SqlKeyword.HASH;
 import static org.apache.ignite.internal.sql.SqlKeyword.INDEX;
 import static org.apache.ignite.internal.sql.SqlKeyword.PRIMARY;
 import static org.apache.ignite.internal.sql.SqlKeyword.SPATIAL;
+import static org.apache.ignite.internal.sql.SqlKeyword.TABLE;
 import static org.apache.ignite.internal.sql.SqlKeyword.UNIQUE;
 import static org.apache.ignite.internal.sql.SqlParserUtils.errorUnexpectedToken;
 import static org.apache.ignite.internal.sql.SqlParserUtils.errorUnsupportedIfMatchesKeyword;
@@ -99,6 +102,9 @@ private SqlCommand nextCommand0() {
                             cmd = processDrop();
 
                             break;
+
+                        case ALTER:
+                            cmd = processAlter();
                     }
 
                     if (cmd != null) {
@@ -109,7 +115,7 @@ private SqlCommand nextCommand0() {
                         return cmd;
                     }
                     else
-                        throw errorUnexpectedToken(lex, CREATE, DROP);
+                        throw errorUnexpectedToken(lex, CREATE, DROP, ALTER);
 
                 case QUOTED:
                 case MINUS:
@@ -178,4 +184,27 @@ private SqlCommand processDrop() {
 
         throw errorUnexpectedToken(lex, INDEX);
     }
+
+    /**
+     * Process ALTER keyword.
+     *
+     * @return Command.
+     */
+    private SqlCommand processAlter() {
+        if (lex.shift() && lex.tokenType() == SqlLexerTokenType.DEFAULT) {
+            SqlCommand cmd = null;
+
+            switch (lex.token()) {
+                case TABLE:
+                    cmd = new SqlAlterTableCommand();
+
+                    break;
+            }
+
+            if (cmd != null)
+                return cmd.parse(lex);
+        }
+
+        throw errorUnexpectedToken(lex, TABLE);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlAlterTableCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlAlterTableCommand.java
new file mode 100644
index 0000000000000..fe60910dc9bbf
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/command/SqlAlterTableCommand.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.sql.command;
+
+import org.apache.ignite.internal.sql.SqlLexer;
+import org.apache.ignite.internal.sql.SqlLexerToken;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.sql.SqlKeyword.LOGGING;
+import static org.apache.ignite.internal.sql.SqlKeyword.NOLOGGING;
+import static org.apache.ignite.internal.sql.SqlParserUtils.errorUnexpectedToken;
+import static org.apache.ignite.internal.sql.SqlParserUtils.matchesKeyword;
+import static org.apache.ignite.internal.sql.SqlParserUtils.parseQualifiedIdentifier;
+
+/**
+ * ALTER TABLE command.
+ */
+public class SqlAlterTableCommand implements SqlCommand  {
+    /** Schema name. */
+    private String schemaName;
+
+    /** Schema name. */
+    private String tblName;
+
+    /** Logging flag. */
+    private Boolean logging;
+
+    /** {@inheritDoc} */
+    @Override public String schemaName() {
+        return schemaName;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void schemaName(String schemaName) {
+        this.schemaName = schemaName;
+    }
+
+    /**
+     * @return Table name.
+     */
+    public String tableName() {
+        return tblName;
+    }
+
+    /**
+     * @return Logging status or {@code null} if no changes to logging is requested.
+     */
+    @Nullable public Boolean logging() {
+        return logging;
+    }
+
+    /** {@inheritDoc} */
+    @Override public SqlCommand parse(SqlLexer lex) {
+        SqlQualifiedName tblQName = parseQualifiedIdentifier(lex);
+
+        schemaName = tblQName.schemaName();
+        tblName = tblQName.name();
+
+        parseLogging(lex);
+
+        if (!hasCommands())
+            throw errorUnexpectedToken(lex, LOGGING, NOLOGGING);
+
+        return this;
+    }
+
+    /**
+     * Parse LOGGING and/or NOLOGGING statement.
+     */
+    private void parseLogging(SqlLexer lex) {
+        SqlLexerToken token = lex.lookAhead();
+
+        if (matchesKeyword(token, LOGGING)) {
+            lex.shift();
+
+            logging = true;
+        }
+        else if (matchesKeyword(token, NOLOGGING)) {
+            lex.shift();
+
+            logging = false;
+        }
+    }
+
+    /**
+     * Check if statement contain any commands.
+     *
+     * @return {@code True} if statement is not dummy and contains at least one command.
+     */
+    private boolean hasCommands() {
+        return logging != null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(SqlAlterTableCommand.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java
index e5b7b1bef1f92..ebe2e0765c787 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java
@@ -427,7 +427,7 @@ public int[] array() {
 
         b.a(']');
 
-        return S.toString(GridIntList.class, this, "arr", b);
+        return b.toString();
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAbstractSelfTest.java
new file mode 100644
index 0000000000000..048d72003eb97
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAbstractSelfTest.java
@@ -0,0 +1,282 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import java.util.concurrent.Callable;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.LOCAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+
+/**
+ * Test dynamic WAL mode change.
+ */
+
+public abstract class WalModeChangeAbstractSelfTest extends WalModeChangeCommonAbstractSelfTest {
+    /** Whether coordinator node should be filtered out. */
+    private final boolean filterOnCrd;
+
+    /**
+     * Constructor.
+     *
+     * @param filterOnCrd Whether coordinator node should be filtered out.
+     * @param jdbc Whether this is JDBC test.
+     */
+    protected WalModeChangeAbstractSelfTest(boolean filterOnCrd, boolean jdbc) {
+        super(jdbc);
+
+        this.filterOnCrd = filterOnCrd;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        deleteWorkFiles();
+
+        startGrid(config(SRV_1, false, filterOnCrd));
+        startGrid(config(SRV_2, false, false));
+        startGrid(config(SRV_3, false, !filterOnCrd));
+
+        Ignite cli = startGrid(config(CLI, true, false));
+
+        cli.cluster().active(true);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        deleteWorkFiles();
+    }
+
+    /**
+     * Negative case: cache name is null.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNullCacheName() throws Exception {
+        forAllNodes(new IgniteInClosureX() {
+            @Override public void applyx(Ignite ignite) throws IgniteCheckedException {
+                assertThrows(new Callable() {
+                    @Override public Void call() throws Exception {
+                        walEnable(ignite, null);
+
+                        return null;
+                    }
+                }, NullPointerException.class, null);
+            }
+        });
+    }
+
+    /**
+     * Negative case: no cache.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNoCache() throws Exception {
+        forAllNodes(new IgniteInClosureX() {
+            @Override public void applyx(Ignite ignite) throws IgniteCheckedException {
+                assertThrows(new Callable() {
+                    @Override public Void call() throws Exception {
+                        walEnable(ignite, CACHE_NAME);
+
+                        return null;
+                    }
+                }, IgniteException.class, (jdbc ? "Table doesn't exist" : "Cache doesn't exist"));
+            }
+        });
+    }
+
+
+    /**
+     * Negative case: trying to disable WAL for cache in a shared cache group.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSharedCacheGroup() throws Exception {
+        forAllNodes(new IgniteInClosureX() {
+            @Override public void applyx(Ignite ignite) throws IgniteCheckedException {
+                createCache(ignite, cacheConfig(CACHE_NAME, PARTITIONED, TRANSACTIONAL).setGroupName("grp"));
+                createCache(ignite, cacheConfig(CACHE_NAME_2, PARTITIONED, TRANSACTIONAL).setGroupName("grp"));
+
+                assertThrows(new Callable() {
+                    @Override public Void call() throws Exception {
+                        walDisable(ignite, CACHE_NAME);
+
+                        return null;
+                    }
+                }, IgniteException.class, "Cannot change WAL mode because not all cache names belonging to the " +
+                    "group are provided");
+
+                assertThrows(new Callable() {
+                    @Override public Void call() throws Exception {
+                        walEnable(ignite, CACHE_NAME);
+
+                        return null;
+                    }
+                }, IgniteException.class, "Cannot change WAL mode because not all cache names belonging to the " +
+                    "group are provided");
+
+                assert ignite.cluster().isWalEnabled(CACHE_NAME);
+                assert ignite.cluster().isWalEnabled(CACHE_NAME_2);
+            }
+        });
+    }
+
+    /**
+     * Negative test case: disabled persistence.
+     *
+     * @throws Exception If failed.
+     */
+    public void testPersistenceDisabled() throws Exception {
+        forAllNodes(new IgniteInClosureX() {
+            @Override public void applyx(Ignite ignite) throws IgniteCheckedException {
+                U.sleep(500);
+
+                CacheConfiguration ccfg = cacheConfig(CACHE_NAME_3, PARTITIONED, TRANSACTIONAL)
+                    .setDataRegionName(REGION_VOLATILE);
+
+                createCache(ignite, ccfg);
+
+                assertThrows(new Callable() {
+                    @Override public Void call() throws Exception {
+                        walDisable(ignite, CACHE_NAME_3);
+
+                        return null;
+                    }
+                }, IgniteException.class, "Cannot change WAL mode because persistence is not enabled for cache(s)");
+
+                assertThrows(new Callable() {
+                    @Override public Void call() throws Exception {
+                        walEnable(ignite, CACHE_NAME_3);
+
+                        return null;
+                    }
+                }, IgniteException.class, "Cannot change WAL mode because persistence is not enabled for cache(s)");
+
+                assert !ignite.cluster().isWalEnabled(CACHE_NAME_3);
+            }
+        });
+    }
+
+    /**
+     * Negative case: LOCAL cache.
+     *
+     * @throws Exception If failed.
+     */
+    public void testLocalCache() throws Exception {
+        if (jdbc)
+            // Doesn't make sense for JDBC.
+            return;
+
+        forAllNodes(new IgniteInClosureX() {
+            @Override public void applyx(Ignite ignite) throws IgniteCheckedException {
+                createCache(ignite, cacheConfig(LOCAL).setDataRegionName(REGION_VOLATILE));
+
+                assertThrows(new Callable() {
+                    @Override public Void call() throws Exception {
+                        walDisable(ignite, CACHE_NAME);
+
+                        return null;
+                    }
+                }, IgniteException.class, "WAL mode cannot be changed for LOCAL cache(s)");
+
+                assertThrows(new Callable() {
+                    @Override public Void call() throws Exception {
+                        walEnable(ignite, CACHE_NAME);
+
+                        return null;
+                    }
+                }, IgniteException.class, "WAL mode cannot be changed for LOCAL cache(s)");
+
+                assert !ignite.cluster().isWalEnabled(CACHE_NAME);
+            }
+        });
+    }
+
+    /**
+     * Test enable/disable for PARTITIONED ATOMIC cache.
+     *
+     * @throws Exception If failed.
+     */
+    public void testEnableDisablePartitionedAtomic() throws Exception {
+        checkEnableDisable(PARTITIONED, ATOMIC);
+    }
+
+    /**
+     * Test enable/disable for PARTITIONED TRANSACTIONAL cache.
+     *
+     * @throws Exception If failed.
+     */
+    public void testEnableDisablePartitionedTransactional() throws Exception {
+        checkEnableDisable(PARTITIONED, TRANSACTIONAL);
+    }
+
+    /**
+     * Test enable/disable for REPLICATED ATOMIC cache.
+     *
+     * @throws Exception If failed.
+     */
+    public void testEnableDisableReplicatedAtomic() throws Exception {
+        checkEnableDisable(REPLICATED, ATOMIC);
+    }
+
+    /**
+     * Test enable/disable for REPLICATED TRANSACTIONAL cache.
+     *
+     * @throws Exception If failed.
+     */
+    public void testEnableDisableReplicatedTransactional() throws Exception {
+        checkEnableDisable(REPLICATED, TRANSACTIONAL);
+    }
+
+    /**
+     * Check normal disable-enable flow.
+     *
+     * @throws Exception If failed.
+     */
+    private void checkEnableDisable(CacheMode mode, CacheAtomicityMode atomicityMode) throws Exception {
+        forAllNodes(new IgniteInClosureX() {
+            @Override public void applyx(Ignite ignite) throws IgniteCheckedException {
+                createCache(ignite, cacheConfig(CACHE_NAME, mode, atomicityMode));
+
+                for (int i = 0; i < 2; i++) {
+                    assertForAllNodes(CACHE_NAME, true);
+
+                    assertWalEnable(ignite, CACHE_NAME, false);
+                    assertWalDisable(ignite, CACHE_NAME, true);
+
+                    assertForAllNodes(CACHE_NAME, false);
+
+                    assertWalDisable(ignite, CACHE_NAME, false);
+                    assertWalEnable(ignite, CACHE_NAME, true);
+                }
+            }
+        });
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAdvancedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAdvancedSelfTest.java
new file mode 100644
index 0000000000000..2c6bccff3d7f6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeAdvancedSelfTest.java
@@ -0,0 +1,535 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
+import org.apache.ignite.internal.util.typedef.X;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Concurrent and advanced tests for WAL state change.
+ */
+@SuppressWarnings("unchecked")
+public class WalModeChangeAdvancedSelfTest extends WalModeChangeCommonAbstractSelfTest {
+    /**
+     * Constructor.
+     */
+    public WalModeChangeAdvancedSelfTest() {
+        super(false);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        deleteWorkFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        deleteWorkFiles();
+    }
+
+    /**
+     * Test cache cleanup on restart.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheCleanup() throws Exception {
+        Ignite srv = startGrid(config(SRV_1, false, false));
+
+        srv.cluster().active(true);
+
+        IgniteCache cache1 = srv.getOrCreateCache(cacheConfig(CACHE_NAME, PARTITIONED, TRANSACTIONAL));
+        IgniteCache cache2 = srv.getOrCreateCache(cacheConfig(CACHE_NAME_2, PARTITIONED, TRANSACTIONAL));
+
+        assertForAllNodes(CACHE_NAME, true);
+        assertForAllNodes(CACHE_NAME_2, true);
+
+        for (int i = 0; i < 10; i++) {
+            cache1.put(i, i);
+            cache2.put(i, i);
+        }
+
+        srv.cluster().disableWal(CACHE_NAME);
+
+        assertForAllNodes(CACHE_NAME, false);
+        assertForAllNodes(CACHE_NAME_2, true);
+
+        for (int i = 10; i < 20; i++) {
+            cache1.put(i, i);
+            cache2.put(i, i);
+        }
+
+        srv.cluster().disableWal(CACHE_NAME_2);
+
+        assertForAllNodes(CACHE_NAME, false);
+        assertForAllNodes(CACHE_NAME_2, false);
+
+        for (int i = 20; i < 30; i++) {
+            cache1.put(i, i);
+            cache2.put(i, i);
+        }
+
+        assertEquals(cache1.size(), 30);
+        assertEquals(cache2.size(), 30);
+
+        srv.cluster().enableWal(CACHE_NAME);
+
+        assertForAllNodes(CACHE_NAME, true);
+        assertForAllNodes(CACHE_NAME_2, false);
+
+        assertEquals(cache1.size(), 30);
+        assertEquals(cache2.size(), 30);
+
+        stopAllGrids(true);
+
+        srv = startGrid(config(SRV_1, false, false));
+
+        srv.cluster().active(true);
+
+        cache1 = srv.cache(CACHE_NAME);
+        cache2 = srv.cache(CACHE_NAME_2);
+
+        assertForAllNodes(CACHE_NAME, true);
+        assertForAllNodes(CACHE_NAME_2, false);
+
+        assertEquals(30, cache1.size());
+        assertEquals(0, cache2.size());
+    }
+
+    /**
+     * Test simple node join.
+     *
+     * @throws Exception If failed.
+     */
+    public void testJoin() throws Exception {
+        checkJoin(false);
+    }
+
+    /**
+     * Test simple node join when operations is performed from coordinator.
+     *
+     * @throws Exception If failed.
+     */
+    public void testJoinCoordinator() throws Exception {
+        checkJoin(true);
+    }
+
+    /**
+     * Check node join behavior.
+     *
+     * @param crdFiltered {@code True} if first node is coordinator.
+     * @throws Exception If failed.
+     */
+    private void checkJoin(boolean crdFiltered) throws Exception {
+        // Start node and disable WAL.
+        Ignite srv = startGrid(config(SRV_1, false, crdFiltered));
+
+        srv.cluster().active(true);
+
+        srv.getOrCreateCache(cacheConfig(PARTITIONED));
+        assertForAllNodes(CACHE_NAME, true);
+
+        if (!crdFiltered) {
+            srv.cluster().disableWal(CACHE_NAME);
+            assertForAllNodes(CACHE_NAME, false);
+        }
+
+        // Start other nodes.
+        startGrid(config(SRV_2, false, false));
+        assertForAllNodes(CACHE_NAME, false);
+
+        if (crdFiltered) {
+            srv.cluster().disableWal(CACHE_NAME);
+            assertForAllNodes(CACHE_NAME, false);
+        }
+
+        startGrid(config(SRV_3, false, !crdFiltered));
+        assertForAllNodes(CACHE_NAME, false);
+
+        startGrid(config(CLI, true, false));
+        assertForAllNodes(CACHE_NAME, false);
+
+        // Stop nodes and restore WAL state on the first node.
+        stopGrid(SRV_2, true);
+        stopGrid(SRV_3, true);
+        stopGrid(CLI, true);
+
+        if (!crdFiltered) {
+            srv.cluster().enableWal(CACHE_NAME);
+            assertForAllNodes(CACHE_NAME, true);
+        }
+
+        // Start other nodes again.
+        startGrid(config(SRV_2, false, false));
+        assertForAllNodes(CACHE_NAME, true);
+
+        if (crdFiltered) {
+            srv.cluster().enableWal(CACHE_NAME);
+            assertForAllNodes(CACHE_NAME, true);
+        }
+
+        startGrid(config(SRV_3, false, !crdFiltered));
+        assertForAllNodes(CACHE_NAME, true);
+
+        startGrid(config(CLI, true, false));
+        assertForAllNodes(CACHE_NAME, true);
+    }
+
+    /**
+     * Test server restart (non-coordinator).
+     *
+     * @throws Exception If failed.
+     */
+    public void testServerRestartNonCoordinator() throws Exception {
+        checkNodeRestart(false);
+    }
+
+    /**
+     * Test server restart (coordinator).
+     *
+     * @throws Exception If failed.
+     */
+    public void testServerRestartCoordinator() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-7472");
+
+        checkNodeRestart(true);
+    }
+
+    /**
+     * Test coordinator node migration.
+     *
+     * @param failCrd Whether to fail coordinator nodes.
+     * @throws Exception If failed.
+     */
+    public void checkNodeRestart(boolean failCrd) throws Exception {
+        startGrid(config(SRV_1, false, false));
+        startGrid(config(SRV_2, false, false));
+
+        Ignite cli = startGrid(config(CLI, true, false));
+
+        cli.cluster().active(true);
+
+        cli.getOrCreateCache(cacheConfig(PARTITIONED));
+
+        final AtomicInteger restartCnt = new AtomicInteger();
+
+        final int restarts = 10;
+
+        Thread t = new Thread(new Runnable() {
+            @Override public void run() {
+                boolean firstOrSecond = true;
+
+                while (restartCnt.get() < restarts) {
+                    String victimName;
+
+                    if (failCrd) {
+                        victimName = firstOrSecond ? SRV_1 : SRV_2;
+
+                        firstOrSecond = !firstOrSecond;
+                    }
+                    else
+                        victimName = SRV_2;
+
+                    try {
+                        stopGrid(victimName);
+                        startGrid(config(victimName, false, false));
+
+                        Thread.sleep(500);
+                    }
+                    catch (Exception e) {
+                        throw new RuntimeException();
+                    }
+
+                    restartCnt.incrementAndGet();
+
+                    X.println(">>> Finished restart: " + restartCnt.get());
+                }
+            }
+        });
+
+        t.start();
+
+        boolean state = true;
+
+        while (restartCnt.get() < restarts && !Thread.currentThread().isInterrupted()) {
+            try {
+                if (state)
+                    cli.cluster().disableWal(CACHE_NAME);
+                else
+                    cli.cluster().enableWal(CACHE_NAME);
+
+                state = !state;
+            }
+            catch (IgniteException e) {
+                // Possible disconnect, re-try.
+            }
+        }
+    }
+
+    /**
+     * Test client re-connect.
+     *
+     * @throws Exception If failed.
+     */
+    public void testClientReconnect() throws Exception {
+        final Ignite srv = startGrid(config(SRV_1, false, false));
+        Ignite cli = startGrid(config(CLI, true, false));
+
+        cli.cluster().active(true);
+
+        cli.getOrCreateCache(cacheConfig(PARTITIONED));
+
+        final AtomicBoolean done = new AtomicBoolean();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        // Start load.
+        Thread t = new Thread(new Runnable() {
+            @Override public void run() {
+                boolean state = false;
+
+                while (!done.get()) {
+                    try {
+                        if (state)
+                            cli.cluster().enableWal(CACHE_NAME);
+                        else
+                            cli.cluster().disableWal(CACHE_NAME);
+                    }
+                    catch (IgniteException e) {
+                        String msg = e.getMessage();
+
+                        assert msg.startsWith("Client node disconnected") ||
+                            msg.startsWith("Client node was disconnected") : e.getMessage();
+                    }
+                    finally {
+                        state = !state;
+                    }
+                }
+
+                latch.countDown();
+            }
+        });
+
+        t.setName("wal-load-" + cli.name());
+
+        t.start();
+
+        // Now perform multiple client reconnects.
+        for (int i = 1; i <= 10; i++) {
+            Thread.sleep(ThreadLocalRandom.current().nextLong(200, 1000));
+
+            IgniteClientReconnectAbstractTest.reconnectClientNode(log, cli, srv, new Runnable() {
+                @Override public void run() {
+                    // No-op.
+                }
+            });
+
+            X.println(">>> Finished iteration: " + i);
+        }
+
+        done.set(true);
+
+        latch.await();
+    }
+
+    /**
+     * Test client re-connect.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheDestroy() throws Exception {
+        final Ignite srv = startGrid(config(SRV_1, false, false));
+        Ignite cli = startGrid(config(CLI, true, false));
+
+        cli.cluster().active(true);
+
+        srv.createCache(cacheConfig(PARTITIONED));
+
+        final AtomicBoolean done = new AtomicBoolean();
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        // Start load.
+        Thread t = new Thread(new Runnable() {
+            @Override public void run() {
+                boolean state = false;
+
+                while (!done.get()) {
+                    try {
+                        if (state)
+                            cli.cluster().enableWal(CACHE_NAME);
+                        else
+                            cli.cluster().disableWal(CACHE_NAME);
+                    }
+                    catch (IgniteException e) {
+                        String msg = e.getMessage();
+
+                        assert msg.startsWith("Cache doesn't exist") ||
+                            msg.startsWith("Failed to change WAL mode because some caches no longer exist") :
+                            e.getMessage();
+                    }
+                    finally {
+                        state = !state;
+                    }
+                }
+
+                latch.countDown();
+            }
+        });
+
+        t.setName("wal-load-" + cli.name());
+
+        t.start();
+
+        // Now perform multiple client reconnects.
+        for (int i = 1; i <= 20; i++) {
+            Thread.sleep(ThreadLocalRandom.current().nextLong(200, 1000));
+
+            srv.destroyCache(CACHE_NAME);
+
+            Thread.sleep(100);
+
+            srv.createCache(cacheConfig(PARTITIONED));
+
+            X.println(">>> Finished iteration: " + i);
+        }
+
+        done.set(true);
+
+        latch.await();
+    }
+
+    /**
+     * Test that concurrent enable/disable events doesn't leave to hangs.
+     *
+     * @throws Exception If failed.
+     */
+    public void testConcurrentOperations() throws Exception {
+        final Ignite srv1 = startGrid(config(SRV_1, false, false));
+        final Ignite srv2 = startGrid(config(SRV_2, false, false));
+        final Ignite srv3 = startGrid(config(SRV_3, false, true));
+
+        final Ignite cli = startGrid(config(CLI, true, false));
+
+        final Ignite cacheCli = startGrid(config(CLI_2, true, false));
+
+        cacheCli.cluster().active(true);
+
+        final IgniteCache cache = cacheCli.getOrCreateCache(cacheConfig(PARTITIONED));
+
+        for (int i = 1; i <= 3; i++) {
+            // Start pushing requests.
+            Collection walNodes = new ArrayList<>();
+
+            walNodes.add(srv1);
+            walNodes.add(srv2);
+            walNodes.add(srv3);
+            walNodes.add(cli);
+
+            final AtomicBoolean done = new AtomicBoolean();
+
+            final CountDownLatch latch = new CountDownLatch(5);
+
+            for (Ignite node : walNodes) {
+                final Ignite node0 = node;
+
+                Thread t = new Thread(new Runnable() {
+                    @Override public void run() {
+                        checkConcurrentOperations(done, node0);
+
+                        latch.countDown();
+                    }
+                });
+
+                t.setName("wal-load-" + node0.name());
+
+                t.start();
+            }
+
+            // Do some cache loading in the mean time.
+            Thread t = new Thread(new Runnable() {
+                @Override public void run() {
+                    int i = 0;
+
+                    while (!done.get())
+                        cache.put(i, i++);
+
+                    latch.countDown();
+                }
+            });
+
+            t.setName("cache-load");
+
+            t.start();
+
+            Thread.sleep(20_000);
+
+            done.set(true);
+
+            X.println(">>> Stopping iteration: " + i);
+
+            latch.await();
+
+            X.println(">>> Iteration finished: " + i);
+        }
+    }
+
+    /**
+     * Check concurrent operations.
+     *
+     * @param done Done flag.
+     * @param node Node.
+     */
+    private static void checkConcurrentOperations(AtomicBoolean done, Ignite node) {
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        boolean state = rnd.nextBoolean();
+
+        while (!done.get()) {
+            if (state)
+                node.cluster().enableWal(CACHE_NAME);
+            else
+                node.cluster().disableWal(CACHE_NAME);
+
+            state = !state;
+        }
+
+        try {
+            Thread.sleep(rnd.nextLong(200, 1000));
+        }
+        catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java
new file mode 100644
index 0000000000000..4467f7e97eb9e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java
@@ -0,0 +1,338 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.Callable;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ * Test dynamic WAL mode change.
+ */
+
+public abstract class WalModeChangeCommonAbstractSelfTest extends GridCommonAbstractTest {
+    /** Shared IP finder. */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Node filter. */
+    private static final IgnitePredicate FILTER = new CacheNodeFilter();
+
+    /** Cache name. */
+    protected static final String CACHE_NAME = "cache";
+
+    /** Cache name 2. */
+    protected static final String CACHE_NAME_2 = "cache_2";
+
+    /** Cache name 2. */
+    protected static final String CACHE_NAME_3 = "cache_3";
+
+    /** Volatile data region. */
+    protected static final String REGION_VOLATILE = "volatile";
+
+    /** Server 1. */
+    protected static final String SRV_1 = "srv_1";
+
+    /** Server 2. */
+    protected static final String SRV_2 = "srv_2";
+
+    /** Server 3. */
+    protected static final String SRV_3 = "srv_3";
+
+    /** Client. */
+    protected static final String CLI = "cli";
+
+    /** Client 2. */
+    protected static final String CLI_2 = "cli_2";
+
+    /** Node attribute for filtering. */
+    protected static final String FILTER_ATTR = "FILTER";
+
+    /** Whether this is JDBC test. */
+    protected final boolean jdbc;
+
+    /**
+     * Constructor.
+     *
+     * @param jdbc Whether this is JDBC test.
+     */
+    protected WalModeChangeCommonAbstractSelfTest(boolean jdbc) {
+        this.jdbc = jdbc;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        for (Ignite node0 : Ignition.allGrids()) {
+            Collection cacheNames = node0.cacheNames();
+
+            for (String cacheName : cacheNames)
+                destroyCache(node0, cacheName);
+        }
+    }
+
+    /**
+     * Create cache.
+     *
+     * @param node Node.
+     * @param ccfg Cache configuration.
+     */
+    @SuppressWarnings("unchecked")
+    protected void createCache(Ignite node, CacheConfiguration ccfg) {
+        node.createCache(ccfg);
+    }
+
+    /**
+     * Destroy cache.
+     *
+     * @param node Node.
+     * @param cacheName Cache name.
+     */
+    protected void destroyCache(Ignite node, String cacheName) {
+        node.destroyCache(cacheName);
+    }
+
+    /**
+     * Enable WAL.
+     *
+     * @param node Node.
+     * @param cacheName Cache name.
+     * @return Result.
+     */
+    protected boolean walEnable(Ignite node, String cacheName) {
+        return node.cluster().enableWal(cacheName);
+    }
+
+    /**
+     * Disable WAL.
+     *
+     * @param node Node.
+     * @param cacheName Cache name.
+     * @return Result.
+     */
+    protected boolean walDisable(Ignite node, String cacheName) {
+        return node.cluster().disableWal(cacheName);
+    }
+
+    /**
+     * Enable WAL.
+     *
+     * @param node Node.
+     * @param cacheName Cache name.
+     * @param expRes Expected result.
+     */
+    protected void assertWalEnable(Ignite node, String cacheName, boolean expRes) {
+        boolean res = walEnable(node, cacheName);
+
+        if (!jdbc)
+            assertEquals(expRes, res);
+    }
+
+    /**
+     * Disable WAL.
+     *
+     * @param node Node.
+     * @param cacheName Cache name.
+     * @param expRes Expected result.
+     */
+    protected void assertWalDisable(Ignite node, String cacheName, boolean expRes) {
+        boolean res = walDisable(node, cacheName);
+
+        if (!jdbc)
+            assertEquals(expRes, res);
+    }
+
+    /**
+     * Assert WAL state on all nodes.
+     *
+     * @param cacheName Cache name.
+     * @param expState Expected state.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void assertForAllNodes(String cacheName, boolean expState) throws IgniteCheckedException {
+        for (final Ignite node : Ignition.allGrids()) {
+            info(">>> Checking WAL state on node: " + node.name());
+
+            assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return node.cluster().isWalEnabled(cacheName) == expState;
+                }
+            }, 1000L);
+        }
+    }
+
+    /**
+     * Ensure exception is thrown.
+     *
+     * @param cmd Command.
+     * @param errCls Expected error class.
+     * @param errMsg Expected error message.
+     */
+    protected void assertThrows(Callable cmd, Class errCls, String errMsg) {
+        try {
+            cmd.call();
+
+            fail("Exception is not thrown");
+        }
+        catch (Exception e) {
+            if (jdbc) {
+                e = (Exception) e.getCause();
+
+                assert e instanceof SQLException : e.getClass().getName();
+            }
+            else
+                assert F.eq(errCls, e.getClass());
+
+            if (errMsg != null) {
+                assert e.getMessage() != null;
+                assert e.getMessage().startsWith(errMsg) : e.getMessage();
+            }
+        }
+    }
+
+    /**
+     * Execute certain logic for all nodes.
+     *
+     * @param task Task.
+     * @throws Exception If failed.
+     */
+    protected void forAllNodes(IgniteInClosureX task) throws Exception {
+        for (Ignite node : Ignition.allGrids()) {
+            try {
+                info("");
+                info(">>> Executing test on node: " + node.name());
+
+                task.applyx(node);
+            }
+            finally {
+                for (Ignite node0 : Ignition.allGrids()) {
+                    Collection cacheNames = node0.cacheNames();
+
+                    for (String cacheName : cacheNames)
+                        destroyCache(node0, cacheName);
+                }
+            }
+        }
+    }
+
+    /**
+     * Create node configuration.
+     *
+     * @param name Name.
+     * @param cli Client flag.
+     * @param filter Whether node should be filtered out.
+     * @return Node configuration.
+     */
+    protected IgniteConfiguration config(String name, boolean cli, boolean filter) {
+        IgniteConfiguration cfg = new IgniteConfiguration();
+
+        cfg.setIgniteInstanceName(name);
+        cfg.setClientMode(cli);
+        cfg.setLocalHost("127.0.0.1");
+
+        cfg.setDiscoverySpi(new IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        DataRegionConfiguration regionCfg = new DataRegionConfiguration().setPersistenceEnabled(true);
+
+        DataRegionConfiguration volatileRegionCfg = new DataRegionConfiguration().setName(REGION_VOLATILE)
+            .setPersistenceEnabled(false);
+
+        DataStorageConfiguration storageCfg = new DataStorageConfiguration();
+
+        storageCfg.setDefaultDataRegionConfiguration(regionCfg);
+        storageCfg.setDataRegionConfigurations(volatileRegionCfg);
+
+        cfg.setDataStorageConfiguration(storageCfg);
+
+        if (filter)
+            cfg.setUserAttributes(Collections.singletonMap(FILTER_ATTR, true));
+
+        return cfg;
+    }
+
+    /**
+     * Create common cache configuration (default name, transactional).
+     *
+     * @param mode Mode.
+     * @return Cache configuration.
+     */
+    protected CacheConfiguration cacheConfig(CacheMode mode) {
+        return cacheConfig(CACHE_NAME, mode, TRANSACTIONAL);
+    }
+
+    /**
+     * Create cache configuration.
+     *
+     * @param name Name.
+     * @param mode Mode.
+     * @param atomicityMode Atomicity mode.
+     * @return Cache configuration.
+     */
+    protected CacheConfiguration cacheConfig(String name, CacheMode mode, CacheAtomicityMode atomicityMode) {
+        CacheConfiguration ccfg = new CacheConfiguration();
+
+        ccfg.setName(name);
+        ccfg.setCacheMode(mode);
+        ccfg.setAtomicityMode(atomicityMode);
+
+        ccfg.setNodeFilter(FILTER);
+
+        return ccfg;
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void deleteWorkFiles() throws IgniteCheckedException {
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+    }
+
+    /**
+     * Cache node filter.
+     */
+    protected static class CacheNodeFilter implements IgnitePredicate {
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode node) {
+            Object filterAttr = node.attribute(FILTER_ATTR);
+
+            return filterAttr == null;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCoordinatorNotAffinityNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCoordinatorNotAffinityNodeSelfTest.java
new file mode 100644
index 0000000000000..0394a08687196
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCoordinatorNotAffinityNodeSelfTest.java
@@ -0,0 +1,30 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+/**
+ * Tests for WAL mode change feature when coordinator node is not affinity node for cache.
+ */
+public class WalModeChangeCoordinatorNotAffinityNodeSelfTest extends WalModeChangeAbstractSelfTest {
+    /**
+     * Constructor.
+     */
+    public WalModeChangeCoordinatorNotAffinityNodeSelfTest() {
+        super(true, false);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeSelfTest.java
new file mode 100644
index 0000000000000..7bbb8ba0b1c0c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeSelfTest.java
@@ -0,0 +1,30 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+/**
+ * Tests for WAL mode change feature when coordinator node is ordinary cache node.
+ */
+public class WalModeChangeSelfTest extends WalModeChangeAbstractSelfTest {
+    /**
+     * Constructor.
+     */
+    public WalModeChangeSelfTest() {
+        super(false, false);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
index d6bfe107e70bb..13e811cf26476 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
@@ -53,6 +53,7 @@ public class BPlusTreePageMemoryImplTest extends BPlusTreeSelfTest {
             null,
             new NoOpPageStoreManager(),
             new NoOpWALManager(),
+            null,
             new IgniteCacheDatabaseSharedManager(),
             null,
             null,
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
index dabd5320d21a6..ce5deba3da961 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
@@ -54,6 +54,7 @@ public class BPlusTreeReuseListPageMemoryImplTest extends BPlusTreeReuseSelfTest
             null,
             new NoOpPageStoreManager(),
             new NoOpWALManager(),
+            null,
             new IgniteCacheDatabaseSharedManager(),
             null,
             null,
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
index 4b5dbc943643a..effbe7e746dc5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
@@ -69,6 +69,7 @@ public class IndexStoragePageMemoryImplTest extends IndexStorageSelfTest {
             null,
             new NoOpPageStoreManager(),
             new NoOpWALManager(),
+            null,
             new IgniteCacheDatabaseSharedManager(),
             null,
             null,
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index 39a794dc4b846..64acf0211ddcf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -166,6 +166,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void onReconnected(boolean active) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void printMemoryStats() {
         // No-op.
@@ -186,6 +191,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public void beforeCacheGroupStart(CacheGroupDescriptor grpDesc) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void onActivate(GridKernalContext kctx) {
         // No-op.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 00f3f8fc92a04..6a35c8ac5830f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -86,6 +86,11 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean disabled(int grpId) {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException {
         // No-op.
@@ -106,6 +111,11 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void onReconnected(boolean active) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void printMemoryStats() {
         // No-op.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
index db6d321e1b75d..3d806b10c4922 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
@@ -58,6 +58,7 @@ public class PageMemoryImplNoLoadTest extends PageMemoryNoLoadSelfTest {
             null,
             new NoOpPageStoreManager(),
             new NoOpWALManager(),
+            null,
             new IgniteCacheDatabaseSharedManager(),
             null,
             null,
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
index bcdc6bb44413b..5c91c596093a9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -91,6 +91,7 @@ private PageMemoryImpl createPageMemory() throws Exception {
             null,
             new NoOpPageStoreManager(),
             new NoOpWALManager(),
+            null,
             new IgniteCacheDatabaseSharedManager(),
             null,
             null,
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
index 6a1d4f4668843..393395393c15c 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager;
 import org.apache.ignite.internal.processors.cache.GridCacheTtlManager;
+import org.apache.ignite.internal.processors.cache.WalStateManager;
 import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
 import org.apache.ignite.internal.processors.cache.dr.GridOsCacheDrManager;
 import org.apache.ignite.internal.processors.cache.jta.CacheNoopJtaManager;
@@ -69,6 +70,7 @@ public GridCacheTestContext(GridTestKernalContext ctx) throws Exception {
                 new GridCacheMvccManager(),
                 null,
                 null,
+                new WalStateManager(null),
                 new IgniteCacheDatabaseSharedManager(),
                 new IgniteCacheSnapshotManager(),
                 new GridCacheDeploymentManager(),
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
index 075ab9165decc..266c1f1800932 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
@@ -165,6 +165,21 @@ public IgniteClusterProcessProxy(IgniteProcessProxy proxy) {
         throw new UnsupportedOperationException("Operation is not supported yet.");
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean enableWal(String cacheName) throws IgniteException {
+        throw new UnsupportedOperationException("Operation is not supported yet.");
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean disableWal(String cacheName) throws IgniteException {
+        throw new UnsupportedOperationException("Operation is not supported yet.");
+    }
+    
+    /** {@inheritDoc} */
+    @Override public boolean isWalEnabled(String cacheName) {
+        throw new UnsupportedOperationException("Operation is not supported yet.");
+    }
+
     /** {@inheritDoc} */
     @Override public boolean isAsync() {
         throw new UnsupportedOperationException("Operation is not supported yet.");
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 8ad2e38406b1d..43a49fad5783e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -18,6 +18,9 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.WalModeChangeAdvancedSelfTest;
+import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
+import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest;
 import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest;
@@ -57,6 +60,10 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgnitePdsCacheAssignmentNodeRestartsTest.class);
 
+        suite.addTestSuite(WalModeChangeSelfTest.class);
+        suite.addTestSuite(WalModeChangeCoordinatorNotAffinityNodeSelfTest.class);
+        suite.addTestSuite(WalModeChangeAdvancedSelfTest.class);
+
 //        TODO enable this test after IGNITE-6753, now it takes too long
 //        suite.addTestSuite(IgniteOutOfMemoryPropagationTest.class);
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 38a04c4a9328e..1784496db322e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -118,6 +118,7 @@
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.sql.SqlParseException;
 import org.apache.ignite.internal.sql.SqlParser;
+import org.apache.ignite.internal.sql.command.SqlAlterTableCommand;
 import org.apache.ignite.internal.sql.command.SqlCommand;
 import org.apache.ignite.internal.sql.command.SqlCreateIndexCommand;
 import org.apache.ignite.internal.sql.command.SqlDropIndexCommand;
@@ -1384,7 +1385,9 @@ UpdateResult runDistributedUpdate(
      */
     private List>> tryQueryDistributedSqlFieldsNative(String schemaName, SqlFieldsQuery qry) {
         // Heuristic check for fast return.
-        if (!qry.getSql().toUpperCase().contains("INDEX"))
+        String sqlUpper = qry.getSql().toUpperCase();
+
+        if (!(sqlUpper.contains("INDEX") || sqlUpper.contains("ALTER")))
             return null;
 
         // Parse.
@@ -1399,8 +1402,9 @@ private List>> tryQueryDistributedSqlFieldsNative(Stri
             if (parser.nextCommand() != null)
                 return null;
 
-            // Only CREATE/DROP INDEX is supported for now.
-            if (!(cmd instanceof SqlCreateIndexCommand || cmd instanceof SqlDropIndexCommand))
+            // Only CREATE/DROP INDEX and ALTER TABLE commands are supported for now.
+            if (!(cmd instanceof SqlCreateIndexCommand || cmd instanceof SqlDropIndexCommand ||
+                cmd instanceof SqlAlterTableCommand))
                 return null;
         }
         catch (Exception e) {
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index 01629ce1965f3..67137d4b6efab 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.Set;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCluster;
 import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.cache.QueryIndexType;
@@ -53,6 +54,7 @@
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
 import org.apache.ignite.internal.processors.query.schema.SchemaOperationException;
+import org.apache.ignite.internal.sql.command.SqlAlterTableCommand;
 import org.apache.ignite.internal.sql.command.SqlCommand;
 import org.apache.ignite.internal.sql.command.SqlCreateIndexCommand;
 import org.apache.ignite.internal.sql.command.SqlDropIndexCommand;
@@ -159,6 +161,35 @@ else if (cmd instanceof SqlDropIndexCommand) {
                             cmd0.indexName());
                 }
             }
+            else if (cmd instanceof SqlAlterTableCommand) {
+                SqlAlterTableCommand cmd0 = (SqlAlterTableCommand)cmd;
+
+                GridH2Table tbl = idx.dataTable(cmd0.schemaName(), cmd0.tableName());
+
+                if (tbl == null) {
+                    ctx.cache().createMissingQueryCaches();
+
+                    tbl = idx.dataTable(cmd0.schemaName(), cmd0.tableName());
+                }
+
+                if (tbl == null) {
+                    throw new SchemaOperationException(SchemaOperationException.CODE_TABLE_NOT_FOUND,
+                        cmd0.tableName());
+                }
+
+                Boolean logging = cmd0.logging();
+
+                assert logging != null : "Only LOGGING/NOLOGGING are supported at the moment.";
+
+                IgniteCluster cluster = ctx.grid().cluster();
+
+                if (logging)
+                    cluster.enableWal(tbl.cacheName());
+                else
+                    cluster.disableWal(tbl.cacheName());
+
+                fut = new GridFinishedFuture();
+            }
             else
                 throw new IgniteSQLException("Unsupported DDL operation: " + sql,
                     IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
@@ -180,7 +211,7 @@ else if (cmd instanceof SqlDropIndexCommand) {
             throw e;
         }
         catch (Exception e) {
-            throw new IgniteSQLException("Unexpected DDL operation failure: " + e.getMessage(), e);
+            throw new IgniteSQLException(e.getMessage(), e);
         }
     }
 
@@ -441,7 +472,7 @@ else if (stmt0 instanceof GridSqlAlterTableDropColumn) {
             throw e;
         }
         catch (Exception e) {
-            throw new IgniteSQLException("Unexpected DDL operation failure: " + e.getMessage(), e);
+            throw new IgniteSQLException(e.getMessage(), e);
         }
     }
 

From 7fdc979555f96809cc7c901e493f15224326e576 Mon Sep 17 00:00:00 2001
From: Alexander Paschenko 
Date: Mon, 22 Jan 2018 22:25:30 +0300
Subject: [PATCH 31/65] IGNITE-5571: Decoupled SQL fields query from cache.
 This closes #2283.

---
 .../jdbc/suite/IgniteJdbcDriverTestSuite.java |   3 +
 .../jdbc/thin/JdbcThinAbstractSelfTest.java   |  79 +++
 .../thin/JdbcThinLocalQueriesSelfTest.java    |  73 ++
 .../JdbcQueryMultipleStatementsTask.java      |   2 +-
 .../cache/IgniteCacheProxyImpl.java           |   2 +-
 .../odbc/jdbc/JdbcRequestHandler.java         |   4 +-
 .../odbc/odbc/OdbcRequestHandler.java         |   7 +-
 .../ClientCacheSqlFieldsQueryRequest.java     |   8 +-
 .../processors/query/GridQueryIndexing.java   |  18 +-
 .../processors/query/GridQueryProcessor.java  | 136 +---
 .../internal/processors/query/QueryUtils.java |   6 +-
 .../internal/visor/query/VisorQueryTask.java  |   2 +-
 ...niteClientCacheInitializationFailTest.java |   7 +-
 .../query/h2/DmlStatementsProcessor.java      |  21 +-
 ...PlanKey.java => H2CachedStatementKey.java} |  39 +-
 .../processors/query/h2/H2StatementCache.java |  28 +-
 .../processors/query/h2/IgniteH2Indexing.java | 629 ++++++++++++------
 .../processors/query/h2/ParsingResult.java    | 102 +++
 .../query/h2/ddl/DdlStatementsProcessor.java  |  24 +
 .../query/h2/sql/GridSqlQueryParser.java      |  60 ++
 .../h2/twostep/GridMapQueryExecutor.java      |   2 +-
 .../GridCacheCrossCacheQuerySelfTest.java     |  12 +-
 ...eckClusterStateBeforeExecuteQueryTest.java |   2 +-
 .../cache/SqlFieldsQuerySelfTest.java         |  53 ++
 .../IgniteCacheReplicatedQuerySelfTest.java   |  29 +-
 .../cache/index/AbstractSchemaSelfTest.java   |   3 +
 .../index/DynamicColumnsAbstractTest.java     |   2 +-
 .../DynamicIndexAbstractBasicSelfTest.java    |  29 +-
 .../index/DynamicIndexAbstractSelfTest.java   |  33 +-
 .../index/H2DynamicIndexingComplexTest.java   |   2 +-
 .../cache/index/H2DynamicTableSelfTest.java   |   8 +-
 .../local/IgniteCacheLocalQuerySelfTest.java  |  62 +-
 .../IgnitePersistentStoreSchemaLoadTest.java  |  12 +-
 .../query/IgniteCachelessQueriesSelfTest.java | 420 ++++++++++++
 .../query/IgniteQueryDedicatedPoolTest.java   |  15 +-
 .../query/IgniteSqlDefaultValueTest.java      |   2 +-
 .../query/IgniteSqlNotNullConstraintTest.java |   2 +-
 .../MultipleStatementsSqlQuerySelfTest.java   |   8 +-
 .../processors/query/SqlSchemaSelfTest.java   |  12 +-
 .../IgniteCacheQuerySelfTestSuite.java        |   2 +
 .../Client/Cache/SqlQueryTest.cs              |   3 +-
 41 files changed, 1507 insertions(+), 456 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinLocalQueriesSelfTest.java
 rename modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/{H2DmlPlanKey.java => H2CachedStatementKey.java} (65%)
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ParsingResult.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteCachelessQueriesSelfTest.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index 87eeb84d7556f..ff4d69f14c702 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -48,6 +48,7 @@
 import org.apache.ignite.jdbc.thin.JdbcThinEmptyCacheSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinErrorsSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSelfTest;
+import org.apache.ignite.jdbc.thin.JdbcThinLocalQueriesSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinMetadataSelfTest;
 import org.apache.ignite.jdbc.thin.JdbcThinMissingLongArrayResultsTest;
@@ -164,6 +165,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.class));
         suite.addTest(new TestSuite(JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.class));
 
+        suite.addTest(new TestSuite(JdbcThinLocalQueriesSelfTest.class));
+
         // Various commands.
         suite.addTest(new TestSuite(JdbcThinWalModeChangeSelfTest.class));
 
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractSelfTest.java
index 1c38df21fdb03..178cd3afba9d4 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractSelfTest.java
@@ -17,11 +17,26 @@
 
 package org.apache.ignite.jdbc.thin;
 
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.SQLFeatureNotSupportedException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProcessor;
+import org.apache.ignite.internal.processors.port.GridPortRecord;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Connection test.
@@ -93,4 +108,68 @@ interface RunnableX {
          */
         void run() throws Exception;
     }
+
+    /**
+     * @param node Node to connect to.
+     * @param params Connection parameters.
+     * @return Thin JDBC connection to specified node.
+     */
+    static Connection connect(IgniteEx node, String params) throws SQLException {
+        Collection recs = node.context().ports().records();
+
+        GridPortRecord cliLsnrRec = null;
+
+        for (GridPortRecord rec : recs) {
+            if (rec.clazz() == ClientListenerProcessor.class) {
+                cliLsnrRec = rec;
+
+                break;
+            }
+        }
+
+        assertNotNull(cliLsnrRec);
+
+        String connStr = "jdbc:ignite:thin://127.0.0.1:" + cliLsnrRec.port();
+
+        if (!F.isEmpty(params))
+            connStr += "/?" + params;
+
+        return DriverManager.getConnection(connStr);
+    }
+
+    /**
+     * @param sql Statement.
+     * @param args Arguments.
+     * @return Result set.
+     * @throws RuntimeException if failed.
+     */
+    static List> execute(Connection conn, String sql, Object... args) throws SQLException {
+        try (PreparedStatement s = conn.prepareStatement(sql)) {
+            for (int i = 0; i < args.length; i++)
+                s.setObject(i + 1, args[i]);
+
+            if (s.execute()) {
+                List> res = new ArrayList<>();
+
+                try (ResultSet rs = s.getResultSet()) {
+                    ResultSetMetaData meta = rs.getMetaData();
+
+                    int cnt = meta.getColumnCount();
+
+                    while (rs.next()) {
+                        List row = new ArrayList<>(cnt);
+
+                        for (int i = 1; i <= cnt; i++)
+                            row.add(rs.getObject(i));
+
+                        res.add(row);
+                    }
+                }
+
+                return res;
+            }
+            else
+                return Collections.emptyList();
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinLocalQueriesSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinLocalQueriesSelfTest.java
new file mode 100644
index 0000000000000..1e28e52d3e3d6
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinLocalQueriesSelfTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc.thin;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Test that replicated-only query is executed locally.
+ */
+public class JdbcThinLocalQueriesSelfTest extends JdbcThinAbstractSelfTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        startGrid(0);
+
+        startGrid(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     *
+     */
+    public void testLocalThinJdbcQuery() throws SQLException {
+        try (Connection c = connect(grid(0), "replicatedOnly=true")) {
+            execute(c, "CREATE TABLE Company(id int primary key, name varchar) WITH " +
+                "\"template=replicated,cache_name=Company\"");
+
+            execute(c, "CREATE TABLE Person(id int primary key, name varchar, companyid int) WITH " +
+                "\"template=replicated,cache_name=Person\"");
+
+            execute(c, "insert into Company(id, name) values (1, 'Apple')");
+
+            execute(c, "insert into Person(id, name, companyid) values (2, 'John', 1)");
+
+            List> res = execute(c, "SELECT p.id, p.name, c.name from Person p left join Company c on " +
+                "p.companyid = c.id");
+
+            assertEqualsCollections(F.asList(2, "John", "Apple"), res.get(0));
+
+            Map twoStepCache = U.field(grid(0).context().query().getIndexing(), "twoStepCache");
+
+            // No two step queries cached => local select.
+            assertEquals(0, twoStepCache.size());
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java
index f907525571a5f..3e5d57564c14f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java
@@ -123,7 +123,7 @@ public JdbcQueryMultipleStatementsTask(Ignite ignite, String schemaName, String
 
         GridKernalContext ctx = ((IgniteKernal)ignite).context();
 
-        List>> curs = ctx.query().querySqlFieldsNoCache(qry, true, false);
+        List>> curs = ctx.query().querySqlFields(qry, true, false);
 
         List resultsInfo = new ArrayList<>(curs.size());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index dc6793b2313e2..7f71c743769bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -604,7 +604,7 @@ private QueryCursor> queryContinuous(ContinuousQuery qry, bool
 
             if (qry instanceof SqlFieldsQuery)
                 return (FieldsQueryCursor)ctx.kernalContext().query().querySqlFields(ctx, (SqlFieldsQuery)qry,
-                    keepBinary);
+                    keepBinary, true).get(0);
 
             if (qry instanceof ScanQuery)
                 return query((ScanQuery)qry, null, projection(qry.isLocal()));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
index d0f007375d431..11b50ece95cc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java
@@ -307,7 +307,7 @@ private JdbcResponse executeQuery(JdbcQueryExecuteRequest req) {
 
             qry.setSchema(schemaName);
 
-            List>> results = ctx.query().querySqlFieldsNoCache(qry, true,
+            List>> results = ctx.query().querySqlFields(qry, true,
                 protocolVer.compareTo(VER_2_3_0) < 0);
 
             if (results.size() == 1) {
@@ -531,7 +531,7 @@ private ClientListenerResponse executeBatch(JdbcBatchExecuteRequest req) {
     private void executeBatchedQuery(SqlFieldsQueryEx qry, List updCntsAcc,
         IgniteBiTuple firstErr) {
         try {
-            List>> qryRes = ctx.query().querySqlFieldsNoCache(qry, true, true);
+            List>> qryRes = ctx.query().querySqlFields(qry, true, true);
 
             for (FieldsQueryCursor> cur : qryRes) {
                 assert !((QueryCursorImpl)cur).isQuery();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
index bfc8d9e77f445..4ce99e33df652 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
@@ -248,7 +248,7 @@ private ClientListenerResponse executeQuery(OdbcQueryExecuteRequest req) {
 
             SqlFieldsQuery qry = makeQuery(req.schema(), sql, req.arguments(), req.timeout());
 
-            List>> cursors = ctx.query().querySqlFieldsNoCache(qry, true, false);
+            List>> cursors = ctx.query().querySqlFields(qry, true, false);
 
             OdbcQueryResults results = new OdbcQueryResults(cursors);
 
@@ -299,7 +299,8 @@ private ClientListenerResponse executeBatchQuery(OdbcQueryExecuteBatchRequest re
             // Getting meta and do the checks for the first execution.
             qry.setArgs(paramSet[0]);
 
-            QueryCursorImpl> qryCur = (QueryCursorImpl>)ctx.query().querySqlFieldsNoCache(qry, true);
+            QueryCursorImpl> qryCur = (QueryCursorImpl>)ctx.query()
+                .querySqlFields(qry, true, true).get(0);
 
             if (qryCur.isQuery())
                 throw new IgniteException("Batching of parameters only supported for DML statements. [query=" +
@@ -330,7 +331,7 @@ private ClientListenerResponse executeBatchQuery(OdbcQueryExecuteBatchRequest re
     private long executeQuery(SqlFieldsQuery qry, Object[] row) {
         qry.setArgs(row);
 
-        QueryCursor> cur = ctx.query().querySqlFieldsNoCache(qry, true);
+        QueryCursor> cur = ctx.query().querySqlFields(qry, true, true).get(0);
 
         return OdbcUtils.rowsAffected(cur);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
index ca3595dec5a7e..cfd4498cb4852 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.platform.client.cache;
 
+import java.util.List;
+import java.util.concurrent.TimeUnit;
 import org.apache.ignite.cache.query.FieldsQueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.internal.binary.BinaryRawReaderEx;
@@ -28,9 +30,6 @@
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
 import org.apache.ignite.internal.processors.query.QueryUtils;
 
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
 /**
  * Sql query request.
  */
@@ -101,8 +100,7 @@ public ClientCacheSqlFieldsQueryRequest(BinaryRawReaderEx reader) {
                 }
             }
 
-            List>> curs = ctx.kernalContext().query()
-                    .querySqlFieldsNoCache(qry, true, true);
+            List>> curs = ctx.kernalContext().query().querySqlFields(qry, true, true);
 
             assert curs.size() == 1;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index 96242447263ab..6b425a1ce4a58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -67,28 +67,22 @@ public interface GridQueryIndexing {
      * @param cacheName Cache name.
      * @param qry Query.
      * @param keepBinary Keep binary flag.
-     * @param mainCacheId Main cache ID.    @return Cursor.
      * @throws IgniteCheckedException If failed.
      */
     public  QueryCursor> queryDistributedSql(String schemaName, String cacheName, SqlQuery qry,
-        boolean keepBinary, int mainCacheId) throws IgniteCheckedException;
+        boolean keepBinary) throws IgniteCheckedException;
 
     /**
-     * Parses SQL query into two step query and executes it.
-     *
+     * Detect whether SQL query should be executed in distributed or local manner and execute it.
      * @param schemaName Schema name.
      * @param qry Query.
      * @param keepBinary Keep binary flag.
-     * @param cancel Query cancel.
-     * @param mainCacheId Main cache ID.
-     * @param failOnMultipleStmts If {@code true} the method must throws exception when query contains
-     *      more then one SQL statement.
+     * @param failOnMultipleStmts Whether an exception should be thrown for multiple statements query.
+     * @param cancel Query cancel state handler.
      * @return Cursor.
-     * @throws IgniteCheckedException If failed.
      */
-    public List>> queryDistributedSqlFields(String schemaName, SqlFieldsQuery qry,
-        boolean keepBinary, GridQueryCancel cancel, @Nullable Integer mainCacheId, boolean failOnMultipleStmts)
-        throws IgniteCheckedException;
+    public List>> querySqlFields(String schemaName, SqlFieldsQuery qry, boolean keepBinary,
+        boolean failOnMultipleStmts, GridQueryCancel cancel);
 
     /**
      * Perform a MERGE statement using data streamer as receiver.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 9184a497a09f5..43bed4026ed2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -705,7 +705,8 @@ public void onCacheStart0(GridCacheContext cctx, QuerySchema schema)
                     }
                 }
 
-                // Ensure that candidates has unique index names. Otherwise we will not be able to apply pending operations.
+                // Ensure that candidates has unique index names.
+                // Otherwise we will not be able to apply pending operations.
                 Map tblTypMap = new HashMap<>();
                 Map idxTypMap = new HashMap<>();
 
@@ -1969,11 +1970,15 @@ private void checkxEnabled() throws IgniteException {
     /**
      * Query SQL fields.
      *
-     * @param cctx Cache context.
      * @param qry Query.
      * @param keepBinary Keep binary flag.
      * @return Cursor.
      */
+    public List>> querySqlFields(final SqlFieldsQuery qry, final boolean keepBinary,
+        final boolean failOnMultipleStmts) {
+        return querySqlFields(null, qry, keepBinary, failOnMultipleStmts);
+    }
+
     @SuppressWarnings("unchecked")
     public FieldsQueryCursor> querySqlFields(final GridCacheContext cctx, final SqlFieldsQuery qry,
         final boolean keepBinary) {
@@ -1983,141 +1988,71 @@ public FieldsQueryCursor> querySqlFields(final GridCacheContext cct
     /**
      * Query SQL fields.
      *
-     * @param cctx Cache context.
-     * @param qry Query.
-     * @param keepBinary Keep binary flag.
-     * @param failOnMultipleStmts If {@code true} the method must throws exception when query contains
-     *      more then one SQL statement.
-     * @return Cursor.
-     */
-    @SuppressWarnings("unchecked")
-    public List>> querySqlFields(final GridCacheContext cctx, final SqlFieldsQuery qry,
-        final boolean keepBinary, final boolean failOnMultipleStmts) {
-        checkxEnabled();
-
-        validateSqlFieldsQuery(qry);
-
-        boolean loc = (qry.isReplicatedOnly() && cctx.isReplicatedAffinityNode()) || cctx.isLocal() || qry.isLocal();
-
-        if (!busyLock.enterBusy())
-            throw new IllegalStateException("Failed to execute query (grid is stopping).");
-
-        GridCacheContext oldCctx = curCache.get();
-
-        curCache.set(cctx);
-
-        try {
-            final String schemaName = qry.getSchema() != null ? qry.getSchema() : idx.schema(cctx.name());
-            final int mainCacheId = cctx.cacheId();
-
-            IgniteOutClosureX>>> clo;
-
-            if (loc) {
-                clo = new IgniteOutClosureX>>>() {
-                    @Override public List>> applyx() throws IgniteCheckedException {
-                        GridQueryCancel cancel = new GridQueryCancel();
-
-                        List>> cursors;
-
-                        if (cctx.config().getQueryParallelism() > 1) {
-                            qry.setDistributedJoins(true);
-
-                            cursors = idx.queryDistributedSqlFields(schemaName, qry,
-                                keepBinary, cancel, mainCacheId, true);
-                        }
-                        else {
-                            IndexingQueryFilter filter = idx.backupFilter(requestTopVer.get(), qry.getPartitions());
-
-                            cursors = new ArrayList<>(1);
-
-                            cursors.add(idx.queryLocalSqlFields(schemaName, qry, keepBinary, filter, cancel));
-                        }
-
-                        sendQueryExecutedEvent(qry.getSql(), qry.getArgs(), cctx.name());
-
-                        return cursors;
-                    }
-                };
-            }
-            else {
-                clo = new IgniteOutClosureX>>>() {
-                    @Override public List>> applyx() throws IgniteCheckedException {
-                        return idx.queryDistributedSqlFields(schemaName, qry, keepBinary, null, mainCacheId, failOnMultipleStmts);
-                    }
-                };
-            }
-
-            return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), cctx, clo, true);
-        }
-        catch (IgniteCheckedException e) {
-            throw new CacheException(e);
-        }
-        finally {
-            curCache.set(oldCctx);
-
-            busyLock.leaveBusy();
-        }
-    }
-
-    /**
-     * Query SQL fields without strict dependency on concrete cache.
-     *
      * @param qry Query.
      * @param keepBinary Keep binary flag.
      * @return Cursor.
      */
-    public FieldsQueryCursor> querySqlFieldsNoCache(final SqlFieldsQuery qry,
-        final boolean keepBinary) {
-        return querySqlFieldsNoCache(qry, keepBinary, true).get(0);
+    public FieldsQueryCursor> querySqlFields(final SqlFieldsQuery qry, final boolean keepBinary) {
+        return querySqlFields(null, qry, keepBinary, true).get(0);
     }
 
     /**
-     * Query SQL fields without strict dependency on concrete cache.
+     * Query SQL fields.
      *
+     * @param cctx Cache context.
      * @param qry Query.
      * @param keepBinary Keep binary flag.
      * @param failOnMultipleStmts If {@code true} the method must throws exception when query contains
      *      more then one SQL statement.
      * @return Cursor.
      */
-    public List>> querySqlFieldsNoCache(final SqlFieldsQuery qry,
-        final boolean keepBinary, final boolean failOnMultipleStmts) {
+    @SuppressWarnings("unchecked")
+    public List>> querySqlFields(@Nullable final GridCacheContext cctx,
+        final SqlFieldsQuery qry, final boolean keepBinary, final boolean failOnMultipleStmts) {
         checkxEnabled();
 
         validateSqlFieldsQuery(qry);
 
-        if (qry.isLocal())
-            throw new IgniteException("Local query is not supported without specific cache.");
-
         if (!ctx.state().publicApiActiveState(true)) {
             throw new IgniteException("Can not perform the operation because the cluster is inactive. Note, that " +
                 "the cluster is considered inactive by default if Ignite Persistent Store is used to let all the nodes " +
                 "join the cluster. To activate the cluster call Ignite.active(true).");
         }
 
-        if (qry.getSchema() == null)
-            qry.setSchema(QueryUtils.DFLT_SCHEMA);
-
         if (!busyLock.enterBusy())
             throw new IllegalStateException("Failed to execute query (grid is stopping).");
 
+        GridCacheContext oldCctx = curCache.get();
+
+        curCache.set(cctx);
+
+        final String schemaName = qry.getSchema() != null ? qry.getSchema()
+            : (cctx != null ? idx.schema(cctx.name()) : QueryUtils.DFLT_SCHEMA);
+
         try {
             IgniteOutClosureX>>> clo =
                 new IgniteOutClosureX>>>() {
                 @Override public List>> applyx() throws IgniteCheckedException {
                     GridQueryCancel cancel = new GridQueryCancel();
 
-                    return idx.queryDistributedSqlFields(qry.getSchema(), qry, keepBinary, cancel, null,
-                        failOnMultipleStmts);
+                    List>> res =
+                        idx.querySqlFields(schemaName, qry, keepBinary, failOnMultipleStmts, cancel);
+
+                    if (cctx != null)
+                        sendQueryExecutedEvent(qry.getSql(), qry.getArgs(), cctx.name());
+
+                    return res;
                 }
             };
 
-            return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), null, clo, true);
+            return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), cctx, clo, true);
         }
         catch (IgniteCheckedException e) {
             throw new CacheException(e);
         }
         finally {
+            curCache.set(oldCctx);
+
             busyLock.leaveBusy();
         }
     }
@@ -2204,12 +2139,11 @@ private  QueryCursor> queryDistributedSql(final GridCacheC
 
         try {
             final String schemaName = idx.schema(cctx.name());
-            final int mainCacheId = cctx.cacheId();
 
             return executeQuery(GridCacheQueryType.SQL, qry.getSql(), cctx,
                 new IgniteOutClosureX>>() {
                     @Override public QueryCursor> applyx() throws IgniteCheckedException {
-                        return idx.queryDistributedSql(schemaName, cctx.name(), qry, keepBinary, mainCacheId);
+                        return idx.queryDistributedSql(schemaName, cctx.name(), qry, keepBinary);
                     }
                 }, true);
         }
@@ -2233,7 +2167,6 @@ private  QueryCursor> queryLocalSql(final GridCacheConte
             throw new IllegalStateException("Failed to execute query (grid is stopping).");
 
         final String schemaName = idx.schema(cctx.name());
-        final int mainCacheId = cctx.cacheId();
 
         try {
             return executeQuery(GridCacheQueryType.SQL, qry.getSql(), cctx,
@@ -2253,7 +2186,7 @@ private  QueryCursor> queryLocalSql(final GridCacheConte
                         if (cctx.config().getQueryParallelism() > 1) {
                             qry.setDistributedJoins(true);
 
-                            return idx.queryDistributedSql(schemaName, cctx.name(), qry, keepBinary, mainCacheId);
+                            return idx.queryDistributedSql(schemaName, cctx.name(), qry, keepBinary);
                         }
                         else
                             return idx.queryLocalSql(schemaName, cctx.name(), qry, idx.backupFilter(requestTopVer.get(),
@@ -2437,7 +2370,8 @@ private void sendQueryExecutedEvent(String sqlQry, Object[] params, String cache
      * @param cols Columns to add.
      * @throws IgniteCheckedException If failed to update type descriptor.
      */
-    private void processDynamicAddColumn(QueryTypeDescriptorImpl d, List cols) throws IgniteCheckedException {
+    private void processDynamicAddColumn(QueryTypeDescriptorImpl d, List cols)
+        throws IgniteCheckedException {
         List props = new ArrayList<>(cols.size());
 
         for (QueryField col : cols) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
index 33974928033ed..0cee8318731b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
@@ -388,8 +388,10 @@ public static QueryTypeCandidate typeForQueryEntity(String cacheName, String sch
 
         // Key and value classes still can be available if they are primitive or JDK part.
         // We need that to set correct types for _key and _val columns.
-        Class keyCls = U.classForName(qryEntity.findKeyType(), null);
-        Class valCls = U.classForName(qryEntity.findValueType(), null);
+        // We better box these types - otherwise, if user provides, say, raw 'byte' for
+        // key or value (which they could), we'll deem key or value as Object which clearly is not right.
+        Class keyCls = U.box(U.classForName(qryEntity.findKeyType(), null, true));
+        Class valCls = U.box(U.classForName(qryEntity.findValueType(), null, true));
 
         // If local node has the classes and they are externalizable, we must use reflection properties.
         boolean keyMustDeserialize = mustDeserializeBinary(ctx, keyCls);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java
index 1daa1f2bd9ca7..2e322763dc703 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java
@@ -90,7 +90,7 @@ private VisorQueryJob(VisorQueryTaskArg arg, boolean debug) {
                 String cacheName = arg.getCacheName();
 
                 if (F.isEmpty(cacheName))
-                    qryCursors = ignite.context().query().querySqlFieldsNoCache(qry, true, false);
+                    qryCursors = ignite.context().query().querySqlFields(qry, true, false);
                 else {
                     IgniteCache c = ignite.cache(cacheName);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
index 3f769d71107cb..df27c5fa59ba5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -238,14 +238,13 @@ private static class FailedIndexing implements GridQueryIndexing {
 
         /** {@inheritDoc} */
         @Override public  QueryCursor> queryDistributedSql(String schemaName, String cacheName,
-            SqlQuery qry, boolean keepBinary, int mainCacheId) throws IgniteCheckedException {
+            SqlQuery qry, boolean keepBinary) throws IgniteCheckedException {
             return null;
         }
 
         /** {@inheritDoc} */
-        @Override public List>> queryDistributedSqlFields(String schemaName,
-            SqlFieldsQuery qry, boolean keepBinary, GridQueryCancel cancel,
-            @Nullable Integer mainCacheId, boolean failOnMultipleStmts) throws IgniteCheckedException {
+        @Override public List>> querySqlFields(String schemaName, SqlFieldsQuery qry,
+            boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
             return null;
         }
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
index c06fad78658e8..72e80e2254d1a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
@@ -96,7 +96,7 @@ public class DmlStatementsProcessor {
     private static final int PLAN_CACHE_SIZE = 1024;
 
     /** Update plans cache. */
-    private final ConcurrentMap planCache =
+    private final ConcurrentMap planCache =
         new GridBoundedConcurrentLinkedHashMap<>(PLAN_CACHE_SIZE);
 
     /**
@@ -117,7 +117,7 @@ public void start(GridKernalContext ctx, IgniteH2Indexing idx) {
      * @param cacheName Cache name.
      */
     public void onCacheStop(String cacheName) {
-        Iterator> iter = planCache.entrySet().iterator();
+        Iterator> iter = planCache.entrySet().iterator();
 
         while (iter.hasNext()) {
             UpdatePlan plan = iter.next().getValue();
@@ -353,7 +353,7 @@ List>> updateSqlFieldsDistributed(String schemaName, Con
      *
      * @param schemaName Schema.
      * @param conn Connection.
-     * @param stmt Prepared statement.
+     * @param prepared H2 prepared command.
      * @param fieldsQry Fields query.
      * @param filters Cache name and key filter.
      * @param cancel Query cancel.
@@ -361,10 +361,10 @@ List>> updateSqlFieldsDistributed(String schemaName, Con
      * @throws IgniteCheckedException if failed.
      */
     @SuppressWarnings("unchecked")
-    GridQueryFieldsResult updateSqlFieldsLocal(String schemaName, Connection conn, PreparedStatement stmt,
+    GridQueryFieldsResult updateSqlFieldsLocal(String schemaName, Connection conn, Prepared prepared,
         SqlFieldsQuery fieldsQry, IndexingQueryFilter filters, GridQueryCancel cancel)
         throws IgniteCheckedException {
-        UpdateResult res = updateSqlFields(schemaName, conn, GridSqlQueryParser.prepared(stmt), fieldsQry, true,
+        UpdateResult res = updateSqlFields(schemaName, conn, prepared, fieldsQry, true,
             filters, cancel);
 
         return new GridQueryFieldsResultAdapter(UPDATE_RESULT_META,
@@ -474,10 +474,8 @@ long streamUpdateQuery(IgniteDataStreamer streamer, PreparedStatement stmt, fina
      */
     @SuppressWarnings({"ConstantConditions", "unchecked"})
     private UpdateResult executeUpdateStatement(String schemaName, final GridCacheContext cctx, Connection c,
-        Prepared prepared, SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel)
-        throws IgniteCheckedException {
-        int mainCacheId = cctx.cacheId();
-
+        Prepared prepared, SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters,
+        GridQueryCancel cancel) throws IgniteCheckedException {
         Integer errKeysPos = null;
 
         UpdatePlan plan = getPlanForStatement(schemaName, c, prepared, fieldsQry, loc, errKeysPos);
@@ -510,7 +508,8 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo
                 .setPageSize(fieldsQry.getPageSize())
                 .setTimeout(fieldsQry.getTimeout(), TimeUnit.MILLISECONDS);
 
-            cur = idx.queryDistributedSqlFields(schemaName, newFieldsQry, true, cancel, mainCacheId, true).get(0);
+            cur = (QueryCursorImpl>)idx.querySqlFields(schemaName, newFieldsQry, true, true,
+                cancel).get(0);
         }
         else if (plan.hasRows())
             cur = plan.createRows(fieldsQry.getArgs());
@@ -602,7 +601,7 @@ private UpdateResult processDmlSelectResult(GridCacheContext cctx, UpdatePlan pl
     @SuppressWarnings({"unchecked", "ConstantConditions"})
     private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared p, SqlFieldsQuery fieldsQry,
         boolean loc, @Nullable Integer errKeysPos) throws IgniteCheckedException {
-        H2DmlPlanKey planKey = new H2DmlPlanKey(schema, p.getSQL(), loc, fieldsQry);
+        H2CachedStatementKey planKey = H2CachedStatementKey.forDmlStatement(schema, p.getSQL(), fieldsQry, loc);
 
         UpdatePlan res = (errKeysPos == null ? planCache.get(planKey) : null);
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2CachedStatementKey.java
similarity index 65%
rename from modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java
rename to modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2CachedStatementKey.java
index 455b5e5a3b94d..7b43f52a9f6e5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2CachedStatementKey.java
@@ -23,9 +23,9 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
- * H2 DML plan key.
+ * H2 cached statement key.
  */
-public class H2DmlPlanKey {
+class H2CachedStatementKey {
     /** Schema name. */
     private final String schemaName;
 
@@ -41,11 +41,38 @@ public class H2DmlPlanKey {
      * @param schemaName Schema name.
      * @param sql SQL.
      */
-    public H2DmlPlanKey(String schemaName, String sql, boolean loc, SqlFieldsQuery fieldsQry) {
+    H2CachedStatementKey(String schemaName, String sql) {
+        this(schemaName, sql, null, false);
+    }
+
+    /**
+     * Build key with details relevant to DML plans cache.
+     *
+     * @param schemaName Schema name.
+     * @param sql SQL.
+     * @param fieldsQry Query with flags.
+     * @param loc DML {@code SELECT} Locality flag.
+     * @return Statement key.
+     * @see UpdatePlanBuilder
+     * @see DmlStatementsProcessor#getPlanForStatement
+     */
+    static H2CachedStatementKey forDmlStatement(String schemaName, String sql, SqlFieldsQuery fieldsQry, boolean loc) {
+        return new H2CachedStatementKey(schemaName, sql, fieldsQry, loc);
+    }
+
+    /**
+     * Full-fledged constructor.
+     *
+     * @param schemaName Schema name.
+     * @param sql SQL.
+     * @param fieldsQry Query with flags.
+     * @param loc DML {@code SELECT} Locality flag.
+     */
+    private H2CachedStatementKey(String schemaName, String sql, SqlFieldsQuery fieldsQry, boolean loc) {
         this.schemaName = schemaName;
         this.sql = sql;
 
-        if (loc || !UpdatePlanBuilder.isSkipReducerOnUpdateQuery(fieldsQry))
+        if (fieldsQry == null || loc || !UpdatePlanBuilder.isSkipReducerOnUpdateQuery(fieldsQry))
             this.flags = 0; // flags only relevant for server side updates.
         else {
             this.flags = (byte)(1 +
@@ -69,13 +96,13 @@ public H2DmlPlanKey(String schemaName, String sql, boolean loc, SqlFieldsQuery f
         if (o == null || getClass() != o.getClass())
             return false;
 
-        H2DmlPlanKey other = (H2DmlPlanKey)o;
+        H2CachedStatementKey other = (H2CachedStatementKey)o;
 
         return F.eq(sql, other.sql) && F.eq(schemaName, other.schemaName) && flags == other.flags;
     }
 
     /** {@inheritDoc} */
     @Override public String toString() {
-        return S.toString(H2DmlPlanKey.class, this);
+        return S.toString(H2CachedStatementKey.class, this);
     }
 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2StatementCache.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2StatementCache.java
index d39511202234f..673625f865d53 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2StatementCache.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2StatementCache.java
@@ -17,16 +17,16 @@
 
 package org.apache.ignite.internal.processors.query.h2;
 
-import org.apache.ignite.internal.util.typedef.internal.U;
-
 import java.sql.PreparedStatement;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Statement cache.
  */
-public class H2StatementCache extends LinkedHashMap {
+class H2StatementCache extends LinkedHashMap {
     /** */
     private int size;
 
@@ -43,7 +43,7 @@ public class H2StatementCache extends LinkedHashMap {
     }
 
     /** {@inheritDoc} */
-    @Override protected boolean removeEldestEntry(Map.Entry eldest) {
+    @Override protected boolean removeEldestEntry(Map.Entry eldest) {
         boolean rmv = size() > size;
 
         if (rmv) {
@@ -55,6 +55,16 @@ public class H2StatementCache extends LinkedHashMap {
         return rmv;
     }
 
+    /**
+     * Get statement for given schema and SQL.
+     * @param schemaName Schema name.
+     * @param sql SQL statement.
+     * @return Cached {@link PreparedStatement}, or {@code null} if none found.
+     */
+    @Nullable public PreparedStatement get(String schemaName, String sql) {
+        return get(new H2CachedStatementKey(schemaName, sql));
+    }
+
     /**
      * The timestamp of the last usage of the cache.
      *
@@ -70,4 +80,14 @@ public long lastUsage() {
     public void updateLastUsage() {
         lastUsage = U.currentTimeMillis();
     }
+
+    /**
+     * Remove statement for given schema and SQL.
+     * @param schemaName Schema name.
+     * @param sql SQL statement.
+     * @return Cached {@link PreparedStatement}, or {@code null} if none found.
+     */
+    @Nullable public PreparedStatement remove(String schemaName, String sql) {
+        return remove(new H2CachedStatementKey(schemaName, sql));
+    }
 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 1784496db322e..96b8935d87964 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -107,8 +107,10 @@
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuery;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
 import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor;
 import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
 import org.apache.ignite.internal.processors.query.h2.twostep.MapQueryLazyWorker;
@@ -156,6 +158,7 @@
 import org.h2.table.IndexColumn;
 import org.h2.tools.Server;
 import org.h2.util.JdbcUtils;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
@@ -377,6 +380,22 @@ public Connection connectionForSchema(String schema) {
         }
     }
 
+    /**
+     * @param c Connection.
+     * @param sql SQL.
+     * @return Cached prepared statement.
+     */
+    @SuppressWarnings("ConstantConditions")
+    @Nullable private PreparedStatement cachedStatement(Connection c, String sql) {
+        try {
+            return prepareStatement(c, sql, true, true);
+        }
+        catch (SQLException e) {
+            // We actually don't except anything SQL related here as we're supposed to work with cache only.
+            throw new AssertionError(e);
+        }
+    }
+
     /**
      * @param c Connection.
      * @param sql SQL.
@@ -384,34 +403,45 @@ public Connection connectionForSchema(String schema) {
      * @return Prepared statement.
      * @throws SQLException If failed.
      */
-    private PreparedStatement prepareStatement(Connection c, String sql, boolean useStmtCache) throws SQLException {
-        if (useStmtCache) {
-            Thread curThread = Thread.currentThread();
-
-            H2StatementCache cache = stmtCache.get(curThread);
-
-            if (cache == null) {
-                H2StatementCache cache0 = new H2StatementCache(PREPARED_STMT_CACHE_SIZE);
+    @SuppressWarnings("ConstantConditions")
+    @NotNull private PreparedStatement prepareStatement(Connection c, String sql, boolean useStmtCache)
+        throws SQLException {
+        return prepareStatement(c, sql, useStmtCache, false);
+    }
 
-                cache = stmtCache.putIfAbsent(curThread, cache0);
+    /**
+     * @param c Connection.
+     * @param sql SQL.
+     * @param useStmtCache If {@code true} uses statement cache.
+     * @param cachedOnly Whether parsing should be avoided if statement has not been found in cache.
+     * @return Prepared statement.
+     * @throws SQLException If failed.
+     */
+    @Nullable private PreparedStatement prepareStatement(Connection c, String sql, boolean useStmtCache,
+        boolean cachedOnly) throws SQLException {
+        // We can't avoid parsing and avoid using cache at the same time.
+        assert useStmtCache || !cachedOnly;
 
-                if (cache == null)
-                    cache = cache0;
-            }
+        if (useStmtCache) {
+            H2StatementCache cache = getStatementsCacheForCurrentThread();
 
-            cache.updateLastUsage();
+            H2CachedStatementKey key = new H2CachedStatementKey(c.getSchema(), sql);
 
-            PreparedStatement stmt = cache.get(sql);
+            PreparedStatement stmt = cache.get(key);
 
-            if (stmt != null && !stmt.isClosed() && !((JdbcStatement)stmt).isCancelled()) {
+            if (stmt != null && !stmt.isClosed() && !((JdbcStatement)stmt).isCancelled() &&
+                !GridSqlQueryParser.prepared(stmt).needRecompile()) {
                 assert stmt.getConnection() == c;
 
                 return stmt;
             }
 
+            if (cachedOnly)
+                return null;
+
             stmt = prepare0(c, sql);
 
-            cache.put(sql, stmt);
+            cache.put(key, stmt);
 
             return stmt;
         }
@@ -444,6 +474,28 @@ private PreparedStatement prepare0(Connection c, String sql) throws SQLException
             return c.prepareStatement(sql);
     }
 
+    /**
+     * @return {@link H2StatementCache} associated with current thread.
+     */
+    @NotNull private H2StatementCache getStatementsCacheForCurrentThread() {
+        Thread curThread = Thread.currentThread();
+
+        H2StatementCache cache = stmtCache.get(curThread);
+
+        if (cache == null) {
+            H2StatementCache cache0 = new H2StatementCache(PREPARED_STMT_CACHE_SIZE);
+
+            cache = stmtCache.putIfAbsent(curThread, cache0);
+
+            if (cache == null)
+                cache = cache0;
+        }
+
+        cache.updateLastUsage();
+
+        return cache;
+    }
+
     /** {@inheritDoc} */
     @Override public PreparedStatement prepareNativeStatement(String schemaName, String sql) throws SQLException {
         Connection conn = connectionForSchema(schemaName);
@@ -822,7 +874,7 @@ private void executeSql(String schemaName, String sql) throws IgniteCheckedExcep
      * @param inlineSize Index inline size.
      * @return Index.
      */
-    public GridH2IndexBase createSortedIndex(String name, GridH2Table tbl, boolean pk, List cols,
+    GridH2IndexBase createSortedIndex(String name, GridH2Table tbl, boolean pk, List cols,
         int inlineSize) {
         try {
             GridCacheContext cctx = tbl.cache();
@@ -878,7 +930,7 @@ public GridH2IndexBase createSortedIndex(String name, GridH2Table tbl, boolean p
      * @throws IgniteCheckedException If failed.
      */
     @SuppressWarnings("unchecked")
-    public GridQueryFieldsResult queryLocalSqlFields(final String schemaName, final String qry,
+    GridQueryFieldsResult queryLocalSqlFields(final String schemaName, final String qry,
         @Nullable final Collection params, final IndexingQueryFilter filter, boolean enforceJoinOrder,
         final int timeout, final GridQueryCancel cancel) throws IgniteCheckedException {
         final Connection conn = connectionForSchema(schemaName);
@@ -901,7 +953,7 @@ public GridQueryFieldsResult queryLocalSqlFields(final String schemaName, final
             fldsQry.setEnforceJoinOrder(enforceJoinOrder);
             fldsQry.setTimeout(timeout, TimeUnit.MILLISECONDS);
 
-            return dmlProc.updateSqlFieldsLocal(schemaName, conn, stmt, fldsQry, filter, cancel);
+            return dmlProc.updateSqlFieldsLocal(schemaName, conn, p, fldsQry, filter, cancel);
         }
         else if (DdlStatementsProcessor.isDdlStatement(p))
             throw new IgniteSQLException("DDL statements are supported for the whole cluster only",
@@ -1224,7 +1276,7 @@ public void bindParameters(PreparedStatement stmt,
      * @throws IgniteCheckedException If failed.
      */
     @SuppressWarnings("unchecked")
-    public  GridCloseableIterator> queryLocalSql(String schemaName, String cacheName,
+     GridCloseableIterator> queryLocalSql(String schemaName, String cacheName,
         final String qry, String alias, @Nullable final Collection params, String type,
         final IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException {
         final H2TableDescriptor tbl = tableDescriptor(schemaName, cacheName, type);
@@ -1314,7 +1366,7 @@ UpdateResult runDistributedUpdate(
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public  QueryCursor> queryDistributedSql(String schemaName, String cacheName,
-        SqlQuery qry, boolean keepBinary, int mainCacheId) {
+        SqlQuery qry, boolean keepBinary) {
         String type = qry.getType();
 
         H2TableDescriptor tblDesc = tableDescriptor(schemaName, cacheName, type);
@@ -1344,7 +1396,7 @@ UpdateResult runDistributedUpdate(
             fqry.setTimeout(qry.getTimeout(), TimeUnit.MILLISECONDS);
 
         final QueryCursor> res =
-            queryDistributedSqlFields(schemaName, fqry, keepBinary, null, mainCacheId, true).get(0);
+            querySqlFields(schemaName, fqry, keepBinary, true, null).get(0);
 
         final Iterable> converted = new Iterable>() {
             @Override public Iterator> iterator() {
@@ -1425,222 +1477,339 @@ private List>> tryQueryDistributedSqlFieldsNative(Stri
 
         // Execute.
         try {
-            List>> ress = new ArrayList<>(1);
-
             FieldsQueryCursor> res = ddlProc.runDdlStatement(qry.getSql(), cmd);
 
-            ress.add(res);
-
-            return ress;
+            return Collections.singletonList(res);
         }
         catch (IgniteCheckedException e) {
             throw new IgniteSQLException("Failed to execute DDL statement [stmt=" + qry.getSql() + ']', e);
         }
     }
 
+    /**
+     * Check expected statement type (when it is set by JDBC) and given statement type.
+     *
+     * @param qry Query.
+     * @param isQry {@code true} for select queries, otherwise (DML/DDL queries) {@code false}.
+     */
+    private void checkQueryType(SqlFieldsQuery qry, boolean isQry) {
+        Boolean qryFlag = qry instanceof SqlFieldsQueryEx ? ((SqlFieldsQueryEx) qry).isQuery() : null;
+
+        if (qryFlag != null && qryFlag != isQry)
+            throw new IgniteSQLException("Given statement type does not match that declared by JDBC driver",
+                IgniteQueryErrorCode.STMT_TYPE_MISMATCH);
+    }
+
+
     /** {@inheritDoc} */
-    @Override public List>> queryDistributedSqlFields(String schemaName, SqlFieldsQuery qry,
-        boolean keepBinary, GridQueryCancel cancel, @Nullable Integer mainCacheId, boolean failOnMultipleStmts) {
+    @SuppressWarnings("StringEquality")
+    @Override public List>> querySqlFields(String schemaName, SqlFieldsQuery qry,
+        boolean keepBinary, boolean failOnMultipleStmts, GridQueryCancel cancel) {
         List>> res = tryQueryDistributedSqlFieldsNative(schemaName, qry);
 
         if (res != null)
             return res;
 
-        Connection c = connectionForSchema(schemaName);
+        {
+            // First, let's check if we already have a two-step query for this statement...
+            H2TwoStepCachedQueryKey cachedQryKey = new H2TwoStepCachedQueryKey(schemaName, qry.getSql(),
+                qry.isCollocated(), qry.isDistributedJoins(), qry.isEnforceJoinOrder(), qry.isLocal());
 
-        final boolean enforceJoinOrder = qry.isEnforceJoinOrder();
-        final boolean distributedJoins = qry.isDistributedJoins();
-        final boolean grpByCollocated = qry.isCollocated();
+            H2TwoStepCachedQuery cachedQry;
 
-        final DistributedJoinMode distributedJoinMode = distributedJoinMode(qry.isLocal(), distributedJoins);
+            if ((cachedQry = twoStepCache.get(cachedQryKey)) != null) {
+                checkQueryType(qry, true);
 
-        String sqlQry = qry.getSql();
+                GridCacheTwoStepQuery twoStepQry = cachedQry.query().copy();
 
-        H2TwoStepCachedQueryKey cachedQryKey = new H2TwoStepCachedQueryKey(schemaName, sqlQry, grpByCollocated,
-            distributedJoins, enforceJoinOrder, qry.isLocal());
+                List meta = cachedQry.meta();
 
-        H2TwoStepCachedQuery cachedQry = twoStepCache.get(cachedQryKey);
+                res = Collections.singletonList(doRunDistributedQuery(schemaName, qry, twoStepQry, meta, keepBinary,
+                    cancel));
 
-        if (cachedQry != null) {
-            checkQueryType(qry, true);
+                if (!twoStepQry.explain())
+                    twoStepCache.putIfAbsent(cachedQryKey, new H2TwoStepCachedQuery(meta, twoStepQry.copy()));
 
-            GridCacheTwoStepQuery twoStepQry = cachedQry.query().copy();
+                return res;
+            }
+        }
 
-            List meta = cachedQry.meta();
+        {
+            // Second, let's check if we already have a parsed statement...
+            PreparedStatement cachedStmt;
+
+            if ((cachedStmt = cachedStatement(connectionForSchema(schemaName), qry.getSql())) != null) {
+                Prepared prepared = GridSqlQueryParser.prepared(cachedStmt);
 
-            return Collections.singletonList(executeTwoStepsQuery(schemaName, qry.getPageSize(), qry.getPartitions(),
-                qry.getArgs(), keepBinary, qry.isLazy(), qry.getTimeout(), cancel, sqlQry, enforceJoinOrder,
-                twoStepQry, meta));
+                // We may use this cached statement only for local queries and non queries.
+                if (qry.isLocal() || !prepared.isQuery())
+                    return (List>>)doRunPrepared(schemaName, prepared, qry, null, null,
+                            keepBinary, cancel);
+            }
         }
 
         res = new ArrayList<>(1);
 
-        Object[] argsOrig = qry.getArgs();
         int firstArg = 0;
-        Object[] args;
-        String remainingSql = sqlQry;
+
+        String remainingSql = qry.getSql();
 
         while (remainingSql != null) {
-            args = null;
-            GridCacheTwoStepQuery twoStepQry = null;
-            List meta;
+            ParsingResult parseRes = parseAndSplit(schemaName,
+                remainingSql != qry.getSql() ? cloneFieldsQuery(qry).setSql(remainingSql) : qry, firstArg);
 
-            final UUID locNodeId = ctx.localNodeId();
+            // Let's avoid second reflection getter call by returning Prepared object too
+            Prepared prepared = parseRes.prepared();
 
-            // Here we will just parse the statement, no need to optimize it at all.
-            H2Utils.setupConnection(c, /*distributedJoins*/false, /*enforceJoinOrder*/true);
+            GridCacheTwoStepQuery twoStepQry = parseRes.twoStepQuery();
 
-            GridH2QueryContext.set(new GridH2QueryContext(locNodeId, locNodeId, 0, PREPARE)
-                .distributedJoinMode(distributedJoinMode));
+            List meta = parseRes.meta();
 
-            PreparedStatement stmt = null;
-            Prepared prepared;
+            SqlFieldsQuery newQry = parseRes.newQuery();
 
-            boolean cachesCreated = false;
+            remainingSql = parseRes.remainingSql();
 
-            try {
+            if (remainingSql != null && failOnMultipleStmts)
+                throw new IgniteSQLException("Multiple statements queries are not supported");
+
+            firstArg += prepared.getParameters().size();
+
+            res.addAll(doRunPrepared(schemaName, prepared, newQry, twoStepQry, meta, keepBinary, cancel));
+
+            if (parseRes.twoStepQuery() != null && parseRes.twoStepQueryKey() != null &&
+                    !parseRes.twoStepQuery().explain())
+                twoStepCache.putIfAbsent(parseRes.twoStepQueryKey(), new H2TwoStepCachedQuery(meta, twoStepQry.copy()));
+        }
+
+        return res;
+    }
+
+    /**
+     * Execute an all-ready {@link SqlFieldsQuery}.
+     * @param schemaName Schema name.
+     * @param prepared H2 command.
+     * @param qry Fields query with flags.
+     * @param twoStepQry Two-step query if this query must be executed in a distributed way.
+     * @param meta Metadata for {@code twoStepQry}.
+     * @param keepBinary Whether binary objects must not be deserialized automatically.
+     * @param cancel Query cancel state holder.
+     * @return Query result.
+     */
+    private List>> doRunPrepared(String schemaName, Prepared prepared,
+        SqlFieldsQuery qry, GridCacheTwoStepQuery twoStepQry, List meta, boolean keepBinary,
+        GridQueryCancel cancel) {
+        String sqlQry = qry.getSql();
+
+        boolean loc = qry.isLocal();
+
+        IndexingQueryFilter filter = (loc ? backupFilter(null, qry.getPartitions()) : null);
+
+        if (!prepared.isQuery()) {
+            if (DmlStatementsProcessor.isDmlStatement(prepared)) {
                 try {
-                    while (true) {
-                        try {
-                            // Do not cache this statement because the whole query object will be cached later on.
-                            stmt = prepareStatement(c, remainingSql, false);
+                    Connection conn = connectionForSchema(schemaName);
 
-                            break;
-                        }
-                        catch (SQLException e) {
-                            if (!cachesCreated && (
-                                e.getErrorCode() == ErrorCode.SCHEMA_NOT_FOUND_1 ||
-                                    e.getErrorCode() == ErrorCode.TABLE_OR_VIEW_NOT_FOUND_1 ||
-                                    e.getErrorCode() == ErrorCode.INDEX_NOT_FOUND_1)
-                                ) {
+                    if (!loc)
+                        return dmlProc.updateSqlFieldsDistributed(schemaName, conn, prepared, qry, cancel);
+                    else {
+                        final GridQueryFieldsResult updRes =
+                            dmlProc.updateSqlFieldsLocal(schemaName, conn, prepared, qry, filter, cancel);
+
+                        return Collections.singletonList(new QueryCursorImpl<>(new Iterable>() {
+                            @Override public Iterator> iterator() {
                                 try {
-                                    ctx.cache().createMissingQueryCaches();
+                                    return new GridQueryCacheObjectsIterator(updRes.iterator(), objectContext(),
+                                        true);
                                 }
-                                catch (IgniteCheckedException ignored) {
-                                    throw new CacheException("Failed to create missing caches.", e);
+                                catch (IgniteCheckedException e) {
+                                    throw new IgniteException(e);
                                 }
-
-                                cachesCreated = true;
                             }
-                            else
-                                throw new IgniteSQLException("Failed to parse query. " + e.getMessage(),
-                                    IgniteQueryErrorCode.PARSING, e);
-                        }
+                        }, cancel));
                     }
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteSQLException("Failed to execute DML statement [stmt=" + sqlQry +
+                        ", params=" + Arrays.deepToString(qry.getArgs()) + "]", e);
+                }
+            }
 
-                    GridSqlQueryParser.PreparedWithRemaining prep = GridSqlQueryParser.preparedWithRemaining(stmt);
+            if (DdlStatementsProcessor.isDdlStatement(prepared)) {
+                if (loc)
+                    throw new IgniteSQLException("DDL statements are not supported for LOCAL caches",
+                        IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
 
-                    // remaining == null if the query string contains single SQL statement.
-                    remainingSql = prep.remainingSql();
+                try {
+                    return Collections.singletonList(ddlProc.runDdlStatement(sqlQry, prepared));
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteSQLException("Failed to execute DDL statement [stmt=" + sqlQry + ']', e);
+                }
+            }
 
-                    if (remainingSql != null && failOnMultipleStmts)
-                        throw new IgniteSQLException("Multiple statements queries are not supported");
+            if (prepared instanceof NoOperation) {
+                QueryCursorImpl> resCur = (QueryCursorImpl>)new QueryCursorImpl(
+                    Collections.singletonList(Collections.singletonList(0L)), null, false);
 
-                    sqlQry = prep.prepared().getSQL();
+                resCur.fieldsMeta(UPDATE_RESULT_META);
 
-                    prepared = prep.prepared();
+                return Collections.singletonList(resCur);
+            }
 
-                    int paramsCnt = prepared.getParameters().size();
+            throw new IgniteSQLException("Unsupported DDL/DML operation: " + prepared.getClass().getName());
+        }
 
-                    if (!DmlUtils.isBatched(qry) && paramsCnt > 0) {
-                        if (argsOrig == null || argsOrig.length < firstArg + paramsCnt) {
-                            throw new IgniteException("Invalid number of query parameters. " +
-                                "Cannot find " + (argsOrig.length + 1 - firstArg) + " parameter.");
-                        }
+        if (twoStepQry != null) {
+            if (log.isDebugEnabled())
+                log.debug("Parsed query: `" + sqlQry + "` into two step query: " + twoStepQry);
 
-                        args = Arrays.copyOfRange(argsOrig, firstArg, firstArg + paramsCnt);
+            checkQueryType(qry, true);
 
-                        firstArg += paramsCnt;
-                    }
+            return Collections.singletonList(doRunDistributedQuery(schemaName, qry, twoStepQry, meta, keepBinary,
+                cancel));
+        }
+
+        // We've encountered a local query, let's just run it.
+        try {
+            return Collections.singletonList(queryLocalSqlFields(schemaName, qry, keepBinary, filter, cancel));
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteSQLException("Failed to execute local statement [stmt=" + sqlQry +
+                ", params=" + Arrays.deepToString(qry.getArgs()) + "]", e);
+        }
+    }
+
+    /**
+     * Parse and split query if needed, cache either two-step query or statement.
+     * @param schemaName Schema name.
+     * @param qry Query.
+     * @param firstArg Position of the first argument of the following {@code Prepared}.
+     * @return Result: prepared statement, H2 command, two-step query (if needed),
+     *     metadata for two-step query (if needed), evaluated query local execution flag.
+     */
+    private ParsingResult parseAndSplit(String schemaName, SqlFieldsQuery qry, int firstArg) {
+        Connection c = connectionForSchema(schemaName);
 
-                    cachedQryKey = new H2TwoStepCachedQueryKey(schemaName, sqlQry, grpByCollocated,
-                        distributedJoins, enforceJoinOrder, qry.isLocal());
+        // For queries that are explicitly local, we rely on the flag specified in the query
+        // because this parsing result will be cached and used for queries directly.
+        // For other queries, we enforce join order at this stage to avoid premature optimizations
+        // (and therefore longer parsing) as long as there'll be more parsing at split stage.
+        boolean enforceJoinOrderOnParsing = (!qry.isLocal() || qry.isEnforceJoinOrder());
 
-                    cachedQry = twoStepCache.get(cachedQryKey);
+        H2Utils.setupConnection(c, /*distributedJoins*/false, /*enforceJoinOrder*/enforceJoinOrderOnParsing);
 
-                    if (cachedQry != null) {
-                        checkQueryType(qry, true);
+        boolean loc = qry.isLocal();
 
-                        twoStepQry = cachedQry.query().copy();
-                        meta = cachedQry.meta();
+        PreparedStatement stmt = prepareStatementAndCaches(c, qry.getSql());
 
-                        res.add(executeTwoStepsQuery(schemaName, qry.getPageSize(), qry.getPartitions(), args, keepBinary,
-                            qry.isLazy(), qry.getTimeout(), cancel, sqlQry, enforceJoinOrder,
-                            twoStepQry, meta));
+        if (loc && GridSqlQueryParser.checkMultipleStatements(stmt))
+            throw new IgniteSQLException("Multiple statements queries are not supported for local queries");
 
-                        continue;
-                    }
-                    else {
-                        checkQueryType(qry, prepared.isQuery());
+        GridSqlQueryParser.PreparedWithRemaining prep = GridSqlQueryParser.preparedWithRemaining(stmt);
 
-                        if (prepared.isQuery()) {
-                            bindParameters(stmt, F.asList(args));
+        Prepared prepared = prep.prepared();
 
-                            twoStepQry = GridSqlQuerySplitter.split(c, prepared, args,
-                                grpByCollocated, distributedJoins, enforceJoinOrder, this);
+        checkQueryType(qry, prepared.isQuery());
 
-                            assert twoStepQry != null;
-                        }
-                    }
-                }
-                finally {
-                    GridH2QueryContext.clearThreadLocal();
-                }
+        String remainingSql = prep.remainingSql();
 
-                // It is a DML statement if we did not create a twoStepQuery.
-                if (twoStepQry == null) {
-                    if (DmlStatementsProcessor.isDmlStatement(prepared)) {
-                        try {
-                            res.addAll(dmlProc.updateSqlFieldsDistributed(schemaName, c, prepared,
-                                qry.copy().setSql(sqlQry).setArgs(args), cancel));
+        int paramsCnt = prepared.getParameters().size();
 
-                            continue;
-                        }
-                        catch (IgniteCheckedException e) {
-                            throw new IgniteSQLException("Failed to execute DML statement [stmt=" + sqlQry +
-                                ", params=" + Arrays.deepToString(args) + "]", e);
-                        }
-                    }
+        Object[] argsOrig = qry.getArgs();
 
-                    if (DdlStatementsProcessor.isDdlStatement(prepared)) {
-                        try {
-                            res.add(ddlProc.runDdlStatement(sqlQry, prepared));
+        Object[] args = null;
 
-                            continue;
-                        }
-                        catch (IgniteCheckedException e) {
-                            throw new IgniteSQLException("Failed to execute DDL statement [stmt=" + sqlQry + ']', e);
-                        }
-                    }
+        if (!DmlUtils.isBatched(qry) && paramsCnt > 0) {
+            if (argsOrig == null || argsOrig.length < firstArg + paramsCnt) {
+                throw new IgniteException("Invalid number of query parameters. " +
+                    "Cannot find " + (argsOrig != null ? argsOrig.length + 1 - firstArg : 1) + " parameter.");
+            }
 
-                    if (prepared instanceof NoOperation) {
-                        QueryCursorImpl> resCur = (QueryCursorImpl>)new QueryCursorImpl(
-                            Collections.singletonList(Collections.singletonList(0L)), null, false);
+            args = Arrays.copyOfRange(argsOrig, firstArg, firstArg + paramsCnt);
+        }
 
-                        resCur.fieldsMeta(UPDATE_RESULT_META);
+       if (prepared.isQuery()) {
+            try {
+                bindParameters(stmt, F.asList(args));
+            }
+            catch (IgniteCheckedException e) {
+                U.closeQuiet(stmt);
 
-                        res.add(resCur);
+                throw new IgniteSQLException("Failed to bind parameters: [qry=" + prepared.getSQL() + ", params=" +
+                    Arrays.deepToString(args) + "]", IgniteQueryErrorCode.PARSING, e);
+            }
 
-                        continue;
-                    }
+            GridSqlQueryParser parser = null;
+
+            if (!loc) {
+                parser = new GridSqlQueryParser(false);
+
+                GridSqlStatement parsedStmt = parser.parse(prepared);
+
+                // Legit assertion - we have H2 query flag above.
+                assert parsedStmt instanceof GridSqlQuery;
+
+                loc = parser.isLocalQuery(qry.isReplicatedOnly());
+            }
+
+            if (loc) {
+                if (parser == null) {
+                    parser = new GridSqlQueryParser(false);
+
+                    parser.parse(prepared);
                 }
 
-                assert twoStepQry != null;
+                GridCacheContext cctx = parser.getFirstPartitionedCache();
 
-                List cacheIds = collectCacheIds(mainCacheId, twoStepQry);
+                if (cctx != null && cctx.config().getQueryParallelism() > 1) {
+                    loc = false;
 
-                if (F.isEmpty(cacheIds))
-                    twoStepQry.local(true);
-                else {
-                    twoStepQry.cacheIds(cacheIds);
-                    twoStepQry.local(qry.isLocal());
+                    qry.setDistributedJoins(true);
                 }
+            }
+        }
+
+        SqlFieldsQuery newQry = cloneFieldsQuery(qry).setSql(prepared.getSQL()).setArgs(args);
+
+        boolean hasTwoStep = !loc && prepared.isQuery();
+
+        // Let's not cache multiple statements and distributed queries as whole two step query will be cached later on.
+        if (remainingSql != null || hasTwoStep)
+            getStatementsCacheForCurrentThread().remove(schemaName, qry.getSql());
+
+        if (!hasTwoStep)
+            return new ParsingResult(prepared, newQry, remainingSql, null, null, null);
+
+        final UUID locNodeId = ctx.localNodeId();
+
+        // Now we're sure to have a distributed query. Let's try to get a two-step plan from the cache, or perform the
+        // split if needed.
+        H2TwoStepCachedQueryKey cachedQryKey = new H2TwoStepCachedQueryKey(schemaName, qry.getSql(),
+            qry.isCollocated(), qry.isDistributedJoins(), qry.isEnforceJoinOrder(), qry.isLocal());
+
+        H2TwoStepCachedQuery cachedQry;
+
+        if ((cachedQry = twoStepCache.get(cachedQryKey)) != null) {
+            checkQueryType(qry, true);
+
+            GridCacheTwoStepQuery twoStepQry = cachedQry.query().copy();
+
+            List meta = cachedQry.meta();
+
+            return new ParsingResult(prepared, newQry, remainingSql, twoStepQry, cachedQryKey, meta);
+        }
+
+        try {
+            GridH2QueryContext.set(new GridH2QueryContext(locNodeId, locNodeId, 0, PREPARE)
+                .distributedJoinMode(distributedJoinMode(qry.isLocal(), qry.isDistributedJoins())));
 
-                meta = H2Utils.meta(stmt.getMetaData());
+            try {
+                return new ParsingResult(prepared, newQry, remainingSql, split(prepared, newQry),
+                    cachedQryKey, H2Utils.meta(stmt.getMetaData()));
             }
             catch (IgniteCheckedException e) {
-                throw new CacheException("Failed to bind parameters: [qry=" + sqlQry + ", params=" +
-                    Arrays.deepToString(qry.getArgs()) + "]", e);
+                throw new IgniteSQLException("Failed to bind parameters: [qry=" + newQry.getSql() + ", params=" +
+                    Arrays.deepToString(newQry.getArgs()) + "]", IgniteQueryErrorCode.PARSING, e);
             }
             catch (SQLException e) {
                 throw new IgniteSQLException(e);
@@ -1648,79 +1817,125 @@ private List>> tryQueryDistributedSqlFieldsNative(Stri
             finally {
                 U.close(stmt, log);
             }
-
-            res.add(executeTwoStepsQuery(schemaName, qry.getPageSize(), qry.getPartitions(), args, keepBinary,
-                qry.isLazy(), qry.getTimeout(), cancel, sqlQry, enforceJoinOrder,
-                twoStepQry, meta));
-
-            if (cachedQry == null && !twoStepQry.explain()) {
-                cachedQry = new H2TwoStepCachedQuery(meta, twoStepQry.copy());
-
-                twoStepCache.putIfAbsent(cachedQryKey, cachedQry);
-            }
         }
+        finally {
+            GridH2QueryContext.clearThreadLocal();
+        }
+    }
 
-        return res;
+    /**
+     * Make a copy of {@link SqlFieldsQuery} with all flags and preserving type.
+     * @param oldQry Query to copy.
+     * @return Query copy.
+     */
+    private SqlFieldsQuery cloneFieldsQuery(SqlFieldsQuery oldQry) {
+        return oldQry.copy().setLocal(oldQry.isLocal()).setPageSize(oldQry.getPageSize());
     }
 
     /**
-     * Check expected statement type (when it is set by JDBC) and given statement type.
-     *
-     * @param qry Query.
-     * @param isQry {@code true} for select queries, otherwise (DML/DDL queries) {@code false}.
+     * Split query into two-step query.
+     * @param prepared JDBC prepared statement.
+     * @param qry Original fields query.
+     * @return Two-step query.
+     * @throws IgniteCheckedException in case of error inside {@link GridSqlQuerySplitter}.
+     * @throws SQLException in case of error inside {@link GridSqlQuerySplitter}.
      */
-    private void checkQueryType(SqlFieldsQuery qry, boolean isQry) {
-        if (qry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)qry).isQuery() != null &&
-            ((SqlFieldsQueryEx)qry).isQuery() != isQry)
-            throw new IgniteSQLException("Given statement type does not match that declared by JDBC driver",
-                IgniteQueryErrorCode.STMT_TYPE_MISMATCH);
+    private GridCacheTwoStepQuery split(Prepared prepared, SqlFieldsQuery qry) throws IgniteCheckedException,
+        SQLException {
+        GridCacheTwoStepQuery res = GridSqlQuerySplitter.split(connectionForThread(qry.getSchema()), prepared,
+            qry.getArgs(), qry.isCollocated(), qry.isDistributedJoins(), qry.isEnforceJoinOrder(), this);
+
+        List cacheIds = collectCacheIds(null, res);
+
+        if (F.isEmpty(cacheIds))
+            res.local(true);
+        else {
+            res.cacheIds(cacheIds);
+            res.local(qry.isLocal());
+        }
+
+        res.pageSize(qry.getPageSize());
+
+        return res;
     }
 
     /**
+     * Run distributed query on detected set of partitions.
      * @param schemaName Schema name.
-     * @param pageSize Page size.
-     * @param partitions Partitions.
-     * @param args Arguments.
+     * @param qry Original query.
+     * @param twoStepQry Two-step query.
+     * @param meta Metadata to set to cursor.
      * @param keepBinary Keep binary flag.
-     * @param lazy Lazy flag.
-     * @param timeout Timeout.
-     * @param cancel Cancel.
-     * @param sqlQry SQL query string.
-     * @param enforceJoinOrder Enforce join orded flag.
-     * @param twoStepQry Two-steps query.
-     * @param meta Metadata.
-     * @return Cursor.
-     */
-    private FieldsQueryCursor> executeTwoStepsQuery(String schemaName, int pageSize, int partitions[],
-        Object[] args, boolean keepBinary, boolean lazy, int timeout,
-        GridQueryCancel cancel, String sqlQry, boolean enforceJoinOrder, GridCacheTwoStepQuery twoStepQry,
-        List meta) {
+     * @param cancel Cancel handler.
+     * @return Cursor representing distributed query result.
+     */
+    private FieldsQueryCursor> doRunDistributedQuery(String schemaName, SqlFieldsQuery qry,
+        GridCacheTwoStepQuery twoStepQry, List meta, boolean keepBinary,
+        GridQueryCancel cancel) {
         if (log.isDebugEnabled())
-            log.debug("Parsed query: `" + sqlQry + "` into two step query: " + twoStepQry);
+            log.debug("Parsed query: `" + qry.getSql() + "` into two step query: " + twoStepQry);
 
-        twoStepQry.pageSize(pageSize);
+        twoStepQry.pageSize(qry.getPageSize());
 
         if (cancel == null)
             cancel = new GridQueryCancel();
 
+        int partitions[] = qry.getPartitions();
+
         if (partitions == null && twoStepQry.derivedPartitions() != null) {
             try {
-                partitions = calculateQueryPartitions(twoStepQry.derivedPartitions(), args);
-            } catch (IgniteCheckedException e) {
-                throw new CacheException("Failed to calculate derived partitions: [qry=" + sqlQry + ", params=" +
-                    Arrays.deepToString(args) + "]", e);
+                partitions = calculateQueryPartitions(twoStepQry.derivedPartitions(), qry.getArgs());
+            }
+            catch (IgniteCheckedException e) {
+                throw new CacheException("Failed to calculate derived partitions: [qry=" + qry.getSql() + ", params=" +
+                    Arrays.deepToString(qry.getArgs()) + "]", e);
             }
         }
 
         QueryCursorImpl> cursor = new QueryCursorImpl<>(
-            runQueryTwoStep(schemaName, twoStepQry, keepBinary, enforceJoinOrder, timeout, cancel,
-                args, partitions, lazy), cancel);
+            runQueryTwoStep(schemaName, twoStepQry, keepBinary, qry.isEnforceJoinOrder(), qry.getTimeout(), cancel,
+                qry.getArgs(), partitions, qry.isLazy()), cancel);
 
         cursor.fieldsMeta(meta);
 
         return cursor;
     }
 
+    /**
+     * Do initial parsing of the statement and create query caches, if needed.
+     * @param c Connection.
+     * @param sqlQry Query.
+     * @return H2 prepared statement.
+     */
+    private PreparedStatement prepareStatementAndCaches(Connection c, String sqlQry) {
+        boolean cachesCreated = false;
+
+        while (true) {
+            try {
+                return prepareStatement(c, sqlQry, true);
+            }
+            catch (SQLException e) {
+                if (!cachesCreated && (
+                        e.getErrorCode() == ErrorCode.SCHEMA_NOT_FOUND_1 ||
+                            e.getErrorCode() == ErrorCode.TABLE_OR_VIEW_NOT_FOUND_1 ||
+                            e.getErrorCode() == ErrorCode.INDEX_NOT_FOUND_1)
+                        ) {
+                    try {
+                        ctx.cache().createMissingQueryCaches();
+                    }
+                    catch (IgniteCheckedException ignored) {
+                        throw new CacheException("Failed to create missing caches.", e);
+                    }
+
+                    cachesCreated = true;
+                }
+                else
+                    throw new IgniteSQLException("Failed to parse query. " + e.getMessage(),
+                        IgniteQueryErrorCode.PARSING, e);
+            }
+        }
+    }
+
     /**
      * Run DML request from other node.
      *
@@ -1748,7 +1963,7 @@ public UpdateResult mapDistributedUpdate(String schemaName, SqlFieldsQuery fldsQ
      * @param cacheIds Caches identifiers.
      * @throws IllegalStateException if segmented indices used with non-segmented indices.
      */
-    private void checkCacheIndexSegmentation(List cacheIds) {
+    private void checkCacheIndexSegmentation(Collection cacheIds) {
         if (cacheIds.isEmpty())
             return; // Nothing to check
 
@@ -2692,7 +2907,7 @@ public Map perThreadConnections() {
      * @param twoStepQry Two-step query.
      * @return Result.
      */
-    public List collectCacheIds(@Nullable Integer mainCacheId, GridCacheTwoStepQuery twoStepQry) {
+    @Nullable public List collectCacheIds(@Nullable Integer mainCacheId, GridCacheTwoStepQuery twoStepQry) {
         LinkedHashSet caches0 = new LinkedHashSet<>();
 
         int tblCnt = twoStepQry.tablesCount();
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ParsingResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ParsingResult.java
new file mode 100644
index 0000000000000..98d28481dac4d
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ParsingResult.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2;
+
+import java.util.List;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
+import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
+import org.h2.command.Prepared;
+
+/**
+ * Result of parsing and splitting SQL from {@link SqlFieldsQuery}.
+ */
+final class ParsingResult {
+    /** H2 command. */
+    private final Prepared prepared;
+
+    /** New fields query that may be executed right away. */
+    private final SqlFieldsQuery newQry;
+
+    /** Remaining SQL statements. */
+    private final String remainingSql;
+
+    /** Two-step query, or {@code} null if this result is for local query. */
+    private final GridCacheTwoStepQuery twoStepQry;
+
+    /** Two-step query key to cache {@link #twoStepQry}, or {@code null} if there's no need to worry
+     * about two-step caching. */
+    private final H2TwoStepCachedQueryKey twoStepQryKey;
+
+    /** Metadata for two-step query, or {@code} null if this result is for local query. */
+    private final List meta;
+
+    /** Simple constructor. */
+    ParsingResult(Prepared prepared, SqlFieldsQuery newQry, String remainingSql, GridCacheTwoStepQuery twoStepQry,
+        H2TwoStepCachedQueryKey twoStepQryKey, List meta) {
+        this.prepared = prepared;
+        this.newQry = newQry;
+        this.remainingSql = remainingSql;
+        this.twoStepQry = twoStepQry;
+        this.twoStepQryKey = twoStepQryKey;
+        this.meta = meta;
+    }
+
+    /**
+     * @return Metadata for two-step query, or {@code} null if this result is for local query.
+     */
+    List meta() {
+        return meta;
+    }
+
+    /**
+     * @return New fields query that may be executed right away.
+     */
+    SqlFieldsQuery newQuery() {
+        return newQry;
+    }
+
+    /**
+     * @return H2 command.
+     */
+    Prepared prepared() {
+        return prepared;
+    }
+
+    /**
+     * @return Remaining SQL statements.
+     */
+    String remainingSql() {
+        return remainingSql;
+    }
+
+    /**
+     * @return Two-step query, or {@code} null if this result is for local query.
+     */
+    GridCacheTwoStepQuery twoStepQuery() {
+        return twoStepQry;
+    }
+
+    /**
+     * @return Two-step query key to cache {@link #twoStepQry}, or {@code null} if there's no need to worry
+     * about two-step caching.
+     */
+    H2TwoStepCachedQueryKey twoStepQueryKey() {
+        return twoStepQryKey;
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index 67137d4b6efab..2630239dac24c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.processors.query.GridQueryProperty;
@@ -118,6 +119,8 @@ public FieldsQueryCursor> runDdlStatement(String sql, SqlCommand cmd) th
 
                 assert tbl.rowDescriptor() != null;
 
+                isDdlSupported(tbl);
+
                 QueryIndex newIdx = new QueryIndex();
 
                 newIdx.setName(cmd0.indexName());
@@ -150,6 +153,8 @@ else if (cmd instanceof SqlDropIndexCommand) {
                 GridH2Table tbl = idx.dataTableForIndex(cmd0.schemaName(), cmd0.indexName());
 
                 if (tbl != null) {
+                    isDdlSupported(tbl);
+
                     fut = ctx.query().dynamicIndexDrop(tbl.cacheName(), cmd0.schemaName(), cmd0.indexName(),
                         cmd0.ifExists());
                 }
@@ -241,6 +246,8 @@ public FieldsQueryCursor> runDdlStatement(String sql, Prepared prepared)
 
                 assert tbl.rowDescriptor() != null;
 
+                isDdlSupported(tbl);
+
                 QueryIndex newIdx = new QueryIndex();
 
                 newIdx.setName(cmd.index().getName());
@@ -272,6 +279,8 @@ else if (stmt0 instanceof GridSqlDropIndex) {
                 GridH2Table tbl = idx.dataTableForIndex(cmd.schemaName(), cmd.indexName());
 
                 if (tbl != null) {
+                    isDdlSupported(tbl);
+
                     fut = ctx.query().dynamicIndexDrop(tbl.cacheName(), cmd.schemaName(), cmd.indexName(),
                         cmd.ifExists());
                 }
@@ -476,6 +485,21 @@ else if (stmt0 instanceof GridSqlAlterTableDropColumn) {
         }
     }
 
+    /**
+     * Check if table supports DDL statement.
+     *
+     * @param tbl Table.
+     */
+    private static void isDdlSupported(GridH2Table tbl) {
+        GridCacheContext cctx = tbl.cache();
+
+        assert cctx != null;
+
+        if (cctx.isLocal())
+            throw new IgniteSQLException("DDL statements are not supported on LOCAL caches",
+                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+    }
+
     /**
      * @return {@link IgniteSQLException} with the message same as of {@code this}'s and
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index 52dee60b90d4d..04bc21226e37d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -33,6 +33,8 @@
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.cache.QueryIndexType;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.processors.query.IgniteSQLException;
 import org.apache.ignite.internal.processors.query.QueryUtils;
@@ -90,6 +92,7 @@
 import org.h2.table.Column;
 import org.h2.table.FunctionTable;
 import org.h2.table.IndexColumn;
+import org.h2.table.MetaTable;
 import org.h2.table.RangeTable;
 import org.h2.table.Table;
 import org.h2.table.TableBase;
@@ -626,6 +629,8 @@ else if (tbl instanceof RangeTable) {
                 res.addChild(parseExpression(RANGE_MIN.get((RangeTable)tbl), false));
                 res.addChild(parseExpression(RANGE_MAX.get((RangeTable)tbl), false));
             }
+            else if (tbl instanceof MetaTable)
+                res = new GridSqlTable(tbl);
             else
                 assert0(false, "Unexpected Table implementation [cls=" + tbl.getClass().getSimpleName() + ']');
 
@@ -1550,6 +1555,61 @@ public static Query query(Prepared qry) {
         throw new CacheException("Unsupported query: " + qry);
     }
 
+    /**
+     * Check if query may be run locally on all caches mentioned in the query.
+     * @param replicatedOnlyQry replicated-only query flag from original {@link SqlFieldsQuery}.
+     * @return {@code true} if query may be run locally on all caches mentioned in the query, i.e. there's no need
+     *     to run distributed query.
+     * @see SqlFieldsQuery#isReplicatedOnly()
+     */
+    public boolean isLocalQuery(boolean replicatedOnlyQry) {
+        boolean hasCaches = false;
+
+        for (Object o : h2ObjToGridObj.values()) {
+            if (o instanceof GridSqlAlias)
+                o = GridSqlAlias.unwrap((GridSqlAst)o);
+
+            if (o instanceof GridSqlTable) {
+                GridH2Table tbl = ((GridSqlTable)o).dataTable();
+
+                if (tbl != null) {
+                    hasCaches = true;
+
+                    GridCacheContext cctx = tbl.cache();
+
+                    if (!cctx.isLocal() && !(replicatedOnlyQry && cctx.isReplicatedAffinityNode()))
+                        return false;
+                }
+            }
+        }
+
+        // For consistency with old logic, let's not force locality in absence of caches -
+        // if there are no caches, original SqlFieldsQuery's isLocal flag will be used.
+        return hasCaches;
+    }
+
+    /**
+     * Get first (i.e. random, as we need any one) partitioned cache from parsed query
+     *     to determine expected query parallelism.
+     * @return Context for the first of partitioned caches mentioned in the query,
+     *     or {@code null} if it does not involve partitioned caches.
+     */
+    public GridCacheContext getFirstPartitionedCache() {
+        for (Object o : h2ObjToGridObj.values()) {
+            if (o instanceof GridSqlAlias)
+                o = GridSqlAlias.unwrap((GridSqlAst)o);
+
+            if (o instanceof GridSqlTable) {
+                GridH2Table tbl = ((GridSqlTable)o).dataTable();
+
+                if (tbl != null && tbl.cache().isPartitioned())
+                    return tbl.cache();
+            }
+        }
+
+        return null;
+    }
+
     /**
      * @param stmt Prepared statement.
      * @return Parsed AST.
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index ca7fcd9f5fc35..52b641e7c0859 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -77,8 +77,8 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.thread.IgniteThread;
 import org.apache.ignite.spi.indexing.IndexingQueryFilter;
+import org.apache.ignite.thread.IgniteThread;
 import org.h2.jdbc.JdbcResultSet;
 import org.h2.value.Value;
 import org.jetbrains.annotations.Nullable;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
index bbd3d0d9e85eb..069bdd7092d16 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheCrossCacheQuerySelfTest.java
@@ -140,7 +140,7 @@ public void testTwoStepGroupAndAggregates() throws Exception {
         SqlFieldsQuery qry = new SqlFieldsQuery("select f.productId, p.name, f.price " +
             "from FactPurchase f, \"replicated-prod\".DimProduct p where p.id = f.productId ");
 
-        for (List o : qryProc.querySqlFields(cache.context(), qry, false).getAll()) {
+        for (List o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             set1.add((Integer)o.get(0));
@@ -154,7 +154,7 @@ public void testTwoStepGroupAndAggregates() throws Exception {
 
         qry = new SqlFieldsQuery("select productId from FactPurchase group by productId");
 
-        for (List o : qryProc.querySqlFields(cache.context(), qry, false).getAll()) {
+        for (List o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertTrue(set0.add((Integer) o.get(0)));
@@ -173,7 +173,7 @@ public void testTwoStepGroupAndAggregates() throws Exception {
             "where p.id = f.productId " +
             "group by f.productId, p.name");
 
-        for (List o : qryProc.querySqlFields(cache.context(), qry, false).getAll()) {
+        for (List o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertTrue(names.add((String)o.get(0)));
@@ -190,7 +190,7 @@ public void testTwoStepGroupAndAggregates() throws Exception {
             "group by f.productId, p.name " +
             "having s >= 15");
 
-        for (List o : qryProc.querySqlFields(cache.context(), qry, false).getAll()) {
+        for (List o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertTrue(i(o, 1) >= 15);
@@ -203,7 +203,7 @@ public void testTwoStepGroupAndAggregates() throws Exception {
         qry = new SqlFieldsQuery("select top 3 distinct productId " +
             "from FactPurchase f order by productId desc ");
 
-        for (List o : qryProc.querySqlFields(cache.context(), qry, false).getAll()) {
+        for (List o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertEquals(top--, o.get(0));
@@ -216,7 +216,7 @@ public void testTwoStepGroupAndAggregates() throws Exception {
         qry = new SqlFieldsQuery("select distinct productId " +
             "from FactPurchase f order by productId desc limit 2 offset 1");
 
-        for (List o : qryProc.querySqlFields(cache.context(), qry, false).getAll()) {
+        for (List o : qryProc.querySqlFields(cache.context(), qry, false, true).get(0).getAll()) {
             X.println("___ -> " + o);
 
             assertEquals(top--, o.get(0));
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCheckClusterStateBeforeExecuteQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCheckClusterStateBeforeExecuteQueryTest.java
index aae6a0ca8a2f0..f57a8c6ce534b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCheckClusterStateBeforeExecuteQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCheckClusterStateBeforeExecuteQueryTest.java
@@ -80,7 +80,7 @@ public void testDynamicSchemaChangesPersistence() throws Exception {
 
         assertThrows(log, new Callable() {
             @Override public Void call() throws Exception {
-                ig.context().query().querySqlFieldsNoCache(new SqlFieldsQuery("SELECT 1"), false);
+                ig.context().query().querySqlFields(new SqlFieldsQuery("SELECT 1"), false);
 
                 return null;
             }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlFieldsQuerySelfTest.java
index 8860b2ba495e9..f68484171f7bb 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlFieldsQuerySelfTest.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.cache;
 
 import java.io.Serializable;
+import java.sql.PreparedStatement;
 import java.util.List;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
@@ -25,12 +26,16 @@
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
  *
  */
 public class SqlFieldsQuerySelfTest extends GridCommonAbstractTest {
+    /** INSERT statement. */
+    private final static String INSERT = "insert into Person(_key, name) values (5, 'x')";
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         stopAllGrids();
@@ -60,6 +65,50 @@ public void testSqlFieldsQueryWithTopologyChanges() throws Exception {
         executeQuery();
     }
 
+    /**
+     * @throws Exception If error.
+     */
+    public void testQueryCaching() throws Exception {
+        startGrid(0);
+
+        PreparedStatement stmt = null;
+
+        for (int i = 0; i < 2; i++) {
+            createAndFillCache();
+
+            PreparedStatement stmt0 = grid(0).context().query().prepareNativeStatement("person", INSERT);
+
+            // Statement should either be parsed initially or in response to schema change...
+            assertTrue(stmt != stmt0);
+
+            stmt = stmt0;
+
+            // ...and be properly compiled considering schema changes to be properly parsed
+            new GridSqlQueryParser(false).parse(GridSqlQueryParser.prepared(stmt));
+
+            destroyCache();
+        }
+
+        stmt = null;
+
+        createAndFillCache();
+
+        // Now let's do the same without restarting the cache.
+        for (int i = 0; i < 2; i++) {
+            PreparedStatement stmt0 = grid(0).context().query().prepareNativeStatement("person", INSERT);
+
+            // Statement should either be parsed or taken from cache as no schema changes occurred...
+            assertTrue(stmt == null || stmt == stmt0);
+
+            stmt = stmt0;
+
+            // ...and be properly compiled considering schema changes to be properly parsed
+            new GridSqlQueryParser(false).parse(GridSqlQueryParser.prepared(stmt));
+        }
+
+        destroyCache();
+    }
+
     /**
      *
      */
@@ -101,6 +150,10 @@ private IgniteCache createAndFillCache() {
         return cache;
     }
 
+    private void destroyCache() {
+        grid(0).destroyCache("person");
+    }
+
     /**
      *
      */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
index a679b159771e2..ec1a16d670028 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
@@ -196,7 +196,22 @@ public void testIterator() throws Exception {
     /**
      * @throws Exception If test failed.
      */
-    public void testLocalQuery() throws Exception {
+    public void testLocalQueryWithExplicitFlag() throws Exception {
+        doTestLocalQuery(true);
+    }
+
+    /**
+     * @throws Exception If test failed.
+     */
+    public void testLocalQueryWithoutExplicitFlag() throws Exception {
+        doTestLocalQuery(false);
+    }
+
+    /**
+     * @param loc Explicit query locality flag.
+     * @throws Exception if failed.
+     */
+    private void doTestLocalQuery(boolean loc) throws Exception {
         cache1.clear();
 
         Transaction tx = ignite1.transactions().txStart();
@@ -217,9 +232,9 @@ public void testLocalQuery() throws Exception {
             throw e;
         }
 
-        checkQueryResults(cache1);
-        checkQueryResults(cache2);
-        checkQueryResults(cache3);
+        checkLocalQueryResults(cache1, loc);
+        checkLocalQueryResults(cache2, loc);
+        checkLocalQueryResults(cache3, loc);
     }
 
     /**
@@ -403,11 +418,13 @@ public void testNodeLeft() throws Exception {
 
     /**
      * @param cache Cache.
+     * @param loc Explicit query locality flag.
      * @throws Exception If check failed.
      */
-    private void checkQueryResults(IgniteCache cache) throws Exception {
+    private void checkLocalQueryResults(IgniteCache cache, boolean loc) throws Exception {
         QueryCursor> qry =
-            cache.query(new SqlQuery(CacheValue.class, "val > 1 and val < 4").setLocal(true));
+            cache.query(new SqlQuery(CacheValue.class, "val > 1 and val < 4")
+                .setReplicatedOnly(true).setLocal(loc));
 
         Iterator> iter = qry.iterator();
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java
index dbc6290f0757c..0a0efc72d373d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java
@@ -80,6 +80,9 @@ public abstract class AbstractSchemaSelfTest extends GridCommonAbstractTest {
     /** Index name 2 escaped. */
     protected static final String IDX_NAME_2_ESCAPED = "idx_2";
 
+    /** Index name 2. */
+    protected static final String IDX_NAME_LOCAL = "IDX_LOC";
+
     /** Key ID field. */
     protected static final String FIELD_KEY = "id";
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractTest.java
index e5072170a852e..2cbd3a5cc89de 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractTest.java
@@ -210,7 +210,7 @@ IgniteConfiguration serverConfiguration(int idx) throws Exception {
      */
     protected List> run(Ignite node, String sql) {
         return ((IgniteEx)node).context().query()
-            .querySqlFieldsNoCache(new SqlFieldsQuery(sql).setSchema(QueryUtils.DFLT_SCHEMA), true).getAll();
+            .querySqlFields(new SqlFieldsQuery(sql).setSchema(QueryUtils.DFLT_SCHEMA), true).getAll();
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
index db874c28d8351..97720d5666057 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
@@ -42,7 +42,6 @@
 
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
-import static org.apache.ignite.cache.CacheMode.LOCAL;
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 
@@ -200,7 +199,7 @@ private void checkCreate(CacheMode mode, CacheAtomicityMode atomicityMode, boole
         dynamicIndexCreate(CACHE_NAME, TBL_NAME, idx, false, 0);
         assertIndex(CACHE_NAME, TBL_NAME, IDX_NAME_1, QueryIndex.DFLT_INLINE_SIZE, field(FIELD_NAME_1_ESCAPED));
 
-        assertSchemaException(new RunnableX() {
+        assertIgniteSqlException(new RunnableX() {
             @Override public void run() throws Exception {
                 dynamicIndexCreate(CACHE_NAME, TBL_NAME, idx, false, 0);
             }
@@ -443,7 +442,7 @@ private void checkCreateNoTable(CacheMode mode, CacheAtomicityMode atomicityMode
 
         final QueryIndex idx = index(IDX_NAME_1, field(FIELD_NAME_1_ESCAPED));
 
-        assertSchemaException(new RunnableX() {
+        assertIgniteSqlException(new RunnableX() {
             @Override public void run() throws Exception {
                 dynamicIndexCreate(CACHE_NAME, randomString(), idx, false, 0);
             }
@@ -519,7 +518,7 @@ private void checkCreateIndexNoColumn(CacheMode mode, CacheAtomicityMode atomici
 
         final QueryIndex idx = index(IDX_NAME_1, field(randomString()));
 
-        assertSchemaException(new RunnableX() {
+        assertIgniteSqlException(new RunnableX() {
             @Override public void run() throws Exception {
                 dynamicIndexCreate(CACHE_NAME, TBL_NAME, idx, false, 0);
             }
@@ -594,7 +593,7 @@ private void checkCreateIndexOnColumnWithAlias(CacheMode mode, CacheAtomicityMod
         throws Exception {
         initialize(mode, atomicityMode, near);
 
-        assertSchemaException(new RunnableX() {
+        assertIgniteSqlException(new RunnableX() {
             @Override public void run() throws Exception {
                 QueryIndex idx = index(IDX_NAME_1, field(FIELD_NAME_2_ESCAPED));
 
@@ -732,7 +731,7 @@ private void checkIndexCreatedForInlineSize(int inlineSize) throws Exception {
      * @throws Exception If failed for any other reason than the expected exception.
      */
     private void checkNoIndexIsCreatedForInlineSize(final int inlineSize, int igniteQryErrorCode) throws Exception {
-        assertSchemaException(new RunnableX() {
+        assertIgniteSqlException(new RunnableX() {
             @Override public void run() throws Exception {
                 QueryIndex idx = index(IDX_NAME_1, field(FIELD_NAME_1_ESCAPED));
                 idx.setInlineSize(inlineSize);
@@ -863,7 +862,7 @@ private void checkIndexCreatedForParallelism(int parallel) throws Exception {
      * @throws Exception If failed for any other reason than the expected exception.
      */
     private void checkNoIndexIsCreatedForParallelism(final int parallel, int igniteQryErrorCode) throws Exception {
-        assertSchemaException(new RunnableX() {
+        assertIgniteSqlException(new RunnableX() {
             @Override public void run() throws Exception {
                 QueryIndex idx = index(IDX_NAME_1, field(FIELD_NAME_1_ESCAPED));
                 dynamicIndexCreate(CACHE_NAME, TBL_NAME, idx, false, parallel);
@@ -1034,7 +1033,7 @@ public void testDropNoIndexReplicatedTransactional() throws Exception {
     private void checkDropNoIndex(CacheMode mode, CacheAtomicityMode atomicityMode, boolean near) throws Exception {
         initialize(mode, atomicityMode, near);
 
-        assertSchemaException(new RunnableX() {
+        assertIgniteSqlException(new RunnableX() {
             @Override public void run() throws Exception {
                 dynamicIndexDrop(CACHE_NAME, IDX_NAME_1, false);
             }
@@ -1138,12 +1137,12 @@ private void checkDropNoCache(CacheMode mode, CacheAtomicityMode atomicityMode,
     public void testFailOnLocalCache() throws Exception {
         for (Ignite node : Ignition.allGrids()) {
             if (!node.configuration().isClientMode())
-                createSqlCache(node, cacheConfiguration().setCacheMode(LOCAL));
+                createSqlCache(node, localCacheConfiguration());
         }
 
         final QueryIndex idx = index(IDX_NAME_1, field(FIELD_NAME_1_ESCAPED));
 
-        assertSchemaException(new RunnableX() {
+        assertIgniteSqlException(new RunnableX() {
             @Override public void run() throws Exception {
                 dynamicIndexCreate(CACHE_NAME, TBL_NAME, idx, true, 0);
             }
@@ -1151,9 +1150,9 @@ public void testFailOnLocalCache() throws Exception {
 
         assertNoIndex(CACHE_NAME, TBL_NAME, IDX_NAME_1);
 
-        assertSchemaException(new RunnableX() {
+        assertIgniteSqlException(new RunnableX() {
             @Override public void run() throws Exception {
-                dynamicIndexDrop(CACHE_NAME, IDX_NAME_1, true);
+                dynamicIndexDrop(CACHE_NAME, IDX_NAME_LOCAL, true);
             }
         }, IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
     }
@@ -1368,8 +1367,8 @@ private void assertCompositeIndexOperations(String sql) {
      * @param r Runnable.
      * @param expCode Error code.
      */
-    protected static void assertSchemaException(RunnableX r, int expCode) {
-        assertSchemaException(r, null, expCode);
+    protected static void assertIgniteSqlException(RunnableX r, int expCode) {
+        assertIgniteSqlException(r, null, expCode);
     }
 
     /**
@@ -1379,7 +1378,7 @@ protected static void assertSchemaException(RunnableX r, int expCode) {
      * @param msg Exception message to expect, or {@code null} if it can be waived.
      * @param expCode Error code.
      */
-    protected static void assertSchemaException(RunnableX r, String msg, int expCode) {
+    private static void assertIgniteSqlException(RunnableX r, String msg, int expCode) {
         try {
             r.run();
         }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractSelfTest.java
index a39283b0ca332..452ac96ba9e35 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractSelfTest.java
@@ -17,20 +17,29 @@
 
 package org.apache.ignite.internal.processors.cache.index;
 
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.SqlQuery;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.util.typedef.T2;
@@ -39,14 +48,6 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 
-import javax.cache.Cache;
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
 /**
  * Tests for dynamic index creation.
  */
@@ -181,6 +182,20 @@ protected CacheConfiguration cacheConfiguration() {
         return ccfg;
     }
 
+    /**
+     * @return Local cache configuration with a pre-configured index.
+     */
+    CacheConfiguration localCacheConfiguration() {
+        CacheConfiguration res = cacheConfiguration();
+
+        res.getQueryEntities().iterator().next().setIndexes(Collections.singletonList(
+            new QueryIndex(FIELD_NAME_2_ESCAPED, true, IDX_NAME_LOCAL)));
+
+        res.setCacheMode(CacheMode.LOCAL);
+
+        return res;
+    }
+
     /**
      * Ensure index is used in plan.
      *
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexingComplexTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexingComplexTest.java
index 02e48ed26451e..d7a351b509f45 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexingComplexTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexingComplexTest.java
@@ -330,7 +330,7 @@ private void assertPerson(int id, String name, int age, String company, String c
      * @return Run result.
      */
     private List> executeSql(IgniteEx node, String stmt, Object... args) {
-        return node.context().query().querySqlFieldsNoCache(new SqlFieldsQuery(stmt).setArgs(args), true).getAll();
+        return node.context().query().querySqlFields(new SqlFieldsQuery(stmt).setArgs(args), true).getAll();
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
index fb38c46b15dd3..b20bb59611afe 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
@@ -891,11 +891,11 @@ public void testAffinityKey() throws Exception {
 
             personId2cityCode.put(i, cityCode);
 
-            queryProcessor(client()).querySqlFieldsNoCache(new SqlFieldsQuery("insert into \"Person2\"(\"id\", " +
+            queryProcessor(client()).querySqlFields(new SqlFieldsQuery("insert into \"Person2\"(\"id\", " +
                 "\"city\") values (?, ?)").setArgs(i, cityName), true).getAll();
         }
 
-        List> res = queryProcessor(client()).querySqlFieldsNoCache(new SqlFieldsQuery("select \"id\", " +
+        List> res = queryProcessor(client()).querySqlFields(new SqlFieldsQuery("select \"id\", " +
             "c.\"code\" from \"Person2\" p left join \"City\" c on p.\"city\" = c.\"name\" where c.\"name\" " +
             "is not null"), true).getAll();
 
@@ -1555,7 +1555,7 @@ private IgniteConfiguration commonConfiguration(int idx) throws Exception {
      * @param sql Statement.
      */
     private List> execute(Ignite node, String sql) {
-        return queryProcessor(node).querySqlFieldsNoCache(new SqlFieldsQuery(sql).setSchema("PUBLIC"), true).getAll();
+        return queryProcessor(node).querySqlFields(new SqlFieldsQuery(sql).setSchema("PUBLIC"), true).getAll();
     }
 
     /**
@@ -1564,7 +1564,7 @@ private List> execute(Ignite node, String sql) {
      * @param sql Statement.
      */
     private List> executeLocal(GridCacheContext cctx, String sql) {
-        return queryProcessor(cctx.grid()).querySqlFields(cctx, new SqlFieldsQuery(sql).setLocal(true), true).getAll();
+        return queryProcessor(cctx.grid()).querySqlFields(new SqlFieldsQuery(sql).setLocal(true), true).getAll();
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
index e0c63966d4328..2570bc896c664 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
@@ -47,44 +47,50 @@ public class IgniteCacheLocalQuerySelfTest extends IgniteCacheAbstractQuerySelfT
      * @throws Exception If test failed.
      */
     public void testQueryLocal() throws Exception {
-        IgniteCache cache = jcache(Integer.class, String.class);
+        // Let's do it twice to see how prepared statement caching behaves - without recompilation
+        // check for cached prepared statements this would fail.
+        for (int i = 0; i < 2; i ++) {
+            IgniteCache cache = jcache(Integer.class, String.class);
 
-        cache.put(1, "value1");
-        cache.put(2, "value2");
-        cache.put(3, "value3");
-        cache.put(4, "value4");
-        cache.put(5, "value5");
+            cache.put(1, "value1");
+            cache.put(2, "value2");
+            cache.put(3, "value3");
+            cache.put(4, "value4");
+            cache.put(5, "value5");
 
-        // Tests equals query.
-        QueryCursor> qry =
-            cache.query(new SqlQuery(String.class, "_val='value1'").setLocal(true));
+            // Tests equals query.
+            QueryCursor> qry =
+                cache.query(new SqlQuery(String.class, "_val='value1'").setLocal(true));
 
-        Iterator> iter = qry.iterator();
+            Iterator> iter = qry.iterator();
 
-        Cache.Entry entry = iter.next();
+            Cache.Entry entry = iter.next();
 
-        assert !iter.hasNext();
+            assert !iter.hasNext();
 
-        assert entry != null;
-        assert entry.getKey() == 1;
-        assert "value1".equals(entry.getValue());
+            assert entry != null;
+            assert entry.getKey() == 1;
+            assert "value1".equals(entry.getValue());
 
-        // Tests like query.
-        qry = cache.query(new SqlQuery(String.class, "_val like 'value%'").setLocal(true));
+            // Tests like query.
+            qry = cache.query(new SqlQuery(String.class, "_val like 'value%'").setLocal(true));
 
-        iter = qry.iterator();
+            iter = qry.iterator();
 
-        assert iter.next() != null;
-        assert iter.next() != null;
-        assert iter.next() != null;
-        assert iter.next() != null;
-        assert iter.next() != null;
-        assert !iter.hasNext();
+            assert iter.next() != null;
+            assert iter.next() != null;
+            assert iter.next() != null;
+            assert iter.next() != null;
+            assert iter.next() != null;
+            assert !iter.hasNext();
 
-        // Test explain for primitive index.
-        List> res = cache.query(new SqlFieldsQuery(
-            "explain select _key from String where _val > 'value1'").setLocal(true)).getAll();
+            // Test explain for primitive index.
+            List> res = cache.query(new SqlFieldsQuery(
+                    "explain select _key from String where _val > 'value1'").setLocal(true)).getAll();
 
-        assertTrue("__ explain: \n" + res, ((String)res.get(0).get(0)).toLowerCase().contains("_val_idx"));
+            assertTrue("__ explain: \n" + res, ((String) res.get(0).get(0)).toLowerCase().contains("_val_idx"));
+
+            cache.destroy();
+        }
     }
 }
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
index 2f6977a44ed1f..2b7c5c2748a89 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
@@ -193,8 +193,8 @@ private void checkSchemaStateAfterNodeRestart(boolean aliveCluster) throws Excep
 
         CountDownLatch cnt = checkpointLatch(node);
 
-        node.context().query().querySqlFieldsNoCache(
-            new SqlFieldsQuery("create table \"Person\" (\"id\" int primary key, \"name\" varchar)"), false).getAll();
+        node.context().query().querySqlFields(
+            new SqlFieldsQuery("create table \"Person\" (\"id\" int primary key, \"name\" varchar)"), false);
 
         assertEquals(0, indexCnt(node, SQL_CACHE_NAME));
 
@@ -212,7 +212,7 @@ private void checkSchemaStateAfterNodeRestart(boolean aliveCluster) throws Excep
 
         checkDynamicSchemaChanges(node, SQL_CACHE_NAME);
 
-        node.context().query().querySqlFieldsNoCache(new SqlFieldsQuery("drop table \"Person\""), false).getAll();
+        node.context().query().querySqlFields(new SqlFieldsQuery("drop table \"Person\""), false).getAll();
     }
 
     /** */
@@ -280,15 +280,15 @@ private CountDownLatch checkpointLatch(IgniteEx node) {
      * @param schema Schema name.
      */
     private void makeDynamicSchemaChanges(IgniteEx node, String schema) {
-        node.context().query().querySqlFieldsNoCache(
+        node.context().query().querySqlFields(
             new SqlFieldsQuery("create index \"my_idx\" on \"Person\" (\"id\", \"name\")").setSchema(schema), false)
                 .getAll();
 
-        node.context().query().querySqlFieldsNoCache(
+        node.context().query().querySqlFields(
             new SqlFieldsQuery("alter table \"Person\" add column (\"age\" int, \"city\" char)")
             .setSchema(schema), false).getAll();
 
-        node.context().query().querySqlFieldsNoCache(
+        node.context().query().querySqlFields(
             new SqlFieldsQuery("alter table \"Person\" drop column \"city\"").setSchema(schema), false)
             .getAll();
     }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteCachelessQueriesSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteCachelessQueriesSelfTest.java
new file mode 100644
index 0000000000000..a7dae9e849421
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteCachelessQueriesSelfTest.java
@@ -0,0 +1,420 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheKeyConfiguration;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
+import org.apache.ignite.internal.processors.query.h2.H2TwoStepCachedQuery;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for behavior in various cases of local and distributed queries.
+ */
+public class IgniteCachelessQueriesSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private final static String SELECT =
+        "select count(*) from \"pers\".Person p, \"org\".Organization o where p.orgId = o._key";
+
+    /** */
+    private static final String ORG_CACHE_NAME = "org";
+
+    /** */
+    private static final String PERSON_CAHE_NAME = "pers";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheKeyConfiguration keyCfg = new CacheKeyConfiguration("MyCache", "affKey");
+
+        cfg.setCacheKeyConfiguration(keyCfg);
+
+        cfg.setPeerClassLoadingEnabled(false);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(ipFinder);
+
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    /** @return number of nodes to be prestarted. */
+    private int nodesCount() {
+        return 1;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        startGrids(nodesCount());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * @param name Cache name.
+     * @param mode Cache mode.
+     * @param idxTypes Indexed types.
+     * @return Cache configuration.
+     */
+    protected  CacheConfiguration cacheConfig(String name, TestCacheMode mode, Class... idxTypes) {
+        return new CacheConfiguration()
+            .setName(name)
+            .setCacheMode(mode == TestCacheMode.REPLICATED ? CacheMode.REPLICATED : CacheMode.PARTITIONED)
+            .setQueryParallelism(mode == TestCacheMode.SEGMENTED ? 5 : 1)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setIndexedTypes(idxTypes);
+    }
+
+    /**
+     *
+     */
+    public void testDistributedQueryOnPartitionedCaches() {
+        createCachesAndExecuteQuery(TestCacheMode.PARTITIONED, TestCacheMode.PARTITIONED, false, false);
+
+        assertDistributedQuery();
+    }
+
+    /**
+     *
+     */
+    public void testDistributedQueryOnPartitionedAndReplicatedCache() {
+        createCachesAndExecuteQuery(TestCacheMode.PARTITIONED, TestCacheMode.REPLICATED, false, false);
+
+        assertDistributedQuery();
+    }
+
+    /**
+     *
+     */
+    public void testDistributedQueryOnReplicatedCaches() {
+        createCachesAndExecuteQuery(TestCacheMode.REPLICATED, TestCacheMode.REPLICATED, false, false);
+
+        assertDistributedQuery();
+    }
+
+    /**
+     *
+     */
+    public void testDistributedQueryOnSegmentedCaches() {
+        createCachesAndExecuteQuery(TestCacheMode.SEGMENTED, TestCacheMode.SEGMENTED, false, false);
+
+        assertDistributedQuery();
+    }
+
+    /**
+     *
+     */
+    public void testDistributedQueryOnReplicatedAndSegmentedCache() {
+        createCachesAndExecuteQuery(TestCacheMode.REPLICATED, TestCacheMode.SEGMENTED, false, false);
+
+        assertDistributedQuery();
+    }
+
+    /**
+     *
+     */
+    public void testDistributedQueryOnPartitionedCachesWithReplicatedFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.PARTITIONED, TestCacheMode.PARTITIONED, true, false);
+
+        assertDistributedQuery();
+    }
+
+    /**
+     *
+     */
+    public void testDistributedQueryOnPartitionedAndReplicatedCacheWithReplicatedFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.PARTITIONED, TestCacheMode.REPLICATED, true, false);
+
+        assertDistributedQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalQueryOnReplicatedCachesWithReplicatedFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.REPLICATED, TestCacheMode.REPLICATED, true, false);
+
+        assertLocalQuery();
+    }
+
+    /**
+     *
+     */
+    public void testDistributedQueryOnSegmentedCachesWithReplicatedFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.SEGMENTED, TestCacheMode.SEGMENTED, true, false);
+
+        assertDistributedQuery();
+    }
+
+    /**
+     *
+     */
+    public void testDistributedQueryOnReplicatedAndSegmentedCacheWithReplicatedFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.REPLICATED, TestCacheMode.SEGMENTED, true, false);
+
+        assertDistributedQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalQueryOnPartitionedCachesWithLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.PARTITIONED, TestCacheMode.PARTITIONED, false, true);
+
+        assertLocalQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalQueryOnPartitionedAndReplicatedCacheWithLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.PARTITIONED, TestCacheMode.REPLICATED, false, true);
+
+        assertLocalQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalQueryOnReplicatedCachesWithLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.REPLICATED, TestCacheMode.REPLICATED, false, true);
+
+        assertLocalQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalTwoStepQueryOnSegmentedCachesWithLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.SEGMENTED, TestCacheMode.SEGMENTED, false, true);
+
+        assertLocalTwoStepQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalTwoStepQueryOnReplicatedAndSegmentedCacheWithLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.REPLICATED, TestCacheMode.SEGMENTED, false, true);
+
+        assertLocalTwoStepQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalQueryOnPartitionedCachesWithReplicatedAndLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.PARTITIONED, TestCacheMode.PARTITIONED, false, true);
+
+        assertLocalQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalQueryOnPartitionedAndReplicatedCacheWithReplicatedAndLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.PARTITIONED, TestCacheMode.REPLICATED, true, true);
+
+        assertLocalQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalQueryOnReplicatedCachesWithReplicatedAndLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.REPLICATED, TestCacheMode.REPLICATED, true, true);
+
+        assertLocalQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalTwoStepQueryOnSegmentedCachesWithReplicatedAndLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.SEGMENTED, TestCacheMode.SEGMENTED, true, true);
+
+        assertLocalTwoStepQuery();
+    }
+
+    /**
+     *
+     */
+    public void testLocalTwoStepQueryOnReplicatedAndSegmentedCacheWithReplicatedAndLocalFlag() {
+        createCachesAndExecuteQuery(TestCacheMode.REPLICATED, TestCacheMode.SEGMENTED, true, true);
+
+        assertLocalTwoStepQuery();
+    }
+
+    /**
+     * @param firstCacheMode First cache mode.
+     * @param secondCacheMode Second cache mode.
+     * @param replicatedOnly Replicated only query flag.
+     * @param loc Local query flag.
+     */
+    private void createCachesAndExecuteQuery(TestCacheMode firstCacheMode, TestCacheMode secondCacheMode,
+        boolean replicatedOnly, boolean loc) {
+        Ignite node = ignite(0);
+
+        node.createCache(cacheConfig(PERSON_CAHE_NAME, firstCacheMode, Integer.class, Person.class));
+        node.createCache(cacheConfig(ORG_CACHE_NAME, secondCacheMode, Integer.class, Organization.class));
+
+        IgniteCache c = node.cache(PERSON_CAHE_NAME);
+
+        c.query(new SqlFieldsQuery(SELECT).setReplicatedOnly(replicatedOnly).setLocal(loc)).getAll();
+    }
+
+    /**
+     * @return Cached two-step query, or {@code null} if none occurred.
+     */
+    private GridCacheTwoStepQuery cachedTwoStepQuery() {
+        GridQueryIndexing idx = grid(0).context().query().getIndexing();
+
+        Map m = U.field(idx, "twoStepCache");
+
+        if (m.isEmpty())
+            return null;
+
+        H2TwoStepCachedQuery q = m.values().iterator().next();
+
+        return q.query();
+    }
+
+    /**
+     * Check that truly distributed query has happened.
+     */
+    private void assertDistributedQuery() {
+        GridCacheTwoStepQuery q = cachedTwoStepQuery();
+
+        assertNotNull(q);
+
+        assertFalse(q.isLocal());
+    }
+
+    /**
+     * Check that local two-step query has happened.
+     */
+    private void assertLocalTwoStepQuery() {
+        GridCacheTwoStepQuery q = cachedTwoStepQuery();
+
+        assertNotNull(q);
+
+        assertTrue(q.isLocal());
+    }
+
+
+    /**
+     * Check that no distributed query has happened.
+     */
+    private void assertLocalQuery() {
+        GridCacheTwoStepQuery q = cachedTwoStepQuery();
+
+        assertNull(q);
+    }
+
+    /**
+     *
+     */
+    private static class Person implements Serializable {
+        /** */
+        @QuerySqlField(index = true)
+        Integer orgId;
+
+        /** */
+        @QuerySqlField
+        String name;
+
+        /**
+         *
+         */
+        public Person() {
+            // No-op.
+        }
+
+        /**
+         * @param orgId Organization ID.
+         * @param name Name.
+         */
+        public Person(int orgId, String name) {
+            this.orgId = orgId;
+            this.name = name;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Organization implements Serializable {
+        /** */
+        @QuerySqlField
+        String name;
+
+        /**
+         *
+         */
+        public Organization() {
+            // No-op.
+        }
+
+        /**
+         * @param name Organization name.
+         */
+        public Organization(String name) {
+            this.name = name;
+        }
+    }
+
+    /**
+     * Mode for test cache.
+     */
+    private enum TestCacheMode {
+        /** */
+        SEGMENTED,
+
+        /** */
+        PARTITIONED,
+
+        /** */
+        REPLICATED
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteQueryDedicatedPoolTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteQueryDedicatedPoolTest.java
index 13c0cb2c5e73d..b2f4e47ec2048 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteQueryDedicatedPoolTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteQueryDedicatedPoolTest.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoManager;
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
+import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.spi.IgniteSpiAdapter;
@@ -96,14 +97,20 @@ public class IgniteQueryDedicatedPoolTest extends GridCommonAbstractTest {
     }
 
     /**
-     * Tests that SQL queries are executed in dedicated pool
+     * Tests that SQL queries involving actual network IO are executed in dedicated pool.
      * @throws Exception If failed.
+     * @see GridCacheTwoStepQuery#isLocal()
      */
     public void testSqlQueryUsesDedicatedThreadPool() throws Exception {
         try (Ignite client = startGrid("client")) {
             IgniteCache cache = client.cache(CACHE_NAME);
 
-            QueryCursor> cursor = cache.query(new SqlFieldsQuery("select currentPolicy()"));
+            // We do this in order to have 1 row in results of select - function is called once per each row of result.
+            cache.put(1, 1);
+
+            // We have to refer to a cache explicitly in the query in order for it to be executed
+            // in non local distributed manner (yes, there's a "local distributed" manner too - see link above...)
+            QueryCursor> cursor = cache.query(new SqlFieldsQuery("select currentPolicy() from Integer"));
 
             List> result = cursor.getAll();
 
@@ -113,8 +120,8 @@ public void testSqlQueryUsesDedicatedThreadPool() throws Exception {
 
             Byte plc = (Byte)result.get(0).get(0);
 
-            assert plc != null;
-            assert plc == GridIoPolicy.QUERY_POOL;
+            assertNotNull(plc);
+            assertEquals(GridIoPolicy.QUERY_POOL, (byte)plc);
         }
     }
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDefaultValueTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDefaultValueTest.java
index 6747e2810dd4a..2edc93b5ec5d2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDefaultValueTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDefaultValueTest.java
@@ -228,7 +228,7 @@ private void checkResults(Collection> exp, Collection> actu
      * @return Results set.
      */
     private List> sql(String sql, Object ... args) {
-        return grid(NODE_CLIENT).context().query().querySqlFieldsNoCache(
+        return grid(NODE_CLIENT).context().query().querySqlFields(
             new SqlFieldsQuery(sql).setArgs(args), false).getAll();
     }
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
index 0c3b42c483962..1f4e018889a3b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
@@ -1020,7 +1020,7 @@ private void executeForNodeAndCache(CacheConfiguration ccfg, Ignite ignite, Test
     private List> executeSql(String sqlText) throws Exception {
         GridQueryProcessor qryProc = grid(NODE_CLIENT).context().query();
 
-        return qryProc.querySqlFieldsNoCache(new SqlFieldsQuery(sqlText), true).getAll();
+        return qryProc.querySqlFields(new SqlFieldsQuery(sqlText), true).getAll();
     }
 
     /** */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/MultipleStatementsSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/MultipleStatementsSqlQuerySelfTest.java
index 8b9bf40eab6f2..becd5865c642c 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/MultipleStatementsSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/MultipleStatementsSqlQuerySelfTest.java
@@ -60,7 +60,7 @@ public void testQuery() throws Exception {
                 "select * from test;")
             .setSchema("PUBLIC");
 
-        List>> res = qryProc.querySqlFieldsNoCache(qry, true, false);
+        List>> res = qryProc.querySqlFields(qry, true, false);
 
         assert res.size() == 4 : "Unexpected cursors count: " + res.size();
 
@@ -106,7 +106,7 @@ public void testQueryWithParameters() throws Exception {
             .setSchema("PUBLIC")
             .setArgs(1, "name_1", 2, "name2", 3, "name_3");
 
-        List>> res = qryProc.querySqlFieldsNoCache(qry, true, false);
+        List>> res = qryProc.querySqlFields(qry, true, false);
 
         assert res.size() == 4 : "Unexpected cursors count: " + res.size();
 
@@ -145,10 +145,10 @@ public void testQueryMultipleStatementsFailed() throws Exception {
         GridTestUtils.assertThrows(log,
             new Callable() {
                 @Override public Object call() throws Exception {
-                    node.context().query().querySqlFieldsNoCache(qry, true);
+                    node.context().query().querySqlFields(qry, true, true);
 
                     return null;
                 }
             }, IgniteSQLException.class, "Multiple statements queries are not supported");
     }
-}
\ No newline at end of file
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSchemaSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSchemaSelfTest.java
index a4ee2e3a14d68..b271d806d70c0 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSchemaSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSchemaSelfTest.java
@@ -66,13 +66,13 @@ public void testQueryWithoutCacheOnPublicSchema() throws Exception {
 
         SqlFieldsQuery qry = new SqlFieldsQuery("SELECT 1").setSchema("PUBLIC");
 
-        List> res = qryProc.querySqlFieldsNoCache(qry, true).getAll();
+        List> res = qryProc.querySqlFields(qry, true).getAll();
 
         assertEquals(1, res.size());
         assertEquals(1, res.get(0).size());
         assertEquals(1, res.get(0).get(0));
 
-        Iterator> iter = qryProc.querySqlFieldsNoCache(qry, true).iterator();
+        Iterator> iter = qryProc.querySqlFields(qry, true).iterator();
 
         assertTrue(iter.hasNext());
 
@@ -98,13 +98,13 @@ public void testQueryWithoutCacheOnCacheSchema() throws Exception {
 
         SqlFieldsQuery qry = new SqlFieldsQuery("SELECT 1").setSchema(CACHE_PERSON);
 
-        List> res = qryProc.querySqlFieldsNoCache(qry, true).getAll();
+        List> res = qryProc.querySqlFields(qry, true).getAll();
 
         assertEquals(1, res.size());
         assertEquals(1, res.get(0).size());
         assertEquals(1, res.get(0).get(0));
 
-        Iterator> iter = qryProc.querySqlFieldsNoCache(qry, true).iterator();
+        Iterator> iter = qryProc.querySqlFields(qry, true).iterator();
 
         assertTrue(iter.hasNext());
 
@@ -236,7 +236,7 @@ public void testCustomSchemaConcurrentUse() throws Exception {
             @Override public void run() {
                 for (int i = 0; i < 100; i++) {
                     int idx = maxIdx.incrementAndGet();
-                    
+
                     String tbl = "Person" + idx;
 
                     IgniteCache cache = registerQueryEntity(tbl, "PersonCache" + idx);
@@ -282,7 +282,7 @@ private IgniteCache registerQueryEntity(String tbl, String cacheNa
     private void testQueryEntity(IgniteCache cache, String tbl) {
         cache.put(1L, new Person("Vasya", 2));
 
-        assertEquals(1, node.context().query().querySqlFieldsNoCache(
+        assertEquals(1, node.context().query().querySqlFields(
             new SqlFieldsQuery(String.format("SELECT id, name, orgId FROM TEST.%s where (id = %d)", tbl, 1)), false
         ).getAll().size());
     }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index f4eb393425ee1..68610a1493282 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -137,6 +137,7 @@
 import org.apache.ignite.internal.processors.cache.query.IndexingSpiQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.query.IndexingSpiQueryTxSelfTest;
 import org.apache.ignite.internal.processors.client.ClientConnectorConfigurationValidationSelfTest;
+import org.apache.ignite.internal.processors.query.IgniteCachelessQueriesSelfTest;
 import org.apache.ignite.internal.processors.database.baseline.IgniteStableBaselineBinObjFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.query.IgniteSqlDefaultValueTest;
 import org.apache.ignite.internal.processors.query.IgniteSqlDistributedJoinSelfTest;
@@ -224,6 +225,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(LazyQuerySelfTest.class);
         suite.addTestSuite(IgniteSqlSplitterSelfTest.class);
         suite.addTestSuite(IgniteSqlSegmentedIndexSelfTest.class);
+        suite.addTestSuite(IgniteCachelessQueriesSelfTest.class);
         suite.addTestSuite(IgniteSqlSegmentedIndexMultiNodeSelfTest.class);
         suite.addTestSuite(IgniteSqlSchemaIndexingTest.class);
         suite.addTestSuite(GridCacheQueryIndexDisabledSelfTest.class);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/SqlQueryTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/SqlQueryTest.cs
index b9de090c94be3..02d13f6fa2746 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/SqlQueryTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/SqlQueryTest.cs
@@ -100,9 +100,8 @@ public void TestFieldsQuery()
             Assert.AreEqual("ID", cursor.FieldNames.Single());
 
             // All items local.
-            // TODO: IGNITE-5571 - exception should be fixed.
             qry.Local = true;
-            Assert.Throws(() => Assert.Greater(Count, cache.Query(qry).Count()));
+            Assert.Greater(Count, cache.Query(qry).Count());
 
             // Filter.
             qry = new SqlFieldsQuery("select Name from Person where Id = ?", 1)

From 74352a5f7bb8170a7145f29c0b10eb0ff0541f7a Mon Sep 17 00:00:00 2001
From: devozerov 
Date: Tue, 23 Jan 2018 10:34:17 +0300
Subject: [PATCH 32/65] Fixed missing Apache header.

---
 .../cache/persistence/AllocatedPageTracker.java | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AllocatedPageTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AllocatedPageTracker.java
index bd2a13ef63b06..8f3ff686afa2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AllocatedPageTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/AllocatedPageTracker.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.ignite.internal.processors.cache.persistence;
 
 /**

From 61f659dde38fce7ed7356a0c467975e8c65d0af5 Mon Sep 17 00:00:00 2001
From: YuriBabak 
Date: Tue, 23 Jan 2018 13:03:34 +0300
Subject: [PATCH 33/65] IGNITE-7456: Fix wrong batch logic in distributed MLP
 training.

this closes #3398

(cherry picked from commit 6bb6b3e)
---
 .../ml/nn/MLPGroupTrainerExample.java         | 17 ++--
 .../ml/nn/MLPLocalTrainerExample.java         |  2 +-
 .../java/org/apache/ignite/IgniteLogger.java  | 92 +++++++++++++++---
 .../nn/MLPGroupUpdateTrainerCacheInput.java   | 28 +++---
 .../distributed/MLPGroupUpdateTrainer.java    | 74 +++++++++------
 .../MLPGroupUpdateTrainingData.java           |  6 +-
 .../MLPGroupUpdateTrainingLoopData.java       |  6 +-
 .../distributed/MLPMetaoptimizer.java         |  3 +-
 .../trainers/local/MLPLocalBatchTrainer.java  |  4 +-
 .../RPropUpdateCalculator.java                |  6 +-
 ...eter.java => SimpleGDParameterUpdate.java} | 50 ++++++----
 .../SimpleGDUpdateCalculator.java             | 38 +++++---
 .../trainers/group/BaseLocalProcessorJob.java |  3 +-
 .../group/GroupTrainerBaseProcessorTask.java  |  3 +-
 .../ml/trainers/group/ResultAndUpdates.java   |  3 +-
 .../ml/trainers/group/UpdateStrategies.java   | 47 ++++++++++
 .../ml/trainers/group/UpdatesStrategy.java    | 94 +++++++++++++++++++
 .../ml/trainers/local/LocalBatchTrainer.java  |  8 +-
 .../java/org/apache/ignite/ml/util/Utils.java |  2 +-
 .../ignite/ml/nn/MLPGroupTrainerTest.java     | 38 ++++++--
 .../ignite/ml/nn/MLPLocalTrainerTest.java     |  8 +-
 .../ml/nn/performance/MnistDistributed.java   |  6 +-
 .../ignite/ml/nn/performance/MnistLocal.java  |  4 +-
 23 files changed, 410 insertions(+), 132 deletions(-)
 rename modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/{SimpleGDParameter.java => SimpleGDParameterUpdate.java} (54%)
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdateStrategies.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdatesStrategy.java

diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPGroupTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPGroupTrainerExample.java
index d106fadfee423..d45e9578f3b92 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPGroupTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPGroupTrainerExample.java
@@ -40,8 +40,7 @@
 import org.apache.ignite.thread.IgniteThread;
 
 /**
- * 

- * Example of using distributed {@link MultilayerPerceptron}.

+ * Example of using distributed {@link MultilayerPerceptron}.
  * 

  * Remote nodes should always be started with special configuration file which
  * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.

@@ -57,7 +56,7 @@ public class MLPGroupTrainerExample {
      */
     public static void main(String[] args) throws InterruptedException {
         // IMPL NOTE based on MLPGroupTrainerTest#testXOR
-        System.out.println(">>> Distributed  multilayer perceptron example started.");
+        System.out.println(">>> Distributed multilayer perceptron example started.");
 
         // Start ignite grid.
         try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
@@ -68,7 +67,7 @@ public static void main(String[] args) throws InterruptedException {
             IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
                 MLPGroupTrainerExample.class.getSimpleName(), () -> {
 
-                int samplesCnt = 1000;
+                int samplesCnt = 10000;
 
                 Matrix xorInputs = new DenseLocalOnHeapMatrix(
                     new double[][] {{0.0, 0.0}, {0.0, 1.0}, {1.0, 0.0}, {1.0, 1.0}},
@@ -88,7 +87,7 @@ public static void main(String[] args) throws InterruptedException {
 
                 try (IgniteDataStreamer> streamer =
                          ignite.dataStreamer(cacheName)) {
-                    streamer.perNodeBufferSize(10000);
+                    streamer.perNodeBufferSize(100);
 
                     for (int i = 0; i < samplesCnt; i++) {
                         int col = Math.abs(rnd.nextInt()) % 4;
@@ -99,14 +98,14 @@ public static void main(String[] args) throws InterruptedException {
                 int totalCnt = 100;
                 int failCnt = 0;
                 MLPGroupUpdateTrainer trainer = MLPGroupUpdateTrainer.getDefault(ignite).
-                    withSyncRate(3).
+                    withSyncPeriod(3).
                     withTolerance(0.001).
-                    withMaxGlobalSteps(1000);
+                    withMaxGlobalSteps(20);
 
                 for (int i = 0; i < totalCnt; i++) {
 
                     MLPGroupUpdateTrainerCacheInput trainerInput = new MLPGroupUpdateTrainerCacheInput(conf,
-                        new RandomInitializer(rnd), 6, cache, 4);
+                        new RandomInitializer(rnd), 6, cache, 10);
 
                     MultilayerPerceptron mlp = trainer.train(trainerInput);
 
@@ -125,7 +124,7 @@ public static void main(String[] args) throws InterruptedException {
 
                 System.out.println("\n>>> Fail percentage: " + (failRatio * 100) + "%.");
 
-                System.out.println("\n>>> Distributed  multilayer perceptron example completed.");
+                System.out.println("\n>>> Distributed multilayer perceptron example completed.");
             });
 
             igniteThread.start();
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPLocalTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPLocalTrainerExample.java
index b5574587e3d72..02280ce021cb1 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPLocalTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPLocalTrainerExample.java
@@ -67,7 +67,7 @@ public static void main(String[] args) {
         System.out.println("\n>>> Perform training.");
 
         MultilayerPerceptron mlp = new MLPLocalBatchTrainer<>(LossFunctions.MSE,
-            () -> new RPropUpdateCalculator<>(),
+            RPropUpdateCalculator::new,
             0.0001,
             16000).train(trainerInput);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java b/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
index b1433a8ac230a..412e4141a9c42 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteLogger.java
@@ -65,41 +65,81 @@
  */
 @GridToStringExclude
 public interface IgniteLogger {
+    /**
+     * Marker for log messages that are useful in development environments, but not in production.
+     */
+    String DEV_ONLY = "DEV_ONLY";
+
     /**
      * Creates new logger with given category based off the current instance.
      *
      * @param ctgr Category for new logger.
      * @return New logger with given category.
      */
-    public IgniteLogger getLogger(Object ctgr);
+    IgniteLogger getLogger(Object ctgr);
+
+    /**
+     * Logs out trace message.
+     *
+     * @param msg Trace message.
+     */
+    void trace(String msg);
 
     /**
      * Logs out trace message.
+     * The default implementation calls {@code this.trace(msg)}.
      *
+     * @param marker Name of the marker to be associated with the message.
      * @param msg Trace message.
      */
-    public void trace(String msg);
+    default void trace(@Nullable String marker, String msg) {
+        trace(msg);
+    }
 
     /**
      * Logs out debug message.
      *
      * @param msg Debug message.
      */
-    public void debug(String msg);
+    void debug(String msg);
+
+    /**
+     * Logs out debug message.
+     * The default implementation calls {@code this.debug(msg)}.
+     *
+     * @param marker Name of the marker to be associated with the message.
+     * @param msg Debug message.
+     */
+    default void debug(@Nullable String marker, String msg) {
+        debug(msg);
+    }
 
     /**
      * Logs out information message.
      *
      * @param msg Information message.
      */
-    public void info(String msg);
+    void info(String msg);
+
+    /**
+     * Logs out information message.
+     * The default implementation calls {@code this.info(msg)}.
+     *
+     * @param marker Name of the marker to be associated with the message.
+     * @param msg Information message.
+     */
+    default void info(@Nullable String marker, String msg) {
+        info(msg);
+    }
 
     /**
      * Logs out warning message.
      *
      * @param msg Warning message.
      */
-    public void warning(String msg);
+    default void warning(String msg) {
+        warning(msg, null);
+    }
 
     /**
      * Logs out warning message with optional exception.
@@ -107,55 +147,81 @@ public interface IgniteLogger {
      * @param msg Warning message.
      * @param e Optional exception (can be {@code null}).
      */
-    public void warning(String msg, @Nullable Throwable e);
+    void warning(String msg, @Nullable Throwable e);
+
+    /**
+     * Logs out warning message with optional exception.
+     * The default implementation calls {@code this.warning(msg)}.
+     *
+     * @param marker Name of the marker to be associated with the message.
+     * @param msg Warning message.
+     * @param e Optional exception (can be {@code null}).
+     */
+    default void warning(@Nullable String marker, String msg, @Nullable Throwable e) {
+        warning(msg, e);
+    }
 
     /**
      * Logs out error message.
      *
      * @param msg Error message.
      */
-    public void error(String msg);
+    default void error(String msg) {
+        error(null, msg, null);
+    }
+
+    /**
+     * Logs error message with optional exception.
+     *
+     * @param msg Error message.
+     * @param e Optional exception (can be {@code null}).
+     */
+    void error(String msg, @Nullable Throwable e);
 
     /**
      * Logs error message with optional exception.
+     * The default implementation calls {@code this.error(msg)}.
      *
+     * @param marker Name of the marker to be associated with the message.
      * @param msg Error message.
      * @param e Optional exception (can be {@code null}).
      */
-    public void error(String msg, @Nullable Throwable e);
+    default void error(@Nullable String marker, String msg, @Nullable Throwable e) {
+        error(msg, e);
+    }
 
     /**
      * Tests whether {@code trace} level is enabled.
      *
      * @return {@code true} in case when {@code trace} level is enabled, {@code false} otherwise.
      */
-    public boolean isTraceEnabled();
+    boolean isTraceEnabled();
 
     /**
      * Tests whether {@code debug} level is enabled.
      *
      * @return {@code true} in case when {@code debug} level is enabled, {@code false} otherwise.
      */
-    public boolean isDebugEnabled();
+    boolean isDebugEnabled();
 
     /**
      * Tests whether {@code info} level is enabled.
      *
      * @return {@code true} in case when {@code info} level is enabled, {@code false} otherwise.
      */
-    public boolean isInfoEnabled();
+    boolean isInfoEnabled();
 
     /**
      * Tests whether Logger is in "Quiet mode".
      *
      * @return {@code true} "Quiet mode" is enabled, {@code false} otherwise
      */
-    public boolean isQuiet();
+    boolean isQuiet();
 
     /**
      * Gets name of the file being logged to if one is configured or {@code null} otherwise.
      *
      * @return Name of the file being logged to if one is configured or {@code null} otherwise.
      */
-    public String fileName();
+    String fileName();
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPGroupUpdateTrainerCacheInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPGroupUpdateTrainerCacheInput.java
index 783effa089efa..ce42938db3885 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPGroupUpdateTrainerCacheInput.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPGroupUpdateTrainerCacheInput.java
@@ -60,7 +60,7 @@ public class MLPGroupUpdateTrainerCacheInput extends AbstractMLPGroupUpdateTrain
     /**
      * Random number generator.
      */
-    private Random rand;
+    private final Random rand;
 
     /**
      * Construct instance of this class with given parameters.
@@ -80,6 +80,7 @@ public MLPGroupUpdateTrainerCacheInput(MLPArchitecture arch, MLPInitializer init
         this.batchSize = batchSize;
         this.cache = cache;
         this.mlp = new MultilayerPerceptron(arch, init);
+        this.rand = rand;
     }
 
     /**
@@ -94,17 +95,17 @@ public MLPGroupUpdateTrainerCacheInput(MLPArchitecture arch, MLPInitializer init
     public MLPGroupUpdateTrainerCacheInput(MLPArchitecture arch, MLPInitializer init,
         int networksCnt, IgniteCache> cache,
         int batchSize) {
-        this(arch, init, networksCnt, cache, batchSize, new Random());
+        this(arch, init, networksCnt, cache, batchSize, null);
     }
 
-        /**
-         * Construct instance of this class with given parameters and default initializer.
-         *
-         * @param arch Architecture of multilayer perceptron.
-         * @param networksCnt Count of networks to be trained in parallel by {@link MLPGroupUpdateTrainer}.
-         * @param cache Cache with labeled vectors.
-         * @param batchSize Size of batch to return on each training iteration.
-         */
+    /**
+     * Construct instance of this class with given parameters and default initializer.
+     *
+     * @param arch Architecture of multilayer perceptron.
+     * @param networksCnt Count of networks to be trained in parallel by {@link MLPGroupUpdateTrainer}.
+     * @param cache Cache with labeled vectors.
+     * @param batchSize Size of batch to return on each training iteration.
+     */
     public MLPGroupUpdateTrainerCacheInput(MLPArchitecture arch, int networksCnt,
         IgniteCache> cache,
         int batchSize) {
@@ -114,8 +115,9 @@ public MLPGroupUpdateTrainerCacheInput(MLPArchitecture arch, int networksCnt,
     /** {@inheritDoc} */
     @Override public IgniteSupplier> batchSupplier() {
         String cName = cache.getName();
-        int bs = batchSize;
-        Random r = rand;
+
+        int bs = batchSize; // This line is for prohibiting of 'this' object be caught into serialization context of lambda.
+        Random r = rand; // This line is for prohibiting of 'this' object be caught into serialization context of lambda.
 
         return () -> {
             Ignite ignite = Ignition.localIgnite();
@@ -138,7 +140,7 @@ public MLPGroupUpdateTrainerCacheInput(MLPArchitecture arch, int networksCnt,
             Matrix groundTruth = new DenseLocalOnHeapMatrix(dimEntry.label().size(), bs);
 
             for (int i = 0; i < selected.length; i++) {
-                LabeledVector labeled = cache.get(selected[i]);
+                LabeledVector labeled = cache.get(keys.get(selected[i]));
 
                 inputs.assignColumn(i, labeled.features());
                 groundTruth.assignColumn(i, labeled.label());
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainer.java
index f4647d5b984ad..8e97d878a4470 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainer.java
@@ -35,12 +35,14 @@
 import org.apache.ignite.ml.math.util.MatrixUtil;
 import org.apache.ignite.ml.optimization.LossFunctions;
 import org.apache.ignite.ml.nn.MultilayerPerceptron;
+import org.apache.ignite.ml.optimization.SmoothParametrized;
 import org.apache.ignite.ml.optimization.updatecalculators.ParameterUpdateCalculator;
 import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
 import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
 import org.apache.ignite.ml.trainers.group.GroupTrainerCacheKey;
 import org.apache.ignite.ml.trainers.group.MetaoptimizerGroupTrainer;
 import org.apache.ignite.ml.trainers.group.ResultAndUpdates;
+import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
 import org.apache.ignite.ml.trainers.group.chain.EntryAndContext;
 import org.apache.ignite.ml.util.Utils;
 
@@ -78,9 +80,9 @@ public class MLPGroupUpdateTrainer extends
     private final int maxGlobalSteps;
 
     /**
-     * Synchronize updates between networks every syncRate steps.
+     * Synchronize updates between networks every syncPeriod steps.
      */
-    private final int syncRate;
+    private final int syncPeriod;
 
     /**
      * Function used to reduce updates from different networks (for example, averaging of gradients of all networks).
@@ -96,7 +98,7 @@ public class MLPGroupUpdateTrainer extends
     /**
      * Updates calculator.
      */
-    private final ParameterUpdateCalculator updateCalculator;
+    private final ParameterUpdateCalculator updateCalculator;
 
     /**
      * Default maximal count of global steps.
@@ -123,8 +125,8 @@ public class MLPGroupUpdateTrainer extends
     /**
      * Default update calculator.
      */
-    private static final ParameterUpdateCalculator
-        DEFAULT_UPDATE_CALCULATOR = new RPropUpdateCalculator<>();
+    private static final ParameterUpdateCalculator
+        DEFAULT_UPDATE_CALCULATOR = new RPropUpdateCalculator();
 
     /**
      * Default loss function.
@@ -140,16 +142,16 @@ public class MLPGroupUpdateTrainer extends
      * @param tolerance Error tolerance.
      */
     public MLPGroupUpdateTrainer(int maxGlobalSteps,
-        int syncRate,
+        int syncPeriod,
         IgniteFunction, U> allUpdatesReducer,
         IgniteFunction, U> locStepUpdatesReducer,
-        ParameterUpdateCalculator updateCalculator,
+        ParameterUpdateCalculator updateCalculator,
         IgniteFunction loss,
         Ignite ignite, double tolerance) {
         super(new MLPMetaoptimizer<>(allUpdatesReducer), MLPCache.getOrCreate(ignite), ignite);
 
         this.maxGlobalSteps = maxGlobalSteps;
-        this.syncRate = syncRate;
+        this.syncPeriod = syncPeriod;
         this.allUpdatesReducer = allUpdatesReducer;
         this.locStepUpdatesReducer = locStepUpdatesReducer;
         this.updateCalculator = updateCalculator;
@@ -174,7 +176,7 @@ public static MLPGroupUpdateTrainer getDefault(Ignite igni
 
         MLPGroupUpdateTrainerDataCache.getOrCreate(ignite).put(trainingUUID, new MLPGroupUpdateTrainingData<>(
             updateCalculator,
-            syncRate,
+            syncPeriod,
             locStepUpdatesReducer,
             data.batchSupplier(),
             loss,
@@ -237,26 +239,32 @@ MLPGroupUpdateTrainingContext>, MLPGroupUpdateTrainingLoopData> trainingLo
         return data -> {
             MultilayerPerceptron mlp = data.mlp();
 
-            MultilayerPerceptron mlpCp = Utils.copy(mlp);
-            ParameterUpdateCalculator updateCalculator = data.updateCalculator();
+            // Apply previous update.
+            MultilayerPerceptron newMlp = updateCalculator.update(mlp, data.previousUpdate());
+
+            MultilayerPerceptron mlpCp = Utils.copy(newMlp);
+            ParameterUpdateCalculator updateCalculator = data.updateCalculator();
             IgniteFunction loss = data.loss();
 
             // ParameterUpdateCalculator API to have proper way to setting loss.
             updateCalculator.init(mlpCp, loss);
 
-            U curUpdate = data.previousUpdate();
-
+            // Generate new update.
             int steps = data.stepsCnt();
             List updates = new ArrayList<>(steps);
-
-            IgniteBiTuple batch = data.batchSupplier().get();
+            U curUpdate = data.previousUpdate();
 
             for (int i = 0; i < steps; i++) {
+                IgniteBiTuple batch = data.batchSupplier().get();
                 Matrix input = batch.get1();
                 Matrix truth = batch.get2();
 
                 int batchSize = truth.columnSize();
 
+                curUpdate = updateCalculator.calculateNewUpdate(mlpCp, curUpdate, i, input, truth);
+                mlpCp = updateCalculator.update(mlpCp, curUpdate);
+                updates.add(curUpdate);
+
                 Matrix predicted = mlpCp.apply(input);
 
                 double err = MatrixUtil.zipFoldByColumns(predicted, truth, (predCol, truthCol) ->
@@ -264,18 +272,11 @@ MLPGroupUpdateTrainingContext>, MLPGroupUpdateTrainingLoopData> trainingLo
 
                 if (err < data.tolerance())
                     break;
-
-                mlpCp = updateCalculator.update(mlpCp, curUpdate);
-                updates.add(curUpdate);
-
-                curUpdate = updateCalculator.calculateNewUpdate(mlpCp, curUpdate, i, input, truth);
             }
 
-            U update = data.getUpdateReducer().apply(updates);
+            U accumulatedUpdate = data.getUpdateReducer().apply(updates);
 
-            MultilayerPerceptron newMlp = updateCalculator.update(mlp, data.previousUpdate());
-
-            return new ResultAndUpdates<>(update).
+            return new ResultAndUpdates<>(accumulatedUpdate).
                 updateCache(MLPCache.getOrCreate(Ignition.localIgnite()), data.key(),
                     new MLPGroupTrainingCacheValue(newMlp));
         };
@@ -333,18 +334,18 @@ MLPGroupUpdateTrainingContext>, ResultAndUpdates> final
      * @return New {@link MLPGroupUpdateTrainer} with new maxGlobalSteps value.
      */
     public MLPGroupUpdateTrainer withMaxGlobalSteps(int maxGlobalSteps) {
-        return new MLPGroupUpdateTrainer<>(maxGlobalSteps, syncRate, allUpdatesReducer, locStepUpdatesReducer,
+        return new MLPGroupUpdateTrainer<>(maxGlobalSteps, syncPeriod, allUpdatesReducer, locStepUpdatesReducer,
             updateCalculator, loss, ignite, tolerance);
     }
 
     /**
-     * Create new {@link MLPGroupUpdateTrainer} with new syncRate value.
+     * Create new {@link MLPGroupUpdateTrainer} with new syncPeriod value.
      *
-     * @param syncRate New syncRate value.
-     * @return New {@link MLPGroupUpdateTrainer} with new syncRate value.
+     * @param syncPeriod New syncPeriod value.
+     * @return New {@link MLPGroupUpdateTrainer} with new syncPeriod value.
      */
-    public MLPGroupUpdateTrainer withSyncRate(int syncRate) {
-        return new MLPGroupUpdateTrainer<>(maxGlobalSteps, syncRate
+    public MLPGroupUpdateTrainer withSyncPeriod(int syncPeriod) {
+        return new MLPGroupUpdateTrainer<>(maxGlobalSteps, syncPeriod
             , allUpdatesReducer, locStepUpdatesReducer, updateCalculator, loss, ignite, tolerance);
     }
 
@@ -355,7 +356,18 @@ public MLPGroupUpdateTrainer withSyncRate(int syncRate) {
      * @return New {@link MLPGroupUpdateTrainer} with new tolerance value.
      */
     public MLPGroupUpdateTrainer withTolerance(double tolerance) {
-        return new MLPGroupUpdateTrainer<>(maxGlobalSteps, syncRate, allUpdatesReducer, locStepUpdatesReducer,
+        return new MLPGroupUpdateTrainer<>(maxGlobalSteps, syncPeriod, allUpdatesReducer, locStepUpdatesReducer,
             updateCalculator, loss, ignite, tolerance);
     }
+
+    /**
+     * Create new {@link MLPGroupUpdateTrainer} with new update strategy.
+     *
+     * @param stgy New update strategy.
+     * @return New {@link MLPGroupUpdateTrainer} with new tolerance value.
+     */
+    public  MLPGroupUpdateTrainer withUpdateStrategy(UpdatesStrategy stgy) {
+        return new MLPGroupUpdateTrainer<>(maxGlobalSteps, syncPeriod, stgy.allUpdatesReducer(), stgy.locStepUpdatesReducer(),
+            stgy.getUpdatesCalculator(), loss, ignite, tolerance);
+    }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingData.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingData.java
index 740fac6712127..3031c8f0e339e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingData.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingData.java
@@ -30,7 +30,7 @@
 /** Multilayer perceptron group update training data. */
 public class MLPGroupUpdateTrainingData {
     /** {@link ParameterUpdateCalculator}. */
-    private final ParameterUpdateCalculator updateCalculator;
+    private final ParameterUpdateCalculator updateCalculator;
 
     /**
      * Count of steps which should be done by each of parallel trainings before sending it's update for combining with
@@ -59,7 +59,7 @@ public class MLPGroupUpdateTrainingData {
 
     /** Construct multilayer perceptron group update training data with all parameters provided. */
     public MLPGroupUpdateTrainingData(
-        ParameterUpdateCalculator updateCalculator, int stepsCnt,
+        ParameterUpdateCalculator updateCalculator, int stepsCnt,
         IgniteFunction, U> updateReducer,
         IgniteSupplier> batchSupplier,
         IgniteFunction loss, double tolerance) {
@@ -72,7 +72,7 @@ public MLPGroupUpdateTrainingData(
     }
 
     /** Get update calculator. */
-    public ParameterUpdateCalculator updateCalculator() {
+    public ParameterUpdateCalculator updateCalculator() {
         return updateCalculator;
     }
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
index 2050ee59d9e1c..342e7d5889af9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPGroupUpdateTrainingLoopData.java
@@ -32,7 +32,7 @@
 /** Multilayer perceptron group update training loop data. */
 public class MLPGroupUpdateTrainingLoopData
 implements Serializable {
     /** {@link ParameterUpdateCalculator}. */
-    private final ParameterUpdateCalculator updateCalculator;
+    private final ParameterUpdateCalculator updateCalculator;
 
     /**
      * Count of steps which should be done by each of parallel trainings before sending it's update for combining with
@@ -63,7 +63,7 @@ public class MLPGroupUpdateTrainingLoopData
 implements Serializable {
 
     /** Create multilayer perceptron group update training loop data. */
     public MLPGroupUpdateTrainingLoopData(MultilayerPerceptron mlp,
-        ParameterUpdateCalculator updateCalculator, int stepsCnt,
+        ParameterUpdateCalculator updateCalculator, int stepsCnt,
         IgniteFunction, P> updateReducer, P previousUpdate,
         GroupTrainerCacheKey key, IgniteSupplier> batchSupplier,
         IgniteFunction loss,
@@ -85,7 +85,7 @@ public MultilayerPerceptron mlp() {
     }
 
     /** Get update calculator. */
-    public ParameterUpdateCalculator updateCalculator() {
+    public ParameterUpdateCalculator updateCalculator() {
         return updateCalculator;
     }
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
index 6e314f1d4e705..ff95a2773cf3e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/distributed/MLPMetaoptimizer.java
@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import org.apache.ignite.ml.math.functions.IgniteFunction;
 import org.apache.ignite.ml.trainers.group.Metaoptimizer;
@@ -65,7 +66,7 @@ public MLPMetaoptimizer(IgniteFunction, P> allUpdatesReducer) {
     @Override public P localProcessor(ArrayList
 input, MLPGroupUpdateTrainerLocalContext locCtx) {
         locCtx.incrementCurrentStep();
 
-        return allUpdatesReducer.apply(input);
+        return allUpdatesReducer.apply(input.stream().filter(Objects::nonNull).collect(Collectors.toList()));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
index 059d15afe1ab9..ebb78c0486712 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/trainers/local/MLPLocalBatchTrainer.java
@@ -61,7 +61,7 @@ public class MLPLocalBatchTrainer

      */
     public MLPLocalBatchTrainer(
         IgniteFunction loss,
-        IgniteSupplier> updaterSupplier,
+        IgniteSupplier> updaterSupplier,
         double errorThreshold, int maxIterations) {
         super(loss, updaterSupplier, errorThreshold, maxIterations);
     }
@@ -72,7 +72,7 @@ public MLPLocalBatchTrainer(
      * @return MLPLocalBatchTrainer with default parameters.
      */
     public static MLPLocalBatchTrainer getDefault() {
-        return new MLPLocalBatchTrainer<>(DEFAULT_LOSS, () -> new RPropUpdateCalculator<>(), DEFAULT_ERROR_THRESHOLD,
+        return new MLPLocalBatchTrainer<>(DEFAULT_LOSS, () -> new RPropUpdateCalculator(), DEFAULT_ERROR_THRESHOLD,
             DEFAULT_MAX_ITERATIONS);
     }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/RPropUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/RPropUpdateCalculator.java
index 80345d9b49d37..f706a6c8a260b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/RPropUpdateCalculator.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/RPropUpdateCalculator.java
@@ -30,7 +30,7 @@
  * 

  * See RProp.

  */
-public class RPropUpdateCalculator implements ParameterUpdateCalculator {
+public class RPropUpdateCalculator implements ParameterUpdateCalculator {
     /**
      * Default initial update.
      */
@@ -138,14 +138,14 @@ else if (sign < 0)
     }
 
     /** {@inheritDoc} */
-    @Override public RPropParameterUpdate init(M mdl,
+    @Override public RPropParameterUpdate init(SmoothParametrized mdl,
         IgniteFunction loss) {
         this.loss = loss;
         return new RPropParameterUpdate(mdl.parametersCount(), initUpdate);
     }
 
     /** {@inheritDoc} */
-    @Override public  M1 update(M1 obj, RPropParameterUpdate update) {
+    @Override public  M1 update(M1 obj, RPropParameterUpdate update) {
         Vector updatesToAdd = VectorUtils.elementWiseTimes(update.updatesMask().copy(), update.prevIterationUpdates());
         return (M1)obj.setParameters(obj.parameters().plus(updatesToAdd));
     }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDParameter.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDParameterUpdate.java
similarity index 54%
rename from modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDParameter.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDParameterUpdate.java
index 22fc18afb1983..13731ea4e1df9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDParameter.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDParameterUpdate.java
@@ -18,43 +18,35 @@
 package org.apache.ignite.ml.optimization.updatecalculators;
 
 import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 
 /**
  * Parameters for {@link SimpleGDUpdateCalculator}.
  */
-public class SimpleGDParameter implements Serializable {
-    /**
-     * Gradient.
-     */
+public class SimpleGDParameterUpdate implements Serializable {
+    /** Gradient. */
     private Vector gradient;
 
-    /**
-     * Learning rate.
-     */
-    private double learningRate;
-
     /**
      * Construct instance of this class.
      *
      * @param paramsCnt Count of parameters.
-     * @param learningRate Learning rate.
      */
-    public SimpleGDParameter(int paramsCnt, double learningRate) {
+    public SimpleGDParameterUpdate(int paramsCnt) {
         gradient = new DenseLocalOnHeapVector(paramsCnt);
-        this.learningRate = learningRate;
     }
 
     /**
      * Construct instance of this class.
      *
      * @param gradient Gradient.
-     * @param learningRate Learning rate.
      */
-    public SimpleGDParameter(Vector gradient, double learningRate) {
+    public SimpleGDParameterUpdate(Vector gradient) {
         this.gradient = gradient;
-        this.learningRate = learningRate;
     }
 
     /**
@@ -67,11 +59,31 @@ public Vector gradient() {
     }
 
     /**
-     * Get learning rate.
+     * Method used to sum updates inside of one of parallel trainings.
+     *
+     * @param updates Updates.
+     * @return Sum of SimpleGDParameterUpdate.
+     */
+    public static SimpleGDParameterUpdate sumLocal(List updates) {
+        Vector accumulatedGrad = updates.
+            stream().
+            filter(Objects::nonNull).
+            map(SimpleGDParameterUpdate::gradient).
+            reduce(Vector::plus).
+            orElse(null);
+
+        return accumulatedGrad != null ? new SimpleGDParameterUpdate(accumulatedGrad) : null;
+    }
+
+    /**
+     * Method used to get total update of all parallel trainings.
      *
-     * @return learning rate.
+     * @param updates Updates.
+     * @return Avg of SimpleGDParameterUpdate.
      */
-    public double learningRate() {
-        return learningRate;
+    public static SimpleGDParameterUpdate avg(List updates) {
+        SimpleGDParameterUpdate sum = sumLocal(updates);
+        return sum != null ? new SimpleGDParameterUpdate(sum.gradient().
+            divide(updates.stream().filter(Objects::nonNull).collect(Collectors.toList()).size())) : null;
     }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDUpdateCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDUpdateCalculator.java
index 291e63dbbb47e..f1023961219e3 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDUpdateCalculator.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/updatecalculators/SimpleGDUpdateCalculator.java
@@ -26,17 +26,21 @@
 /**
  * Simple gradient descent parameters updater.
  */
-public class SimpleGDUpdateCalculator implements ParameterUpdateCalculator {
-    /**
-     * Learning rate.
-     */
+public class SimpleGDUpdateCalculator implements ParameterUpdateCalculator {
+    /** Learning rate. */
     private double learningRate;
 
-    /**
-     * Loss function.
-     */
+    /** Loss function. */
     protected IgniteFunction loss;
 
+    /** Default learning rate. */
+    private static final double DEFAULT_LEARNING_RATE = 0.1;
+
+    /** Construct instance of this class with default parameters. */
+    public SimpleGDUpdateCalculator() {
+        this(DEFAULT_LEARNING_RATE);
+    }
+
     /**
      * Construct SimpleGDUpdateCalculator.
      *
@@ -47,21 +51,31 @@ public SimpleGDUpdateCalculator(double learningRate) {
     }
 
     /** {@inheritDoc} */
-    @Override public SimpleGDParameter init(M mdl,
+    @Override public SimpleGDParameterUpdate init(SmoothParametrized mdl,
         IgniteFunction loss) {
         this.loss = loss;
-        return new SimpleGDParameter(mdl.parametersCount(), learningRate);
+        return new SimpleGDParameterUpdate(mdl.parametersCount());
     }
 
     /** {@inheritDoc} */
-    @Override public SimpleGDParameter calculateNewUpdate(SmoothParametrized mlp, SimpleGDParameter updaterParameters,
+    @Override public SimpleGDParameterUpdate calculateNewUpdate(SmoothParametrized mlp, SimpleGDParameterUpdate updaterParameters,
         int iteration, Matrix inputs, Matrix groundTruth) {
-        return new SimpleGDParameter(mlp.differentiateByParameters(loss, inputs, groundTruth), learningRate);
+        return new SimpleGDParameterUpdate(mlp.differentiateByParameters(loss, inputs, groundTruth));
     }
 
     /** {@inheritDoc} */
-    @Override public  M1 update(M1 obj, SimpleGDParameter update) {
+    @Override public  M1 update(M1 obj, SimpleGDParameterUpdate update) {
         Vector params = obj.parameters();
         return (M1)obj.setParameters(params.minus(update.gradient().times(learningRate)));
     }
+
+    /**
+     * Create new instance of this class with same parameters as this one, but with new learning rate.
+     *
+     * @param learningRate Learning rate.
+     * @return New instance of this class with same parameters as this one, but with new learning rate.
+     */
+    public SimpleGDUpdateCalculator withLearningRate(double learningRate) {
+        return new SimpleGDUpdateCalculator(learningRate);
+    }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java
index d70b3f104b534..e20a55a9507e6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java
@@ -19,6 +19,7 @@
 
 import java.io.Serializable;
 import java.util.List;
+import java.util.Objects;
 import java.util.UUID;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -102,7 +103,7 @@ public BaseLocalProcessorJob(
             map(worker).
             collect(Collectors.toList());
 
-        ResultAndUpdates totalRes = ResultAndUpdates.sum(reducer, resultsAndUpdates);
+        ResultAndUpdates totalRes = ResultAndUpdates.sum(reducer, resultsAndUpdates.stream().filter(Objects::nonNull).collect(Collectors.toList()));
 
         totalRes.applyUpdates(ignite());
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerBaseProcessorTask.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerBaseProcessorTask.java
index 755f200a289b0..b192f421f1a88 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerBaseProcessorTask.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerBaseProcessorTask.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.UUID;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -122,7 +123,7 @@ public GroupTrainerBaseProcessorTask(UUID trainingUUID,
 
     /** {@inheritDoc} */
     @Nullable @Override public R reduce(List results) throws IgniteException {
-        return reducer.apply(results.stream().map(res -> (R)res.getData()).collect(Collectors.toList()));
+        return reducer.apply(results.stream().map(res -> (R)res.getData()).filter(Objects::nonNull).collect(Collectors.toList()));
     }
 
     /**
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ResultAndUpdates.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ResultAndUpdates.java
index 2e3f457d2c6ea..9ed18af5242e9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ResultAndUpdates.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ResultAndUpdates.java
@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import org.apache.ignite.Ignite;
@@ -137,7 +138,7 @@ static  ResultAndUpdates sum(IgniteFunction, R> reducer,
             }
         }
 
-        List results = resultsAndUpdates.stream().map(ResultAndUpdates::result).collect(Collectors.toList());
+        List results = resultsAndUpdates.stream().map(ResultAndUpdates::result).filter(Objects::nonNull).collect(Collectors.toList());
 
         return new ResultAndUpdates<>(reducer.apply(results), allUpdates);
     }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdateStrategies.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdateStrategies.java
new file mode 100644
index 0000000000000..33ec96a5ecd45
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdateStrategies.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.trainers.group;
+
+import org.apache.ignite.ml.optimization.SmoothParametrized;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
+
+/**
+ * Holder class for various update strategies.
+ */
+public class UpdateStrategies {
+    /**
+     * Simple GD update strategy.
+     *
+     * @return GD update strategy.
+     */
+    public static UpdatesStrategy GD() {
+        return new UpdatesStrategy<>(new SimpleGDUpdateCalculator(), SimpleGDParameterUpdate::sumLocal, SimpleGDParameterUpdate::avg);
+    }
+
+    /**
+     * RProp update strategy.
+     *
+     * @return RProp update strategy.
+     */
+    public static UpdatesStrategy RProp() {
+        return new UpdatesStrategy<>(new RPropUpdateCalculator(), RPropParameterUpdate::sumLocal, RPropParameterUpdate::avg);
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdatesStrategy.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdatesStrategy.java
new file mode 100644
index 0000000000000..9deb46039058e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdatesStrategy.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.trainers.group;
+
+import java.util.List;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.optimization.updatecalculators.ParameterUpdateCalculator;
+
+/**
+ * Class encapsulating update strategies for group trainers based on updates.
+ *
+ * @param  Type of model to be optimized.
+ * @param  Type of update.
+ */
+public class UpdatesStrategy {
+    /**
+     * {@link ParameterUpdateCalculator}.
+     */
+    private ParameterUpdateCalculator updatesCalculator;
+
+    /**
+     * Function used to reduce updates in one training (for example, sum all sequential gradient updates to get one
+     * gradient update).
+     */
+    private IgniteFunction, U> locStepUpdatesReducer;
+
+    /**
+     * Function used to reduce updates from different trainings (for example, averaging of gradients of all parallel trainings).
+     */
+    private IgniteFunction, U> allUpdatesReducer;
+
+    /**
+     * Construct instance of this class with given parameters.
+     *
+     * @param updatesCalculator Parameter update calculator.
+     * @param locStepUpdatesReducer Function used to reduce updates in one training
+     * (for example, sum all sequential gradient updates to get one gradient update).
+     * @param allUpdatesReducer Function used to reduce updates from different trainings
+     * (for example, averaging of gradients of all parallel trainings).
+     */
+    public UpdatesStrategy(
+        ParameterUpdateCalculator updatesCalculator,
+        IgniteFunction, U> locStepUpdatesReducer,
+        IgniteFunction, U> allUpdatesReducer) {
+        this.updatesCalculator = updatesCalculator;
+        this.locStepUpdatesReducer = locStepUpdatesReducer;
+        this.allUpdatesReducer = allUpdatesReducer;
+    }
+
+    /**
+     * Get parameter update calculator (see {@link ParameterUpdateCalculator}).
+     *
+     * @return Parameter update calculator.
+     */
+    public ParameterUpdateCalculator getUpdatesCalculator() {
+        return updatesCalculator;
+    }
+
+    /**
+     * Get function used to reduce updates in one training
+     * (for example, sum all sequential gradient updates to get one gradient update).
+     *
+     * @return Function used to reduce updates in one training
+     * (for example, sum all sequential gradient updates to get on gradient update).
+     */
+    public IgniteFunction, U> locStepUpdatesReducer() {
+        return locStepUpdatesReducer;
+    }
+
+    /**
+     * Get function used to reduce updates from different trainings
+     * (for example, averaging of gradients of all parallel trainings).
+     *
+     * @return Function used to reduce updates from different trainings.
+     */
+    public IgniteFunction, U> allUpdatesReducer() {
+        return allUpdatesReducer;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/local/LocalBatchTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/local/LocalBatchTrainer.java
index ab31f9ff8d60f..cb6fd899e7582 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/local/LocalBatchTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/local/LocalBatchTrainer.java
@@ -38,7 +38,7 @@ public class LocalBatchTrainer, P>
     /**
      * Supplier for updater function.
      */
-    private final IgniteSupplier> updaterSupplier;
+    private final IgniteSupplier> updaterSupplier;
 
     /**
      * Error threshold.
@@ -69,7 +69,7 @@ public class LocalBatchTrainer, P>
      * @param maxIterations Maximal iterations count.
      */
     public LocalBatchTrainer(IgniteFunction loss,
-        IgniteSupplier> updaterSupplier, double errorThreshold, int maxIterations) {
+        IgniteSupplier> updaterSupplier, double errorThreshold, int maxIterations) {
         this.loss = loss;
         this.updaterSupplier = updaterSupplier;
         this.errorThreshold = errorThreshold;
@@ -82,7 +82,7 @@ public LocalBatchTrainer(IgniteFunction updater = updaterSupplier.get();
+        ParameterUpdateCalculator updater = updaterSupplier.get();
 
         P updaterParams = updater.init(mdl, loss);
 
@@ -130,7 +130,7 @@ public LocalBatchTrainer withLoss(IgniteFunction> updaterSupplier) {
+    public LocalBatchTrainer withUpdater(IgniteSupplier> updaterSupplier) {
         return new LocalBatchTrainer<>(loss, updaterSupplier, errorThreshold, maxIterations);
     }
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/util/Utils.java b/modules/ml/src/main/java/org/apache/ignite/ml/util/Utils.java
index 206e1e9fc71cf..ed0ebd3e85f9a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/util/Utils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/util/Utils.java
@@ -53,7 +53,7 @@ public static  T copy(T orig) {
             obj = in.readObject();
         }
         catch (IOException | ClassNotFoundException e) {
-            throw new IgniteException("Couldn't copy the object.");
+            throw new IgniteException("Couldn't copy the object.", e);
         }
 
         return (T)obj;
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPGroupTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPGroupTrainerTest.java
index 151fead16381a..abd8ad293e549 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPGroupTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPGroupTrainerTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.ml.nn;
 
+import java.io.Serializable;
 import java.util.Random;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
@@ -31,8 +32,11 @@
 import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
 import org.apache.ignite.ml.nn.initializers.RandomInitializer;
 import org.apache.ignite.ml.nn.trainers.distributed.MLPGroupUpdateTrainer;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
 import org.apache.ignite.ml.structures.LabeledVector;
+import org.apache.ignite.ml.trainers.group.UpdateStrategies;
+import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -63,10 +67,27 @@ public class MLPGroupTrainerTest extends GridCommonAbstractTest {
         stopAllGrids();
     }
 
+    /**
+     * Test training 'xor' by RProp.
+     */
+    public void testXORRProp() {
+        doTestXOR(UpdateStrategies.RProp());
+    }
+
+    /**
+     * Test training 'xor' by SimpleGD.
+     */
+    public void testXORGD() {
+        doTestXOR(new UpdatesStrategy<>(
+            new SimpleGDUpdateCalculator().withLearningRate(0.5),
+            SimpleGDParameterUpdate::sumLocal,
+            SimpleGDParameterUpdate::avg));
+    }
+
     /**
      * Test training of 'xor' by {@link MLPGroupUpdateTrainer}.
      */
-    public void testXOR() {
+    private  void doTestXOR(UpdatesStrategy stgy) {
         int samplesCnt = 1000;
 
         Matrix xorInputs = new DenseLocalOnHeapMatrix(new double[][] {{0.0, 0.0}, {0.0, 1.0}, {1.0, 0.0}, {1.0, 1.0}},
@@ -93,18 +114,19 @@ public void testXOR() {
             }
         }
 
-        int totalCnt = 20;
+        int totalCnt = 30;
         int failCnt = 0;
         double maxFailRatio = 0.3;
-        MLPGroupUpdateTrainer trainer = MLPGroupUpdateTrainer.getDefault(ignite).
-            withSyncRate(3).
+
+        MLPGroupUpdateTrainer trainer = MLPGroupUpdateTrainer.getDefault(ignite).
+            withSyncPeriod(3).
             withTolerance(0.001).
-            withMaxGlobalSteps(1000);
+            withMaxGlobalSteps(100).
+            withUpdateStrategy(stgy);
 
         for (int i = 0; i < totalCnt; i++) {
-
             MLPGroupUpdateTrainerCacheInput trainerInput = new MLPGroupUpdateTrainerCacheInput(conf,
-                new RandomInitializer(new Random(123L)), 6, cache, 4, new Random(123L));
+                new RandomInitializer(new Random(123L + i)), 6, cache, 10, new Random(123L + i));
 
             MultilayerPerceptron mlp = trainer.train(trainerInput);
 
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPLocalTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPLocalTrainerTest.java
index b4c14e1d79560..31191707186c8 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPLocalTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPLocalTrainerTest.java
@@ -43,7 +43,7 @@ public class MLPLocalTrainerTest {
      */
     @Test
     public void testXORSimpleGD() {
-        xorTest(() -> new SimpleGDUpdateCalculator<>(0.3));
+        xorTest(() -> new SimpleGDUpdateCalculator(0.3));
     }
 
     /**
@@ -51,7 +51,7 @@ public void testXORSimpleGD() {
      */
     @Test
     public void testXORRProp() {
-        xorTest(() -> new RPropUpdateCalculator<>());
+        xorTest(RPropUpdateCalculator::new);
     }
 
     /**
@@ -67,7 +67,7 @@ public void testXORNesterov() {
      * @param updaterSupplier Updater supplier.
      * @param 
 Updater parameters type.
      */
-    private 
 void xorTest(IgniteSupplier> updaterSupplier) {
+    private 
 void xorTest(IgniteSupplier> updaterSupplier) {
         Matrix xorInputs = new DenseLocalOnHeapMatrix(new double[][] {{0.0, 0.0}, {0.0, 1.0}, {1.0, 0.0}, {1.0, 1.0}},
             StorageConstants.ROW_STORAGE_MODE).transpose();
 
@@ -79,7 +79,7 @@ private 
 void xorTest(IgniteSupplier(LossFunctions.MSE,
             updaterSupplier,
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistDistributed.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistDistributed.java
index 112aaded644e9..5656f688c8862 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistDistributed.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistDistributed.java
@@ -90,7 +90,9 @@ public void testMNISTDistributed() throws IOException {
         IgniteCache> labeledVectorsCache = LabeledVectorsCache.createNew(ignite);
         loadIntoCache(trainingMnistLst, labeledVectorsCache);
 
-        MLPGroupUpdateTrainer trainer = MLPGroupUpdateTrainer.getDefault(ignite).withMaxGlobalSteps(35).withSyncRate(2);
+        MLPGroupUpdateTrainer trainer = MLPGroupUpdateTrainer.getDefault(ignite).
+            withMaxGlobalSteps(35).
+            withSyncPeriod(2);
 
         MLPArchitecture arch = new MLPArchitecture(FEATURES_CNT).
             withAddedLayer(hiddenNeuronsCnt, true, Activators.SIGMOID).
@@ -105,6 +107,8 @@ public void testMNISTDistributed() throws IOException {
 
         Tracer.showAscii(truth);
         Tracer.showAscii(predicted);
+
+        X.println("Accuracy: " + VectorUtils.zipWith(predicted, truth, (x, y) -> x.equals(y) ? 1.0 : 0.0).sum() / truth.size() * 100 + "%.");
     }
 
     /**
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistLocal.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistLocal.java
index cda0413afeed8..14c02aa931de6 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistLocal.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistLocal.java
@@ -74,7 +74,7 @@ public void tstMNISTLocal() throws IOException {
             2000);
 
         MultilayerPerceptron mdl = new MLPLocalBatchTrainer<>(LossFunctions.MSE,
-            () -> new RPropUpdateCalculator<>(0.1, 1.2, 0.5),
+            () -> new RPropUpdateCalculator(0.1, 1.2, 0.5),
             1E-7,
             200).
             train(input);
@@ -89,5 +89,7 @@ public void tstMNISTLocal() throws IOException {
 
         Tracer.showAscii(truth);
         Tracer.showAscii(predicted);
+
+        X.println("Accuracy: " + VectorUtils.zipWith(predicted, truth, (x, y) -> x.equals(y) ? 1.0 : 0.0).sum() / truth.size() * 100 + "%.");
     }
 }

From ba072fdfaa821a7b846a5d8ff9f28834c7949bb0 Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn 
Date: Tue, 23 Jan 2018 14:24:37 +0300
Subject: [PATCH 34/65] IGNITE-7493 .NET: WAL management API added

---
 .../platform/PlatformProcessorImpl.java       | 27 +++++++++++++-
 .../Cache/PersistenceTest.cs                  | 37 +++++++++++++++++++
 .../Apache.Ignite.Core/Cluster/ICluster.cs    | 34 +++++++++++++++++
 .../dotnet/Apache.Ignite.Core/Impl/Ignite.cs  | 29 ++++++++++++++-
 4 files changed, 125 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java
index de51b3d6e32d6..87ec97cf74644 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformProcessorImpl.java
@@ -69,6 +69,9 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import static org.apache.ignite.internal.processors.platform.PlatformAbstractTarget.FALSE;
+import static org.apache.ignite.internal.processors.platform.PlatformAbstractTarget.TRUE;
+
 /**
  * GridGain platform processor.
  */
@@ -152,6 +155,15 @@ public class PlatformProcessorImpl extends GridProcessorAdapter implements Platf
     /** */
     private static final int OP_GET_BASELINE_TOPOLOGY = 26;
 
+    /** */
+    private static final int OP_DISABLE_WAL = 27;
+
+    /** */
+    private static final int OP_ENABLE_WAL = 28;
+
+    /** */
+    private static final int OP_IS_WAL_ENABLED = 29;
+
     /** Start latch. */
     private final CountDownLatch startLatch = new CountDownLatch(1);
 
@@ -412,7 +424,7 @@ private void loggerLog(int level, String message, String category, String errorI
     @Override public long processInLongOutLong(int type, long val) throws IgniteCheckedException {
         switch (type) {
             case OP_LOGGER_IS_LEVEL_ENABLED: {
-                return loggerIsLevelEnabled((int) val) ? PlatformAbstractTarget.TRUE : PlatformAbstractTarget.FALSE;
+                return loggerIsLevelEnabled((int) val) ? TRUE : FALSE;
             }
 
             case OP_RELEASE_START: {
@@ -468,6 +480,19 @@ private void loggerLog(int level, String message, String category, String errorI
                 ctx.grid().addCacheConfiguration(cfg);
 
                 return 0;
+
+            case OP_DISABLE_WAL:
+                ctx.grid().cluster().disableWal(reader.readString());
+
+                return 0;
+
+            case OP_ENABLE_WAL:
+                ctx.grid().cluster().enableWal(reader.readString());
+
+                return 0;
+
+            case OP_IS_WAL_ENABLED:
+                return ctx.grid().cluster().isWalEnabled(reader.readString()) ? TRUE : FALSE;
         }
 
         return PlatformAbstractTarget.throwUnsupported(type);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistenceTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistenceTest.cs
index e3c0acfb75ef9..1893a5a8a1bc2 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistenceTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistenceTest.cs
@@ -274,6 +274,43 @@ public void TestBaselineTopology()
             }
         }
 
+        /// 

+        /// Tests the wal disable/enable functionality.
+        /// 
+        [Test]
+        public void TestWalDisableEnable()
+        {
+            using (var ignite = Ignition.Start(GetPersistentConfiguration()))
+            {
+                var cluster = ignite.GetCluster();
+                cluster.SetActive(true);
+
+                var cache = ignite.CreateCache("foo");
+                Assert.IsTrue(cluster.IsWalEnabled(cache.Name));
+                cache[1] = 1;
+
+                cluster.DisableWal(cache.Name);
+                Assert.IsFalse(cluster.IsWalEnabled(cache.Name));
+                cache[2] = 2;
+
+                cluster.EnableWal(cache.Name);
+                Assert.IsTrue(cluster.IsWalEnabled(cache.Name));
+
+                Assert.AreEqual(1, cache[1]);
+                Assert.AreEqual(2, cache[2]);
+
+                // Check exceptions.
+                var ex = Assert.Throws(() => cluster.IsWalEnabled("bar"));
+                Assert.AreEqual("Cache not found: bar", ex.Message);
+
+                ex = Assert.Throws(() => cluster.DisableWal("bar"));
+                Assert.AreEqual("Cache doesn't exist: bar", ex.Message);
+
+                ex = Assert.Throws(() => cluster.EnableWal("bar"));
+                Assert.AreEqual("Cache doesn't exist: bar", ex.Message);
+            }
+        }
+
         /// 
         /// Checks active state.
         /// 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/ICluster.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/ICluster.cs
index f9bbdf796fa64..45e22e97f6cca 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/ICluster.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cluster/ICluster.cs
@@ -121,5 +121,39 @@ public interface ICluster : IClusterGroup
         /// Returns null if  has not been called.
         /// 
         ICollection GetBaselineTopology();
+
+        /// 
+        /// Disables write-ahead logging for specified cache. When WAL is disabled, changes are not logged to disk.
+        /// This significantly improves cache update speed.The drawback is absence of local crash-recovery guarantees.
+        /// If node is crashed, local content of WAL-disabled cache will be cleared on restart
+        /// to avoid data corruption.
+        /// 
+        /// Internally this method will wait for all current cache operations to finish and prevent new cache 
+        /// operations from being executed.Then checkpoint is initiated to flush all data to disk.Control is returned
+        /// to the callee when all dirty pages are prepared for checkpoint, but not necessarily flushed to disk.
+        /// 
+        /// WAL state can be changed only for persistent caches.
+        /// 
+        /// Name of the cache.
+        void DisableWal(string cacheName);
+
+        /// 
+        /// Enables write-ahead logging for specified cache. Restoring crash-recovery guarantees of a previous call to
+        /// .
+        /// 
+        /// Internally this method will wait for all current cache operations to finish and prevent new cache
+        /// operations from being executed. Then checkpoint is initiated to flush all data to disk.
+        /// Control is returned to the callee when all data is persisted to disk.
+        /// 
+        /// WAL state can be changed only for persistent caches.
+        /// 
+        /// Name of the cache.
+        void EnableWal(string cacheName);
+
+        /// 
+        /// Determines whether write-ahead logging is enabled for specified cache.
+        /// 
+        /// Name of the cache.
+        bool IsWalEnabled(string cacheName);
     }
 }
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs
index 10b083ba1aba9..ffab09f1dfb4f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs
@@ -87,7 +87,10 @@ private enum Op
             AddCacheConfiguration = 23,
             SetBaselineTopologyVersion = 24,
             SetBaselineTopologyNodes = 25,
-            GetBaselineTopology = 26
+            GetBaselineTopology = 26,
+            DisableWal = 27,
+            EnableWal = 28,
+            IsWalEnabled = 29
         }
 
         /** */
@@ -817,6 +820,30 @@ public ICollection GetBaselineTopology()
                 s => Marshaller.StartUnmarshal(s).ReadCollectionRaw(r => (IBaselineNode) new BaselineNode(r)));
         }
 
+        /**  */
+        public void DisableWal(string cacheName)
+        {
+            IgniteArgumentCheck.NotNull(cacheName, "cacheName");
+
+            DoOutOp((int) Op.DisableWal, w => w.WriteString(cacheName));
+        }
+
+        /**  */
+        public void EnableWal(string cacheName)
+        {
+            IgniteArgumentCheck.NotNull(cacheName, "cacheName");
+            
+            DoOutOp((int) Op.EnableWal, w => w.WriteString(cacheName));
+        }
+
+        /**  */
+        public bool IsWalEnabled(string cacheName)
+        {
+            IgniteArgumentCheck.NotNull(cacheName, "cacheName");
+
+            return DoOutOp((int) Op.IsWalEnabled, w => w.WriteString(cacheName)) == True;
+        }
+
         /**  */
 #pragma warning disable 618
         public IPersistentStoreMetrics GetPersistentStoreMetrics()

From e80fda8f0e7f73d33adac3e164d863278bba5293 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko 
Date: Tue, 23 Jan 2018 17:30:41 +0300
Subject: [PATCH 35/65] IGNITE-7500 Set clear flag on last supply message -
 Fixes #3421.

Signed-off-by: Alexey Goncharuk 
---
 .../preloader/GridDhtPartitionSupplier.java   |  10 +-
 ...CacheRebalancingPartitionCountersTest.java | 178 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite3.java     |   2 +
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 6eb31ed37a8c6..7194b24280c16 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -30,9 +30,9 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
 import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
-import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.util.lang.GridCloseableIterator;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.T3;
@@ -303,11 +303,8 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
                             iter = grp.offheap().rebalanceIterator(part, d.topologyVersion(),
                                 d.isHistorical(part) ? d.partitionCounter(part) : null);
 
-                            if (!iter.historical()) {
+                            if (!iter.historical())
                                 assert !grp.persistenceEnabled() || !d.isHistorical(part);
-
-                                s.clean(part);
-                            }
                             else
                                 assert grp.persistenceEnabled() && d.isHistorical(part);
                         }
@@ -416,6 +413,9 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage
                     // Mark as last supply message.
                     s.last(part, loc.updateCounter());
 
+                    if (!d.isHistorical(part))
+                        s.clean(part);
+
                     phase = SupplyContextPhase.NEW;
 
                     sctx = null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
new file mode 100644
index 0000000000000..0676c4503497c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class GridCacheRebalancingPartitionCountersTest extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_NAME = "cache";
+
+    /** */
+    private static final int PARTITIONS_CNT = 10;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        return super.getConfiguration(igniteInstanceName)
+                .setConsistentId(igniteInstanceName)
+                .setDataStorageConfiguration(
+                        new DataStorageConfiguration()
+                                .setCheckpointFrequency(3_000)
+                                .setDefaultDataRegionConfiguration(
+                                        new DataRegionConfiguration()
+                                                .setPersistenceEnabled(true)
+                                                .setMaxSize(100L * 1024 * 1024)))
+                .setCacheConfiguration(new CacheConfiguration(CACHE_NAME)
+                        .setBackups(2)
+                        .setRebalanceBatchSize(4096) // Force to create several supply messages during rebalancing.
+                        .setAffinity(
+                                new RendezvousAffinityFunction(false, PARTITIONS_CNT)));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+    }
+
+    /**
+     *
+     */
+    private boolean contains(int[] arr, int a) {
+        for (int i : arr)
+            if (i == a)
+                return true;
+
+        return false;
+    }
+
+    /**
+     * Tests that after rebalancing all partition update counters have the same value on all nodes.
+     */
+    public void test() throws Exception {
+        IgniteEx ignite = (IgniteEx)startGrids(3);
+
+        ignite.active(true);
+
+        IgniteCache cache = ignite.cache(CACHE_NAME);
+
+        for (int i = 0; i < 256; i++)
+            cache.put(i, i);
+
+        final int problemNode = 2;
+
+        IgniteEx node = (IgniteEx) ignite(problemNode);
+        int[] primaryPartitions = node.affinity(CACHE_NAME).primaryPartitions(node.cluster().localNode());
+
+        ignite.active(false);
+
+        boolean primaryRemoved = false;
+        for (int i = 0; i < PARTITIONS_CNT; i++) {
+            String nodeName = getTestIgniteInstanceName(problemNode);
+
+            Path dirPath = Paths.get(U.defaultWorkDirectory(), "db", nodeName.replace(".", "_"), CACHE_NAME + "-" + CACHE_NAME);
+
+            info("Path: " + dirPath.toString());
+
+            assertTrue(Files.exists(dirPath));
+
+            for (File f : dirPath.toFile().listFiles()) {
+                if (f.getName().equals("part-" + i + ".bin")) {
+                    if (contains(primaryPartitions, i)) {
+                        info("Removing: " + f.getName());
+
+                        primaryRemoved = true;
+
+                        f.delete();
+                    }
+                }
+                else if (f.getName().equals("index.bin")) {
+                    info("Removing: " + f.getName());
+
+                    f.delete();
+                }
+            }
+        }
+
+        assertTrue(primaryRemoved);
+
+        ignite.active(true);
+        waitForRebalancing();
+
+        List issues = new ArrayList<>();
+        HashMap partMap = new HashMap<>();
+
+        for (int i = 0; i < 3; i++)
+            checkUpdCounter((IgniteEx)ignite(i), issues, partMap);
+
+        for (String issue : issues)
+            error(issue);
+
+        assertTrue(issues.isEmpty());
+    }
+
+    /**
+     *
+     */
+    private void checkUpdCounter(IgniteEx ignite, List issues, HashMap partMap) {
+        final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(CU.cacheId(CACHE_NAME));
+
+        assertNotNull(grpCtx);
+
+        GridDhtPartitionTopologyImpl top = (GridDhtPartitionTopologyImpl)grpCtx.topology();
+
+        List locParts = top.localPartitions();
+
+        for (GridDhtLocalPartition part : locParts) {
+            Long cnt = partMap.get(part.id());
+
+            if (cnt == null)
+                partMap.put(part.id(), part.updateCounter());
+
+            if ((cnt != null && part.updateCounter() != cnt) || part.updateCounter() == 0)
+                issues.add("Node name " + ignite.name() + "Part = " + part.id() + " updCounter " + part.updateCounter());
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index a6be07ea07ee0..674b6a290ee8d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -52,6 +52,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
@@ -146,6 +147,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
+        suite.addTestSuite(GridCacheRebalancingPartitionCountersTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);

From d10f920abf3df3d176f1a9445afa18b7a6bb601f Mon Sep 17 00:00:00 2001
From: Ivanov Petr 
Date: Tue, 23 Jan 2018 16:23:27 +0300
Subject: [PATCH 36/65] IGNITE-6730 Java 9: fix project build procedure

Signed-off-by: Anton Vinogradov 

(cherry picked from commit 5216ac5)
Signed-off-by: Anton Vinogradov 
---
 bin/control.bat                               |  5 ++
 bin/control.sh                                | 12 +++++
 bin/ignite.bat                                | 11 ++--
 bin/ignite.sh                                 | 12 +++++
 bin/ignitevisorcmd.bat                        |  5 ++
 bin/ignitevisorcmd.sh                         | 12 +++++
 bin/include/functions.sh                      | 13 ++---
 modules/benchmarks/pom.xml                    |  2 +-
 modules/cassandra/serializers/pom.xml         |  2 +-
 modules/cassandra/store/pom.xml               |  2 +-
 .../apache/ignite/internal/IgniteKernal.java  | 11 ++--
 modules/hadoop/pom.xml                        |  2 +-
 modules/scalar/pom.xml                        |  2 +-
 modules/spark/pom.xml                         |  4 +-
 modules/tools/pom.xml                         | 22 --------
 modules/visor-console/pom.xml                 |  2 +-
 parent/pom.xml                                | 51 +++++++++++++++----
 17 files changed, 116 insertions(+), 54 deletions(-)

diff --git a/bin/control.bat b/bin/control.bat
index bad4c4e52d1f8..54de5a249d751 100644
--- a/bin/control.bat
+++ b/bin/control.bat
@@ -198,6 +198,11 @@ if "%MAIN_CLASS%" == "" set MAIN_CLASS=org.apache.ignite.internal.commandline.Co
 :: set JVM_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8787 %JVM_OPTS%
 ::
 
+::
+:: Final JVM_OPTS for Java 9 compatibility
+::
+"%JAVA_HOME%\bin\java.exe" -version 2>&1 | findstr /R /c:"java version .9\..*" > nul && set JVM_OPTS=--add-exports java.base/jdk.internal.misc=ALL-UNNAMED --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-exports java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED --add-exports jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED  --add-modules java.xml.bind %JVM_OPTS%
+
 if "%INTERACTIVE%" == "1" (
     "%JAVA_HOME%\bin\java.exe" %JVM_OPTS% %QUIET% %RESTART_SUCCESS_OPT% %JMX_MON% ^
     -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^
diff --git a/bin/control.sh b/bin/control.sh
index 8b08d1622da1a..d8c1dd9a08b7b 100755
--- a/bin/control.sh
+++ b/bin/control.sh
@@ -135,6 +135,18 @@ fi
 #
 # JVM_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8787 ${JVM_OPTS}"
 
+#
+# Final JVM_OPTS for Java 9 compatibility
+#
+${JAVA_HOME}/bin/java -version 2>&1 | grep -qE 'java version "9.*"' && {
+JVM_OPTS="--add-exports java.base/jdk.internal.misc=ALL-UNNAMED \
+          --add-exports java.base/sun.nio.ch=ALL-UNNAMED \
+          --add-exports java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED \
+          --add-exports jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED \
+          --add-modules java.xml.bind \
+      ${JVM_OPTS}"
+} || true
+
 ERRORCODE="-1"
 
 while [ "${ERRORCODE}" -ne "130" ]
diff --git a/bin/ignite.bat b/bin/ignite.bat
index f697dc08c8a6d..577e0cfed5b89 100644
--- a/bin/ignite.bat
+++ b/bin/ignite.bat
@@ -37,16 +37,16 @@ goto error_finish
 if exist "%JAVA_HOME%\bin\java.exe" goto checkJdkVersion
     echo %0, ERROR:
     echo JAVA is not found in JAVA_HOME=%JAVA_HOME%.
-    echo Please point JAVA_HOME variable to installation of JDK 1.7 or JDK 1.8.
+    echo Please point JAVA_HOME variable to installation of JDK 1.8 or JDK 9.
     echo You can also download latest JDK at http://java.com/download.
 goto error_finish
 
 :checkJdkVersion
-"%JAVA_HOME%\bin\java.exe" -version 2>&1 | findstr "1\.[78]\." > nul
+"%JAVA_HOME%\bin\java.exe" -version 2>&1 | findstr /R /c:"java version .9\..*" /c:"java version .1\.8\..*" > nul
 if %ERRORLEVEL% equ 0 goto checkIgniteHome1
     echo %0, ERROR:
     echo The version of JAVA installed in %JAVA_HOME% is incorrect.
-    echo Please point JAVA_HOME variable to installation of JDK 1.7 or JDK 1.8.
+    echo Please point JAVA_HOME variable to installation of JDK 1.8 or JDK 9.
     echo You can also download latest JDK at http://java.com/download.
 goto error_finish
 
@@ -213,6 +213,11 @@ if "%MAIN_CLASS%" == "" set MAIN_CLASS=org.apache.ignite.startup.cmdline.Command
 :: set JVM_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8787 %JVM_OPTS%
 ::
 
+::
+:: Final JVM_OPTS for Java 9 compatibility
+::
+"%JAVA_HOME%\bin\java.exe" -version 2>&1 | findstr /R /c:"java version .9\..*" > nul && set JVM_OPTS=--add-exports java.base/jdk.internal.misc=ALL-UNNAMED --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-exports java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED --add-exports jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED  --add-modules java.xml.bind %JVM_OPTS%
+
 if "%INTERACTIVE%" == "1" (
     "%JAVA_HOME%\bin\java.exe" %JVM_OPTS% %QUIET% %RESTART_SUCCESS_OPT% %JMX_MON% ^
     -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_HOME="%IGNITE_HOME%" -DIGNITE_PROG_NAME="%PROG_NAME%" %JVM_XOPTS% ^
diff --git a/bin/ignite.sh b/bin/ignite.sh
index efa37159cfc4e..a12ead692d8c5 100755
--- a/bin/ignite.sh
+++ b/bin/ignite.sh
@@ -149,6 +149,18 @@ fi
 #
 # JVM_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8787 ${JVM_OPTS}"
 
+#
+# Final JVM_OPTS for Java 9 compatibility
+#
+${JAVA_HOME}/bin/java -version 2>&1 | grep -qE 'java version "9.*"' && {
+JVM_OPTS="--add-exports java.base/jdk.internal.misc=ALL-UNNAMED \
+          --add-exports java.base/sun.nio.ch=ALL-UNNAMED \
+          --add-exports java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED \
+          --add-exports jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED \
+          --add-modules java.xml.bind \
+      ${JVM_OPTS}"
+} || true
+
 ERRORCODE="-1"
 
 while [ "${ERRORCODE}" -ne "130" ]
diff --git a/bin/ignitevisorcmd.bat b/bin/ignitevisorcmd.bat
index 236f89620258e..94b8a697d4ca9 100644
--- a/bin/ignitevisorcmd.bat
+++ b/bin/ignitevisorcmd.bat
@@ -141,6 +141,11 @@ if %ENABLE_ASSERTIONS% == 1 set JVM_OPTS_VISOR=%JVM_OPTS_VISOR% -ea
 ::
 if "%ARGS%" == "" set ARGS=%*
 
+::
+:: Final JVM_OPTS for Java 9 compatibility
+::
+"%JAVA_HOME%\bin\java.exe" -version 2>&1 | findstr /R /c:"java version .9\..*" > nul && set JVM_OPTS=--add-exports java.base/jdk.internal.misc=ALL-UNNAMED --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-exports java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED --add-exports jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED  --add-modules java.xml.bind %JVM_OPTS%
+
 ::
 :: Starts Visor console.
 ::
diff --git a/bin/ignitevisorcmd.sh b/bin/ignitevisorcmd.sh
index fe74f6a45f12d..04715505d80c8 100755
--- a/bin/ignitevisorcmd.sh
+++ b/bin/ignitevisorcmd.sh
@@ -105,6 +105,18 @@ function restoreSttySettings() {
 #
 trap restoreSttySettings INT
 
+#
+# Final JVM_OPTS for Java 9 compatibility
+#
+${JAVA_HOME}/bin/java -version 2>&1 | grep -qE 'java version "9.*"' && {
+JVM_OPTS="--add-exports java.base/jdk.internal.misc=ALL-UNNAMED \
+          --add-exports java.base/sun.nio.ch=ALL-UNNAMED \
+          --add-exports java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED \
+          --add-exports jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED \
+          --add-modules java.xml.bind \
+      ${JVM_OPTS}"
+} || true
+
 #
 # Start Visor console.
 #
diff --git a/bin/include/functions.sh b/bin/include/functions.sh
index b18b15016238f..8f4570bd420d9 100755
--- a/bin/include/functions.sh
+++ b/bin/include/functions.sh
@@ -57,22 +57,19 @@ checkJava() {
     if [ ! -e "$JAVA" ]; then
         echo $0", ERROR:"
         echo "JAVA is not found in JAVA_HOME=$JAVA_HOME."
-        echo "Please point JAVA_HOME variable to installation of JDK 1.7 or JDK 1.8."
+        echo "Please point JAVA_HOME variable to installation of JDK 1.8 or JDK 9"
         echo "You can also download latest JDK at http://java.com/download"
 
         exit 1
     fi
 
-    JAVA_VER=`"$JAVA" -version 2>&1 | egrep "1\.[78]\."`
-
-    if [ "$JAVA_VER" == "" ]; then
-        echo $0", ERROR:"
+    "$JAVA" -version 2>&1 | grep -qE 'java version "(1.8.*|9.*)"' || {
+        echo "$0, ERROR:"
         echo "The version of JAVA installed in JAVA_HOME=$JAVA_HOME is incorrect."
-        echo "Please point JAVA_HOME variable to installation of JDK 1.7 or JDK 1.8."
+        echo "Please point JAVA_HOME variable to installation of JDK 1.8 or JDK 9."
         echo "You can also download latest JDK at http://java.com/download"
-
         exit 1
-    fi
+    }
 }
 
 #
diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml
index 1c71b484556d9..0caec7c37a153 100644
--- a/modules/benchmarks/pom.xml
+++ b/modules/benchmarks/pom.xml
@@ -126,7 +126,7 @@
                 
                 
                     maven-surefire-plugin
-                    2.17
+                    2.20.1
                 
             
         
diff --git a/modules/cassandra/serializers/pom.xml b/modules/cassandra/serializers/pom.xml
index 6d040e3b4fde4..ad32d4509aebd 100644
--- a/modules/cassandra/serializers/pom.xml
+++ b/modules/cassandra/serializers/pom.xml
@@ -85,7 +85,7 @@
             
                 org.apache.maven.plugins
                 maven-compiler-plugin
-                3.2
+                3.7.0
                 
                     UTF-8
                     true
diff --git a/modules/cassandra/store/pom.xml b/modules/cassandra/store/pom.xml
index 0729e08683958..6dae645d11aa9 100644
--- a/modules/cassandra/store/pom.xml
+++ b/modules/cassandra/store/pom.xml
@@ -147,7 +147,7 @@
             
                 org.apache.maven.plugins
                 maven-compiler-plugin
-                3.2
+                3.7.0
                 
                     UTF-8
                     true
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 3bd2a15bad73d..16698b5f4a977 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2461,9 +2461,14 @@ private void ackClassPaths(RuntimeMXBean rtBean) {
 
         // Ack all class paths.
         if (log.isDebugEnabled()) {
-            log.debug("Boot class path: " + rtBean.getBootClassPath());
-            log.debug("Class path: " + rtBean.getClassPath());
-            log.debug("Library path: " + rtBean.getLibraryPath());
+            try {
+                log.debug("Boot class path: " + rtBean.getBootClassPath());
+                log.debug("Class path: " + rtBean.getClassPath());
+                log.debug("Library path: " + rtBean.getLibraryPath());
+            }
+            catch (Exception ignore) {
+                // No-op: ignore for Java 9+ and non-standard JVMs.
+            }
         }
     }
 
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index fa0444fba4b54..6ae792d26c2b2 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -205,7 +205,7 @@
             
                 org.apache.maven.plugins
                 maven-surefire-plugin
-                2.17
+                2.20.1
                 
                 
             
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index 5bf56eed3705a..b0029d1e19423 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -44,7 +44,7 @@
         
             org.scala-lang
             scala-library
-            ${scala211.library.version}
+            ${scala.library.version}
         
 
         
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index 458abd86849a5..3f4d96d0c6b71 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -52,13 +52,13 @@
         
             org.scala-lang
             scala-library
-            ${scala211.library.version}
+            ${scala.library.version}
         
 
         
             org.scala-lang
             scala-reflect
-            ${scala211.library.version}
+            ${scala.library.version}
         
 
         
diff --git a/modules/tools/pom.xml b/modules/tools/pom.xml
index 653d470d98316..b4f07694b3e9f 100644
--- a/modules/tools/pom.xml
+++ b/modules/tools/pom.xml
@@ -47,26 +47,4 @@
             1.8.2
         
     
-
-    
-    
-        
-            default-tools.jar
-            
-                
-                    java.vendor
-                    Oracle Corporation
-                
-            
-            
-                
-                    com.sun
-                    tools
-                    ${java.version}
-                    system
-                    ${java.home}/../lib/tools.jar
-                
-            
-        
-    
 
diff --git a/modules/visor-console/pom.xml b/modules/visor-console/pom.xml
index 8c4de098354b1..5e3b12be95f90 100644
--- a/modules/visor-console/pom.xml
+++ b/modules/visor-console/pom.xml
@@ -80,7 +80,7 @@
         
             org.scala-lang
             scala-library
-            ${scala211.library.version}
+            ${scala.library.version}
         
 
         
diff --git a/parent/pom.xml b/parent/pom.xml
index 26339af354791..a958b5af658ea 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -93,7 +93,8 @@
         4.0.2
         5.5.2_1
         5.5.2
-        2.5.4
+        3.5.0
+        1.10.19
         5.1.39
         1.1.2
         2.0.8_6
@@ -104,7 +105,7 @@
         4.2.0
         2.10.4
         2.10.6
-        2.11.8
+        2.11.8
         1.7.7
         1.6.4
         2.6.5
@@ -579,7 +580,7 @@
             
                 org.apache.maven.plugins
                 maven-surefire-plugin
-                2.17
+                2.20.1
                 
                     0
                 
@@ -950,7 +951,7 @@
                     
                         org.apache.maven.plugins
                         maven-surefire-plugin
-                        2.17
+                        2.20.1
                         
                             1
                         
@@ -960,20 +961,52 @@
         
 
         
-            tools.jar-default
+            java-9
+            
+                [1.9,)
+            
+            
+                9
+                9
+                2.12.4
+            
+            
+            
+                
+                    
+                        org.apache.maven.plugins
+                        maven-compiler-plugin
+                        3.7.0
+                        
+                            
+                                --add-exports
+                                java.base/jdk.internal.misc=ALL-UNNAMED
+                                --add-exports
+                                java.base/sun.nio.ch=ALL-UNNAMED
+                                --add-exports
+                                java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED
+                                --add-exports
+                                jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED
+                            
+                        
+                    
+                
+            
+        
 
+        
+            tools.jar-default
             
                 
                     ${java.home}/../lib/tools.jar
                 
             
-
             
                 
                     com.sun
                     tools
-                    1.4.2
                     system
+                    ${java.version}
                     ${java.home}/../lib/tools.jar
                 
             
@@ -981,19 +1014,17 @@
 
         
             tools.jar-mac
-
             
                 
                     ${java.home}/../Classes/classes.jar
                 
             
-
             
                 
                     com.sun
                     tools
-                    1.4.2
                     system
+                    ${java.version}
                     ${java.home}/../Classes/classes.jar
                 
             

From bc23cbd263cdc34eaecb6bc02ee33f21cb676329 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin 
Date: Tue, 23 Jan 2018 18:30:04 +0300
Subject: [PATCH 37/65] IGNITE-7278 Fixed partition state recovery from WAL

---
 .../GridCacheDatabaseSharedManager.java       |  26 +-
 .../IgnitePdsContinuousRestartTest2.java      | 291 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   2 +
 3 files changed, 305 insertions(+), 14 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest2.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 85c40055d92ee..205365b12c845 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1916,26 +1916,24 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
                         break;
 
                     case PARTITION_DESTROY:
-                        if (apply) {
-                            PartitionDestroyRecord destroyRec = (PartitionDestroyRecord)rec;
+                        PartitionDestroyRecord destroyRec = (PartitionDestroyRecord)rec;
 
-                            final int gId = destroyRec.groupId();
+                        final int gId = destroyRec.groupId();
 
-                            if (storeOnly && gId != METASTORAGE_CACHE_ID)
-                                continue;
+                        if (storeOnly && gId != METASTORAGE_CACHE_ID)
+                            continue;
 
-                            if (!ignoreGrps.contains(gId)) {
-                                final int pId = destroyRec.partitionId();
+                        if (!ignoreGrps.contains(gId)) {
+                            final int pId = destroyRec.partitionId();
 
-                                PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
+                            PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
 
-                                pageMem.clearAsync(new P3() {
-                                    @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
-                                        return cacheId == gId && PageIdUtils.partId(pageId) == pId;
-                                    }
-                                }, true).get();
+                            pageMem.clearAsync(new P3() {
+                                @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
+                                    return cacheId == gId && PageIdUtils.partId(pageId) == pId;
+                                }
+                            }, true).get();
 
-                            }
                         }
 
                         break;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest2.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest2.java
new file mode 100644
index 0000000000000..f45fc505860e8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest2.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ * Cause by https://issues.apache.org/jira/browse/IGNITE-7278
+ */
+public class IgnitePdsContinuousRestartTest2 extends GridCommonAbstractTest {
+    /** */
+    private static final int GRID_CNT = 4;
+
+    /** */
+    private static final int ENTRIES_COUNT = 10_000;
+
+    /** */
+    public static final String CACHE_NAME = "cache1";
+
+    /** Checkpoint delay. */
+    private volatile int checkpointDelay = -1;
+
+    /** */
+    private boolean cancel;
+
+    /**
+     * Default constructor.
+     */
+    public IgnitePdsContinuousRestartTest2() {
+
+    }
+
+    /**
+     * @param cancel Cancel.
+     */
+    public IgnitePdsContinuousRestartTest2(boolean cancel) {
+        this.cancel = cancel;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(400 * 1024 * 1024)
+                    .setPersistenceEnabled(true))
+            .setWalMode(WALMode.LOG_ONLY)
+            .setCheckpointFrequency(checkpointDelay);
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        CacheConfiguration ccfg = new CacheConfiguration();
+
+        ccfg.setName(CACHE_NAME);
+        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 128));
+        ccfg.setBackups(2);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        stopAllGrids();
+
+        deleteWorkFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        deleteWorkFiles();
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    private void deleteWorkFiles() throws IgniteCheckedException {
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_1000_500_1_1() throws Exception {
+        checkRebalancingDuringLoad(1000, 500, 1, 1);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_8000_500_1_1() throws Exception {
+        checkRebalancingDuringLoad(8000, 500, 1, 1);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_1000_20000_1_1() throws Exception {
+        checkRebalancingDuringLoad(1000, 20000, 1, 1);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_8000_8000_1_1() throws Exception {
+        checkRebalancingDuringLoad(8000, 8000, 1, 1);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_1000_500_8_1() throws Exception {
+        checkRebalancingDuringLoad(1000, 500, 8, 1);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_8000_500_8_1() throws Exception {
+        checkRebalancingDuringLoad(8000, 500, 8, 1);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_1000_20000_8_1() throws Exception {
+        checkRebalancingDuringLoad(1000, 20000, 8, 1);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_8000_8000_8_1() throws Exception {
+        checkRebalancingDuringLoad(8000, 8000, 8, 1);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_1000_500_8_16() throws Exception {
+        checkRebalancingDuringLoad(1000, 500, 8, 16);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_8000_500_8_16() throws Exception {
+        checkRebalancingDuringLoad(8000, 500, 8, 16);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_1000_20000_8_16() throws Exception {
+        checkRebalancingDuringLoad(1000, 20000, 8, 16);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testRebalancingDuringLoad_8000_8000_8_16() throws Exception {
+        checkRebalancingDuringLoad(8000, 8000, 8, 16);
+    }
+
+    /**
+     *
+     * @throws Exception if failed.
+     */
+    public void testRebalncingDuringLoad_10_10_1_1() throws Exception {
+        checkRebalancingDuringLoad(10, 10, 1, 1);
+    }
+
+    /**
+     *
+     * @throws Exception if failed.
+     */
+    public void testRebalncingDuringLoad_10_500_8_16() throws Exception {
+        checkRebalancingDuringLoad(10, 500, 8, 16);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    private void checkRebalancingDuringLoad(
+        int restartDelay,
+        int checkpointDelay,
+        int threads,
+        final int batch
+    ) throws Exception {
+        this.checkpointDelay = checkpointDelay;
+
+        startGrids(GRID_CNT);
+
+        final Ignite load = ignite(0);
+
+        load.cluster().active(true);
+
+        try (IgniteDataStreamer s = load.dataStreamer(CACHE_NAME)) {
+            s.allowOverwrite(true);
+
+            for (int i = 0; i < ENTRIES_COUNT; i++)
+                s.addData(i, i);
+        }
+
+        final AtomicBoolean done = new AtomicBoolean(false);
+
+        IgniteInternalFuture busyFut = GridTestUtils.runMultiThreadedAsync(new Callable() {
+            /** {@inheritDoc} */
+            @Override public Object call() throws Exception {
+                IgniteCache cache = load.cache(CACHE_NAME);
+                Random rnd = ThreadLocalRandom.current();
+
+                while (!done.get()) {
+                    Map map = new TreeMap<>();
+
+                    for (int i = 0; i < batch; i++)
+                        map.put(rnd.nextInt(ENTRIES_COUNT), rnd.nextInt());
+
+                    cache.putAll(map);
+                }
+
+                return null;
+            }
+        }, threads, "updater");
+
+        long end = System.currentTimeMillis() + 90_000;
+
+        Random rnd = ThreadLocalRandom.current();
+
+        while (System.currentTimeMillis() < end) {
+            int idx = rnd.nextInt(GRID_CNT - 1) + 1;
+
+            stopGrid(idx, cancel);
+
+            U.sleep(restartDelay);
+
+            startGrid(idx);
+
+            U.sleep(restartDelay);
+        }
+
+        done.set(true);
+
+        busyFut.get();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 3852a16283f4f..a3dc5a1f299ad 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteDataStorageMetricsSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest2;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchangeDuringCheckpointTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
@@ -92,6 +93,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgniteWalHistoryReservationsTest.class);
 
         suite.addTestSuite(IgnitePdsContinuousRestartTest.class);
+        suite.addTestSuite(IgnitePdsContinuousRestartTest2.class);
 
         suite.addTestSuite(IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes.class);
 

From 93c7b42f53005ca24b4910101583ba96d5afcace Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Wed, 24 Jan 2018 10:27:16 +0300
Subject: [PATCH 38/65] IGNITE-7510 Muting flaky test

---
 .../java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 383698f99d9f5..a7c549d12b40d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -93,7 +93,8 @@ public static void addRealPageStoreTests(TestSuite suite) {
         // Persistence-enabled.
         suite.addTestSuite(IgnitePdsSingleNodePutGetPersistenceTest.class);
         suite.addTestSuite(IgnitePdsDynamicCacheTest.class);
-        suite.addTestSuite(IgnitePdsClientNearCachePutGetTest.class);
+        // TODO uncomment when https://issues.apache.org/jira/browse/IGNITE-7510 is fixed
+        // suite.addTestSuite(IgnitePdsClientNearCachePutGetTest.class);
         suite.addTestSuite(IgniteDbPutGetWithCacheStoreTest.class);
 
         suite.addTestSuite(IgniteClusterActivateDeactivateTestWithPersistence.class);

From 74f421424097450fd6fe0d3accc425b2d1215463 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Wed, 24 Jan 2018 10:28:20 +0300
Subject: [PATCH 39/65] IGNITE-7505 Node not in baseline should not initialize
 local partition

---
 .../managers/discovery/DiscoCache.java        |  8 +++++++
 .../dht/GridDhtPartitionTopologyImpl.java     | 22 ++++++++++++++-----
 .../GridDhtPartitionsExchangeFuture.java      |  1 +
 .../GridCacheDatabaseSharedManager.java       |  7 ++++--
 .../IgniteCacheDatabaseSharedManager.java     |  7 ++++++
 .../IgniteClusterActivateDeactivateTest.java  |  4 ++--
 6 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index aa471680c0396..f73f4205570f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -213,6 +213,14 @@ public List remoteNodes() {
         return baselineNodes;
     }
 
+    /**
+     * @param nodeId Node ID to check.
+     * @return {@code True} if baseline is not set or the node is in the baseline topology.
+     */
+    public boolean baselineNode(UUID nodeId) {
+        return nodeIdToConsIdx == null || nodeIdToConsIdx.containsKey(nodeId);
+    }
+
     /** @return All nodes. */
     public List allNodes() {
         return allNodes;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 9ccaecf9144df..0a2c1541bd70d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -359,12 +359,7 @@ private void initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsEx
                     assert exchId.isJoined() || added;
 
                     for (int p = 0; p < num; p++) {
-                        IgnitePageStoreManager storeMgr = ctx.pageStore();
-
-                        if (localNode(p, aff)
-                            || (storeMgr instanceof FilePageStoreManager
-                            && grp.persistenceEnabled()
-                            && Files.exists(((FilePageStoreManager)storeMgr).getPath(grp.sharedGroup(), grp.cacheOrGroupName(), p)))) {
+                        if (localNode(p, aff) || initLocalPartition(p, discoCache)) {
                             GridDhtLocalPartition locPart = createPartition(p);
 
                             if (grp.persistenceEnabled()) {
@@ -430,6 +425,21 @@ else if (belongs) {
         updateRebalanceVersion(aff);
     }
 
+    /**
+     * @param p Partition ID to restore.
+     * @param discoCache Disco cache to use.
+     * @return {@code True} if should restore local partition.
+     */
+    private boolean initLocalPartition(int p, DiscoCache discoCache) {
+        IgnitePageStoreManager storeMgr = ctx.pageStore();
+
+        return
+            grp.persistenceEnabled() &&
+            storeMgr instanceof FilePageStoreManager &&
+            discoCache.baselineNode(ctx.localNodeId()) &&
+            Files.exists(((FilePageStoreManager)storeMgr).getPath(grp.sharedGroup(), grp.cacheOrGroupName(), p));
+    }
+
     /**
      * @param affVer Affinity version.
      * @param aff Affinity assignments.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 883cac69f88f3..6c09b6a4373df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1577,6 +1577,7 @@ public void finishMerged() {
         }
 
         cctx.database().releaseHistoryForExchange();
+        cctx.database().rebuildIndexesIfNeeded(this);
 
         if (err == null) {
             for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 205365b12c845..a636464a9f3c0 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1126,7 +1126,10 @@ private void shutdownCheckpointer(boolean cancel) {
         // Before local node join event.
         if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode))
             restoreState();
+    }
 
+    /** {@inheritDoc} */
+    @Override public void rebuildIndexesIfNeeded(GridDhtPartitionsExchangeFuture fut) {
         if (cctx.kernalContext().query().moduleEnabled()) {
             for (final GridCacheContext cacheCtx : (Collection)cctx.cacheContexts()) {
                 if (cacheCtx.startTopologyVersion().equals(fut.initialVersion()) &&
@@ -1145,8 +1148,8 @@ private void shutdownCheckpointer(boolean cancel) {
                             CacheConfiguration ccfg = cacheCtx.config();
 
                             if (ccfg != null) {
-                                log().info("Finished indexes rebuilding for cache: [name=" + ccfg.getName()
-                                    + ", grpName=" + ccfg.getGroupName());
+                                log().info("Finished indexes rebuilding for cache [name=" + ccfg.getName()
+                                    + ", grpName=" + ccfg.getGroupName() + ']');
                             }
                         }
                     });
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 237dacc31dfc5..3242e74208b99 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -752,6 +752,13 @@ public void beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws Igni
         // No-op.
     }
 
+    /**
+     * @param fut Partition exchange future.
+     */
+    public void rebuildIndexesIfNeeded(GridDhtPartitionsExchangeFuture fut) {
+        // No-op.
+    }
+
     /**
      * Needed action before any cache will stop
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 27739a27a2308..71718c9d16cc4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -124,11 +124,11 @@ else if (testDiscoSpi)
         DataStorageConfiguration memCfg = new DataStorageConfiguration();
         memCfg.setPageSize(4 * 1024);
         memCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
-            .setMaxSize(10 * 1024 * 1024)
+            .setMaxSize(300 * 1024 * 1024)
             .setPersistenceEnabled(persistenceEnabled()));
 
         memCfg.setDataRegionConfigurations(new DataRegionConfiguration()
-            .setMaxSize(10 * 1024 * 1024)
+            .setMaxSize(300 * 1024 * 1024)
             .setName(NO_PERSISTENCE_REGION)
             .setPersistenceEnabled(false));
 

From e0444b275c5cb831b427035fb515051a367deea2 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov 
Date: Wed, 24 Jan 2018 18:55:44 +0700
Subject: [PATCH 40/65] IGNITE-7492 Visor CMD: Added output of data region
 metrics to "node" command.

(cherry picked from commit d6585fa)
---
 .../org/apache/ignite/DataRegionMetrics.java  |   2 +-
 .../visor/cache/VisorMemoryMetrics.java       |  74 +++++++++++
 .../ignite/visor/commands/VisorConsole.scala  |  20 ++-
 .../commands/common/VisorTextTable.scala      |  22 ++--
 .../commands/node/VisorNodeCommand.scala      | 123 ++++++++++++------
 5 files changed, 181 insertions(+), 60 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
index f5ae96b153377..dc48f115c4ec4 100644
--- a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
@@ -162,7 +162,7 @@ public interface DataRegionMetrics {
     /**
      * Gets memory page size.
      *
-     * @return page size in bytes.
+     * @return Page size in bytes.
      */
     public int getPageSize();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java
index 37fed5abe3c1c..c19fd3607afd8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java
@@ -59,6 +59,21 @@ public class VisorMemoryMetrics extends VisorDataTransferObject {
     /** */
     private long physicalMemoryPages;
 
+    /** */
+    private long totalAllocatedSz;
+
+    /** */
+    private long physicalMemSz;
+
+    /** */
+    private long cpBufPages;
+
+    /** */
+    private long cpBufSz;
+
+    /** */
+    private int pageSize;
+
     /**
      * Default constructor.
      */
@@ -79,6 +94,11 @@ public VisorMemoryMetrics(DataRegionMetrics m) {
         dirtyPages = m.getDirtyPages();
         pagesReplaceRate = m.getPagesReplaceRate();
         physicalMemoryPages = m.getPhysicalMemoryPages();
+        totalAllocatedSz = m.getTotalAllocatedSize();
+        physicalMemSz = m.getPhysicalMemorySize();
+        cpBufPages = m.getCheckpointBufferPages();
+        cpBufSz = m.getCheckpointBufferSize();
+        pageSize = m.getPageSize();
     }
 
     /**
@@ -144,6 +164,47 @@ public long getPhysicalMemoryPages() {
         return physicalMemoryPages;
     }
 
+
+    /**
+     * @return Total size of memory allocated, in bytes.
+     */
+    public long getTotalAllocatedSize() {
+        return totalAllocatedSz;
+    }
+
+    /**
+     * @return Total size of pages loaded to RAM in bytes.
+     */
+    public long getPhysicalMemorySize() {
+        return physicalMemSz;
+    }
+
+    /**
+     * @return Checkpoint buffer size in pages.
+     */
+    public long getCheckpointBufferPages() {
+        return cpBufPages;
+    }
+
+    /**
+     * @return @return Checkpoint buffer size in bytes.
+     */
+    public long getCheckpointBufferSize() {
+        return cpBufSz;
+    }
+
+    /**
+     * @return Page size in bytes.
+     */
+    public int getPageSize() {
+        return pageSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeString(out, name);
@@ -155,6 +216,11 @@ public long getPhysicalMemoryPages() {
         out.writeLong(dirtyPages);
         out.writeFloat(pagesReplaceRate);
         out.writeLong(physicalMemoryPages);
+        out.writeLong(totalAllocatedSz);
+        out.writeLong(physicalMemSz);
+        out.writeLong(cpBufPages);
+        out.writeLong(cpBufSz);
+        out.writeInt(pageSize);
     }
 
     /** {@inheritDoc} */
@@ -168,6 +234,14 @@ public long getPhysicalMemoryPages() {
         dirtyPages = in.readLong();
         pagesReplaceRate = in.readFloat();
         physicalMemoryPages = in.readLong();
+
+        if (protoVer > V1) {
+            totalAllocatedSz = in.readLong();
+            physicalMemSz = in.readLong();
+            cpBufPages = in.readLong();
+            cpBufSz = in.readLong();
+            pageSize = in.readInt();
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
index 8bf64b7b7dfe8..43e11db999c06 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/VisorConsole.scala
@@ -170,13 +170,21 @@ class VisorConsole {
         }
 
         // Workaround for IDEA terminal.
-        val term = try {
-            Class.forName("com.intellij.rt.execution.application.AppMain")
+        val idea = Seq(
+            "com.intellij.rt.execution.application.AppMain",
+            "com.intellij.rt.execution.application.AppMainV2"
+        ).exists(cls =>
+            try {
+                Class.forName(cls)
+
+                true
+            }
+            catch {
+                case _: ClassNotFoundException => false
+            }
+        )
 
-            new TerminalSupport(false) {}
-        } catch {
-            case _: ClassNotFoundException => null
-        }
+        val term = if (idea) new TerminalSupport(false) {} else null
 
         val reader = new ConsoleReader(inputStream, System.out, term)
 
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/common/VisorTextTable.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/common/VisorTextTable.scala
index 58e3f21764186..6612ba4692589 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/common/VisorTextTable.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/common/VisorTextTable.scala
@@ -127,7 +127,7 @@ class VisorTextTable {
     private val rows = collection.mutable.ArrayBuffer.empty[Seq[Cell]]
 
     /** Current row, if any. */
-    private var curRow: collection.mutable.ArrayBuffer[Cell] = null
+    private var curRow: collection.mutable.ArrayBuffer[Cell] = _
 
     /** Table's margin, if any. */
     private var margin: Margin = Margin()
@@ -146,7 +146,7 @@ class VisorTextTable {
 
     /**
      * Flag indicating whether of not to automatically draw horizontal lines
-     * for multiline rows.
+     * for multi line rows.
      */
     var autoBorder = true
 
@@ -276,7 +276,7 @@ class VisorTextTable {
      */
     def addHeaderCell(lines: Any*): VisorTextTable = {
         assert(lines != null)
-        assert(lines.length > 0)
+        assert(lines.nonEmpty)
 
         // Break up long line into multiple ones - if necessary.
         val lst = lines flatten(_.toString.grouped(maxCellWidth))
@@ -400,7 +400,7 @@ class VisorTextTable {
         var hdrH = 0
 
         // Initialize column widths with header row (if any).
-        for (i <- 0 until hdr.size) {
+        for (i <- hdr.indices) {
             val c = hdr(i)
 
             colWs(i) = c.width
@@ -409,7 +409,7 @@ class VisorTextTable {
         }
 
         // Calc row heights and column widths.
-        for (i <- 0 until rows.length; j <- 0 until colsNum) {
+        for (i <- rows.indices; j <- 0 until colsNum) {
             val c = rows(i)(j)
 
             rowHs(i) = math.max(rowHs(i), c.height)
@@ -422,7 +422,7 @@ class VisorTextTable {
         val tbl = new GridStringBuilder()
 
         // Top margin.
-        for (i <- 0 until margin.top)
+        for (_ <- 0 until margin.top)
             tbl.a(" ").a(NL)
 
         // Print header, if any.
@@ -433,7 +433,7 @@ class VisorTextTable {
                 // Left margin and '|'.
                 tbl.a(blank(margin.left)).a(HDR_VER)
 
-                for (j <- 0 until hdr.size) {
+                for (j <- hdr.indices) {
                     val c = hdr(j)
 
                     if (i >= 0 && i < c.height)
@@ -459,14 +459,14 @@ class VisorTextTable {
                 // Left margin and '+'
                 tbl.a(blank(margin.left)).a(ROW_CRS)
 
-                for (k <- 0 until rows(i).size)
+                for (k <- rows(i).indices)
                     tbl.a(dash(ROW_HOR, colWs(k))).a(ROW_CRS)
 
                 // Right margin.
                 tbl.a(blank(margin.right)).a(NL)
             }
 
-            for (i <- 0 until rows.size) {
+            for (i <- rows.indices) {
                 val r = rows(i)
 
                 val rowH = rowHs(i)
@@ -478,7 +478,7 @@ class VisorTextTable {
                     // Left margin and '|'
                     tbl.a(blank(margin.left)).a(ROW_VER)
 
-                    for (k <- 0 until r.size) {
+                    for (k <- r.indices) {
                         val c = r(k)
                         val w = colWs(k)
 
@@ -502,7 +502,7 @@ class VisorTextTable {
         }
 
         // Bottom margin.
-        for (i <- 1 to margin.bottom)
+        for (_ <- 1 to margin.bottom)
             tbl.a(" ").a(NL)
 
         print(tbl.toString)
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/node/VisorNodeCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/node/VisorNodeCommand.scala
index b240aa5187723..6e1a3397ca57c 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/node/VisorNodeCommand.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/node/VisorNodeCommand.scala
@@ -17,22 +17,22 @@
 
 package org.apache.ignite.visor.commands.node
 
+import java.util.UUID
+
 import org.apache.ignite.cluster.ClusterNode
 import org.apache.ignite.internal.IgniteNodeAttributes._
 import org.apache.ignite.internal.util.lang.{GridFunc => F}
 import org.apache.ignite.internal.util.scala.impl
 import org.apache.ignite.internal.util.typedef.X
 import org.apache.ignite.internal.util.{IgniteUtils => U}
+import org.apache.ignite.internal.visor.node.{VisorNodeDataCollectorTask, VisorNodeDataCollectorTaskArg}
+import org.apache.ignite.internal.visor.util.VisorTaskUtils._
 import org.apache.ignite.visor.VisorTag
 import org.apache.ignite.visor.commands.common.{VisorConsoleCommand, VisorTextTable}
 import org.apache.ignite.visor.visor._
 
 import org.jetbrains.annotations._
 
-import java.util.UUID
-
-import org.apache.ignite.internal.visor.util.VisorTaskUtils._
-
 import scala.collection.JavaConversions._
 import scala.language.{implicitConversions, reflectiveCalls}
 import scala.util.control.Breaks._
@@ -105,6 +105,54 @@ class VisorNodeCommand extends VisorConsoleCommand {
         }
     }
 
+    private def printDataRegions(node: ClusterNode) {
+        val arg = new VisorNodeDataCollectorTaskArg(false, EVT_LAST_ORDER_KEY, EVT_THROTTLE_CNTR_KEY, false, false)
+
+        val res = executeMulti(Seq(node.id()), classOf[VisorNodeDataCollectorTask], arg)
+
+        val t = VisorTextTable()
+
+        t #= ("Name", "Page size", "Pages", "Memory", "Rates", "Checkpoint buffer", "Large entries")
+
+        val mm = res.getMemoryMetrics
+
+        mm.values().flatten.toSeq.sortBy(_.getName.toLowerCase).foreach(m => {
+            // Add row.
+            t += (
+                m.getName,
+                formatMemory(m.getPageSize),
+                (
+                    "Total:  " + formatNumber(m.getTotalAllocatedPages),
+                    "Dirty:  " + formatNumber(m.getDirtyPages),
+                    "Memory: " + formatNumber(m.getPhysicalMemoryPages),
+                    "Fill factor: " + formatDouble(m.getPagesFillFactor * 100) + "%"
+                ),
+                (
+                    "Total:  " + formatMemory(m.getTotalAllocatedSize),
+                    "In RAM: " + formatMemory(m.getPhysicalMemorySize)
+                ),
+                (
+                    "Allocation: " + formatDouble(m.getAllocationRate),
+                    "Eviction:   " + formatDouble(m.getEvictionRate),
+                    "Replace:    " + formatDouble(m.getPagesReplaceRate)
+                ),
+                (
+                    "Pages: " + formatNumber(m.getCheckpointBufferPages),
+                    "Size:  " + formatMemory(m.getCheckpointBufferSize)
+                ),
+                formatDouble(m.getLargeEntriesPagesPercentage * 100) + "%"
+            )
+        })
+
+        nl()
+
+        println("Data region metrics:")
+
+        t.render()
+
+        nl()
+    }
+
     /**
      * ===Command===
      * Prints full node information.
@@ -144,7 +192,7 @@ class VisorNodeCommand extends VisorConsoleCommand {
                         try
                             node = ignite.cluster.node(UUID.fromString(id.get))
                         catch {
-                            case e: IllegalArgumentException => warn("Invalid node ID: " + id.get).^^
+                            case _: IllegalArgumentException => warn("Invalid node ID: " + id.get).^^
                         }
                     else
                         warn("Invalid arguments: " + args).^^
@@ -163,33 +211,38 @@ class VisorNodeCommand extends VisorConsoleCommand {
 
                         (0 /: sortAddresses(node.addresses))((b, a) => { t += ("Address (" + b + ")", a); b + 1 })
 
-                        val m = node.metrics
+                        t += ("OS info", "" +
+                            node.attribute("os.name") + " " +
+                            node.attribute("os.arch") + " " +
+                            node.attribute("os.version"))
 
-                        val igniteInstanceName: String = node.attribute(ATTR_IGNITE_INSTANCE_NAME)
+                        t += ("OS user", node.attribute(ATTR_USER_NAME))
+                        t += ("Deployment mode", node.attribute(ATTR_DEPLOYMENT_MODE))
+                        t += ("Language runtime", node.attribute(ATTR_LANG_RUNTIME))
 
                         val ver = U.productVersion(node)
                         val verStr = ver.major() + "." + ver.minor() + "." + ver.maintenance() +
                             (if (F.isEmpty(ver.stage())) "" else "-" + ver.stage())
 
+                        t += ("Ignite version", verStr)
+
+                        val igniteInstanceName: String = node.attribute(ATTR_IGNITE_INSTANCE_NAME)
+
+                        t += ("Ignite instance name", escapeName(igniteInstanceName))
+
+                        t += ("JRE information", node.attribute(ATTR_JIT_NAME))
+
+                        val m = node.metrics
+
+                        t += ("JVM start time", formatDateTime(m.getStartTime))
+                        t += ("Node start time", formatDateTime(m.getNodeStartTime))
+                        t += ("Up time", X.timeSpan2HMSM(m.getUpTime))
+                        t += ("CPUs", formatNumber(m.getTotalCpus))
+                        t += ("Last metric update", formatDateTime(m.getLastUpdateTime))
+
                         if (all) {
-                            t += ("OS info", "" +
-                                node.attribute("os.name") + " " +
-                                node.attribute("os.arch") + " " +
-                                node.attribute("os.version")
-                            )
-                            t += ("OS user", node.attribute(ATTR_USER_NAME))
-                            t += ("Deployment mode", node.attribute(ATTR_DEPLOYMENT_MODE))
-                            t += ("Language runtime", node.attribute(ATTR_LANG_RUNTIME))
-                            t += ("Ignite version", verStr)
-                            t += ("JRE information", node.attribute(ATTR_JIT_NAME))
                             t += ("Non-loopback IPs", node.attribute(ATTR_IPS))
                             t += ("Enabled MACs", node.attribute(ATTR_MACS))
-                            t += ("Ignite instance name", escapeName(igniteInstanceName))
-                            t += ("JVM start time", formatDateTime(m.getStartTime))
-                            t += ("Node start time", formatDateTime(m.getNodeStartTime))
-                            t += ("Up time", X.timeSpan2HMSM(m.getUpTime))
-                            t += ("CPUs", formatNumber(m.getTotalCpus))
-                            t += ("Last metric update", formatDateTime(m.getLastUpdateTime))
                             t += ("Maximum active jobs", formatNumber(m.getMaximumActiveJobs))
                             t += ("Current active jobs", formatNumber(m.getCurrentActiveJobs))
                             t += ("Average active jobs", formatDouble(m.getAverageActiveJobs))
@@ -209,7 +262,7 @@ class VisorNodeCommand extends VisorConsoleCommand {
                             t += ("Current job wait time", formatNumber(m.getCurrentJobWaitTime) + "ms")
                             t += ("Average job wait time", formatDouble(m.getAverageJobWaitTime) + "ms")
                             t += ("Maximum job execute time", formatNumber(m.getMaximumJobExecuteTime) + "ms")
-                            t += ("Curent job execute time", formatNumber(m.getCurrentJobExecuteTime) + "ms")
+                            t += ("Current job execute time", formatNumber(m.getCurrentJobExecuteTime) + "ms")
                             t += ("Average job execute time", formatDouble(m.getAverageJobExecuteTime) + "ms")
                             t += ("Total busy time", formatNumber(m.getTotalBusyTime) + "ms")
                             t += ("Busy time %", formatDouble(m.getBusyTimePercentage * 100) + "%")
@@ -229,23 +282,7 @@ class VisorNodeCommand extends VisorConsoleCommand {
                             t += ("Current daemon thread count", formatNumber(m.getCurrentDaemonThreadCount))
                         }
                         else {
-                            t += ("OS info", "" +
-                                node.attribute("os.name") + " " +
-                                node.attribute("os.arch") + " " +
-                                node.attribute("os.version")
-                            )
-                            t += ("OS user", node.attribute(ATTR_USER_NAME))
-                            t += ("Deployment mode", node.attribute(ATTR_DEPLOYMENT_MODE))
-                            t += ("Language runtime", node.attribute(ATTR_LANG_RUNTIME))
-                            t += ("Ignite version", verStr)
-                            t += ("JRE information", node.attribute(ATTR_JIT_NAME))
-                            t += ("Ignite instance name", escapeName(igniteInstanceName))
-                            t += ("JVM start time", formatDateTime(m.getStartTime))
-                            t += ("Node start time", formatDateTime(m.getNodeStartTime))
-                            t += ("Up time", X.timeSpan2HMSM(m.getUpTime))
-                            t += ("Last metric update", formatDateTime(m.getLastUpdateTime))
-                            t += ("CPUs", formatNumber(m.getTotalCpus))
-                            t += ("Thread count", formatNumber(m.getCurrentThreadCount))
+                            t += ("Threads count", formatNumber(m.getCurrentThreadCount))
                             t += ("Cur/avg active jobs", formatNumber(m.getCurrentActiveJobs) +
                                 "/" + formatDouble(m.getAverageActiveJobs))
                             t += ("Cur/avg waiting jobs", formatNumber(m.getCurrentWaitingJobs) +
@@ -268,7 +305,9 @@ class VisorNodeCommand extends VisorConsoleCommand {
 
                         t.render()
 
-                        if (!all)
+                        if (all)
+                            printDataRegions(node)
+                        else
                             println("\nUse \"-a\" flag to see detailed statistics.")
                     }
                 }

From d90c700f082cdaa2f1a805eecd7b968442b80d43 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov 
Date: Wed, 24 Jan 2018 19:24:01 +0700
Subject: [PATCH 41/65] Added check icon.

(cherry picked from commit 992b025)
---
 .../frontend/public/images/icons/check.svg    |  3 ++
 .../frontend/public/images/icons/index.js     |  1 +
 .../frontend/public/stylesheets/style.scss    | 33 +++++++++++++++++++
 3 files changed, 37 insertions(+)
 create mode 100644 modules/web-console/frontend/public/images/icons/check.svg

diff --git a/modules/web-console/frontend/public/images/icons/check.svg b/modules/web-console/frontend/public/images/icons/check.svg
new file mode 100644
index 0000000000000..ed7476412deb7
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/check.svg
@@ -0,0 +1,3 @@
+
+    
+
diff --git a/modules/web-console/frontend/public/images/icons/index.js b/modules/web-console/frontend/public/images/icons/index.js
index 48d9227833b88..816ef1f565376 100644
--- a/modules/web-console/frontend/public/images/icons/index.js
+++ b/modules/web-console/frontend/public/images/icons/index.js
@@ -26,6 +26,7 @@ export search from './search.svg';
 export refresh from './refresh.svg';
 export sort from './sort.svg';
 export info from './info.svg';
+export check from './check.svg';
 export checkmark from './checkmark.svg';
 export alert from './alert.svg';
 export attention from './attention.svg';
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index 9d4cad9035603..baa49f87f7a14 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -2211,3 +2211,36 @@ html,body,.splash-screen {
         margin-left: 5px;
     }
 }
+
+header.header-with-selector {
+    margin: 40px 0 30px;
+
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+
+
+    h1 {
+        height: 36px;
+        margin: 0;
+        margin-right: 8px;
+
+        font-family: Roboto, sans-serif;
+        font-size: 24px;
+        line-height: 36px;
+    }
+
+    div {
+        display: flex;
+    }
+
+    .btn-ignite + .btn-ignite-group {
+        position: relative;
+        margin-left: 20px;
+
+        ul.dropdown-menu {
+            min-width: 100%;
+            font-family: Roboto;
+        }
+    }
+}

From 6684b14d9e4aefd7df1b2f2fbcc930a8214217b6 Mon Sep 17 00:00:00 2001
From: Igor Sapego 
Date: Wed, 24 Jan 2018 15:58:34 +0300
Subject: [PATCH 42/65] IGNITE-7443: Native batching for ODBC. This closes
 #3422.

---
 .../odbc/odbc/OdbcRequestHandler.java         | 77 ++++++++++---------
 .../cpp/odbc-test/src/odbc_test_suite.cpp     | 27 +++++--
 .../cpp/odbc-test/src/sql_get_info_test.cpp   |  2 +-
 .../include/ignite/odbc/app/parameter_set.h   | 26 ++++++-
 .../cpp/odbc/include/ignite/odbc/message.h    | 12 ---
 .../cpp/odbc/src/app/parameter_set.cpp        | 21 ++++-
 .../cpp/odbc/src/config/connection_info.cpp   |  2 +-
 modules/platforms/cpp/odbc/src/connection.cpp |  8 ++
 modules/platforms/cpp/odbc/src/message.cpp    |  4 +-
 .../cpp/odbc/src/query/batch_query.cpp        | 13 +++-
 modules/platforms/cpp/odbc/src/statement.cpp  | 18 +++++
 11 files changed, 149 insertions(+), 61 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
index 4ce99e33df652..3d5d7f2dea7d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
@@ -17,11 +17,11 @@
 
 package org.apache.ignite.internal.processors.odbc.odbc;
 
+import java.sql.BatchUpdateException;
 import java.sql.ParameterMetaData;
 import java.sql.PreparedStatement;
 import java.sql.Types;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -30,22 +30,21 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.query.FieldsQueryCursor;
-import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.binary.BinaryWriterExImpl;
 import org.apache.ignite.internal.binary.GridBinaryMarshaller;
 import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx;
-import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
 import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler;
 import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
 import org.apache.ignite.internal.processors.odbc.odbc.escape.OdbcEscapeUtils;
-import org.apache.ignite.internal.processors.query.GridQueryIndexing;
 import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static org.apache.ignite.internal.processors.odbc.odbc.OdbcRequest.META_COLS;
@@ -278,9 +277,6 @@ private ClientListenerResponse executeQuery(OdbcQueryExecuteRequest req) {
      * @return Response.
      */
     private ClientListenerResponse executeBatchQuery(OdbcQueryExecuteBatchRequest req) {
-        List rowsAffected = new ArrayList<>(req.arguments().length);
-        int currentSet = 0;
-
         try {
             String sql = OdbcEscapeUtils.parse(req.sqlQuery());
 
@@ -288,7 +284,7 @@ private ClientListenerResponse executeBatchQuery(OdbcQueryExecuteBatchRequest re
                 log.debug("ODBC query parsed [reqId=" + req.requestId() + ", original=" + req.sqlQuery() +
                         ", parsed=" + sql + ']');
 
-            SqlFieldsQuery qry = makeQuery(req.schema(), sql, req.arguments(), req.timeout());
+            SqlFieldsQueryEx qry = makeQuery(req.schema(), sql, null, req.timeout());
 
             Object[][] paramSet = req.arguments();
 
@@ -297,19 +293,16 @@ private ClientListenerResponse executeBatchQuery(OdbcQueryExecuteBatchRequest re
                         + paramSet.length + ']');
 
             // Getting meta and do the checks for the first execution.
-            qry.setArgs(paramSet[0]);
-
-            QueryCursorImpl> qryCur = (QueryCursorImpl>)ctx.query()
-                .querySqlFields(qry, true, true).get(0);
+            for (Object[] set  : paramSet)
+                qry.addBatchedArgs(set);
 
-            if (qryCur.isQuery())
-                throw new IgniteException("Batching of parameters only supported for DML statements. [query=" +
-                        req.sqlQuery() + ']');
+            List>> qryCurs =
+                ctx.query().querySqlFields(qry, true, true);
 
-            rowsAffected.add(OdbcUtils.rowsAffected(qryCur));
+            List rowsAffected = new ArrayList<>(req.arguments().length);
 
-            for (currentSet = 1; currentSet < paramSet.length; ++currentSet)
-                rowsAffected.add(executeQuery(qry, paramSet[currentSet]));
+            for (FieldsQueryCursor> qryCur : qryCurs)
+                rowsAffected.add(OdbcUtils.rowsAffected(qryCur));
 
             OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected);
 
@@ -318,24 +311,10 @@ private ClientListenerResponse executeBatchQuery(OdbcQueryExecuteBatchRequest re
         catch (Exception e) {
             U.error(log, "Failed to execute SQL query [reqId=" + req.requestId() + ", req=" + req + ']', e);
 
-            return exceptionToBatchResult(e, rowsAffected, currentSet);
+            return exceptionToBatchResult(e);
         }
     }
 
-    /**
-     * Execute query.
-     * @param qry Query
-     * @param row Row
-     * @return Affected rows.
-     */
-    private long executeQuery(SqlFieldsQuery qry, Object[] row) {
-        qry.setArgs(row);
-
-        QueryCursor> cur = ctx.query().querySqlFields(qry, true, true).get(0);
-
-        return OdbcUtils.rowsAffected(cur);
-    }
-
     /**
      * {@link OdbcQueryCloseRequest} command handler.
      *
@@ -658,9 +637,35 @@ private static boolean matches(String str, String ptrn) {
      * @param e Exception to convert.
      * @return resulting {@link OdbcResponse}.
      */
-    private OdbcResponse exceptionToBatchResult(Exception e, Collection rowsAffected, long currentSet) {
-        OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected, currentSet,
-            OdbcUtils.tryRetrieveSqlErrorCode(e), OdbcUtils.tryRetrieveH2ErrorMessage(e));
+    private OdbcResponse exceptionToBatchResult(Exception e) {
+        int code;
+        String msg;
+        List rowsAffected = new ArrayList<>();
+
+        if (e instanceof IgniteSQLException) {
+            BatchUpdateException batchCause = X.cause(e, BatchUpdateException.class);
+
+            if (batchCause != null) {
+                for (long cnt : batchCause.getLargeUpdateCounts())
+                    rowsAffected.add(cnt);
+
+                msg = batchCause.getMessage();
+
+                code = batchCause.getErrorCode();
+            }
+            else {
+                msg = OdbcUtils.tryRetrieveH2ErrorMessage(e);
+
+                code = ((IgniteSQLException)e).statusCode();
+            }
+        }
+        else {
+            msg = e.getMessage();
+
+            code = IgniteQueryErrorCode.UNKNOWN;
+        }
+
+        OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected, -1, code, msg);
 
         return new OdbcResponse(res);
     }
diff --git a/modules/platforms/cpp/odbc-test/src/odbc_test_suite.cpp b/modules/platforms/cpp/odbc-test/src/odbc_test_suite.cpp
index a967b4f10bc6d..970c77d5a5cf2 100644
--- a/modules/platforms/cpp/odbc-test/src/odbc_test_suite.cpp
+++ b/modules/platforms/cpp/odbc-test/src/odbc_test_suite.cpp
@@ -463,21 +463,38 @@ namespace ignite
         {
             Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
 
+            std::vector statuses(recordsNum, 42);
+
+            // Binding statuses array.
+            SQLRETURN ret = SQLSetStmtAttr(stmt, SQL_ATTR_PARAM_STATUS_PTR, &statuses[0], 0);
+
+            if (!SQL_SUCCEEDED(ret))
+                BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
             // Inserting values.
-            int inserted = InsertTestBatch(splitAt, recordsNum, recordsNum - splitAt);
+            int setsProcessed = InsertTestBatch(splitAt, recordsNum, recordsNum - splitAt);
 
-            BOOST_REQUIRE_EQUAL(inserted, recordsNum - splitAt);
+            BOOST_REQUIRE_EQUAL(setsProcessed, recordsNum - splitAt);
 
-            inserted = InsertTestBatch(0, recordsNum, splitAt);
+            for (int i = 0; i < recordsNum - splitAt; ++i)
+                BOOST_REQUIRE_EQUAL(statuses[i], SQL_PARAM_SUCCESS);
 
-            BOOST_REQUIRE_EQUAL(inserted, splitAt);
+            setsProcessed = InsertTestBatch(0, recordsNum, splitAt);
+
+            BOOST_REQUIRE_EQUAL(setsProcessed, recordsNum);
+
+            for (int i = 0; i < splitAt; ++i)
+                BOOST_REQUIRE_EQUAL(statuses[i], SQL_PARAM_SUCCESS);
+
+            for (int i = splitAt; i < recordsNum; ++i)
+                BOOST_REQUIRE_EQUAL(statuses[i], SQL_PARAM_ERROR);
 
             int64_t key = 0;
             char strField[1024] = {0};
             SQLLEN strFieldLen = 0;
 
             // Binding columns.
-            SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
+            ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0);
 
             if (!SQL_SUCCEEDED(ret))
                 BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
diff --git a/modules/platforms/cpp/odbc-test/src/sql_get_info_test.cpp b/modules/platforms/cpp/odbc-test/src/sql_get_info_test.cpp
index ad1236c68c667..f303e2cc683b7 100644
--- a/modules/platforms/cpp/odbc-test/src/sql_get_info_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/sql_get_info_test.cpp
@@ -241,7 +241,7 @@ BOOST_AUTO_TEST_CASE(TestValues)
     CheckIntInfo(SQL_SQL92_VALUE_EXPRESSIONS, SQL_SVE_CASE | SQL_SVE_CAST | SQL_SVE_COALESCE | SQL_SVE_NULLIF);
     CheckIntInfo(SQL_STATIC_CURSOR_ATTRIBUTES1, SQL_CA1_NEXT);
     CheckIntInfo(SQL_STATIC_CURSOR_ATTRIBUTES2, 0);
-    CheckIntInfo(SQL_PARAM_ARRAY_ROW_COUNTS, SQL_PARC_NO_BATCH);
+    CheckIntInfo(SQL_PARAM_ARRAY_ROW_COUNTS, SQL_PARC_BATCH);
     CheckIntInfo(SQL_PARAM_ARRAY_SELECTS, SQL_PAS_NO_SELECT);
     CheckIntInfo(SQL_SCROLL_OPTIONS, SQL_SO_FORWARD_ONLY | SQL_SO_STATIC);
     CheckIntInfo(SQL_ALTER_DOMAIN, 0);
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
index 2ab55808c073c..e30622daf33d0 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/app/parameter_set.h
@@ -225,7 +225,28 @@ namespace ignite
                  *
                  * @return Pointer to write number of parameters processed in batch.
                  */
-                SqlUlen* GetParamsProcessedPtr();
+                SqlUlen* GetParamsProcessedPtr() const;
+
+                /**
+                 * Set pointer to array in which to return the status of each
+                 * set of parameters.
+                 * @param value Value.
+                 */
+                void SetParamsStatusPtr(SQLUSMALLINT* value);
+
+                /**
+                 * Get pointer to array in which to return the status of each
+                 * set of parameters.
+                 * @return Value.
+                 */
+                SQLUSMALLINT* GetParamsStatusPtr() const;
+
+                /**
+                 * Set parameter status.
+                 * @param idx Parameter index.
+                 * @param status Status to set.
+                 */
+                void SetParamStatus(int64_t idx, SQLUSMALLINT status) const;
 
             private:
                 /**
@@ -249,6 +270,9 @@ namespace ignite
                 /** Processed parameters. */
                 SqlUlen* processedParamRows;
 
+                /** Parameters status. */
+                SQLUSMALLINT* paramsStatus;
+
                 /** Parameter set size. */
                 SqlUlen paramSetSize;
 
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
index 51c0b4162fdbd..af032624d1084 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
@@ -676,15 +676,6 @@ namespace ignite
                 return affectedRows;
             }
 
-            /**
-             * Get index of the set which caused an error.
-             * @return Index of the set which caused an error.
-             */
-            int64_t GetErrorSetIdx() const
-            {
-                return static_cast(affectedRows.size());
-            }
-
             /**
              * Get error message.
              * @return Error message.
@@ -714,9 +705,6 @@ namespace ignite
             /** Affected rows. */
             std::vector affectedRows;
 
-            /** Index of the set which caused an error. */
-            int64_t errorSetIdx;
-
             /** Error message. */
             std::string errorMessage;
 
diff --git a/modules/platforms/cpp/odbc/src/app/parameter_set.cpp b/modules/platforms/cpp/odbc/src/app/parameter_set.cpp
index c110d05dc5b38..55dd332d510fb 100644
--- a/modules/platforms/cpp/odbc/src/app/parameter_set.cpp
+++ b/modules/platforms/cpp/odbc/src/app/parameter_set.cpp
@@ -27,6 +27,7 @@ namespace ignite
                 parameters(),
                 paramTypes(),
                 paramBindOffset(0),
+                paramsStatus(0),
                 processedParamRows(0),
                 paramSetSize(1),
                 paramSetPos(0),
@@ -99,11 +100,29 @@ namespace ignite
                 processedParamRows = ptr;
             }
 
-            SqlUlen* ParameterSet::GetParamsProcessedPtr()
+            SqlUlen* ParameterSet::GetParamsProcessedPtr() const
             {
                 return processedParamRows;
             }
 
+            void ParameterSet::SetParamsStatusPtr(SQLUSMALLINT* value)
+            {
+                paramsStatus = value;
+            }
+
+            SQLUSMALLINT* ParameterSet::GetParamsStatusPtr() const
+            {
+                return paramsStatus;
+            }
+
+            void ParameterSet::SetParamStatus(int64_t idx, SQLUSMALLINT status) const
+            {
+                if (idx < 0 || !paramsStatus || idx >= static_cast(paramSetSize))
+                    return;
+
+                paramsStatus[idx] = status;
+            }
+
             void ParameterSet::SetParamsProcessed(SqlUlen processed) const
             {
                 if (processedParamRows)
diff --git a/modules/platforms/cpp/odbc/src/config/connection_info.cpp b/modules/platforms/cpp/odbc/src/config/connection_info.cpp
index 0f8e50b08b2e9..a70121dfc3957 100644
--- a/modules/platforms/cpp/odbc/src/config/connection_info.cpp
+++ b/modules/platforms/cpp/odbc/src/config/connection_info.cpp
@@ -1273,7 +1273,7 @@ namespace ignite
                 //     resulting from the execution of the statement for the entire array of parameters. This is
                 //     conceptually equivalent to treating the statement together with the complete parameter array as
                 //     one atomic unit. Errors are handled the same as if one statement were executed.
-                intParams[SQL_PARAM_ARRAY_ROW_COUNTS] = SQL_PARC_NO_BATCH;
+                intParams[SQL_PARAM_ARRAY_ROW_COUNTS] = SQL_PARC_BATCH;
 #endif // SQL_PARAM_ARRAY_ROW_COUNTS
 
 #ifdef SQL_PARAM_ARRAY_SELECTS
diff --git a/modules/platforms/cpp/odbc/src/connection.cpp b/modules/platforms/cpp/odbc/src/connection.cpp
index 8b0387627d20d..4e8f4abc91819 100644
--- a/modules/platforms/cpp/odbc/src/connection.cpp
+++ b/modules/platforms/cpp/odbc/src/connection.cpp
@@ -29,6 +29,9 @@
 #include "ignite/odbc/message.h"
 #include "ignite/odbc/config/configuration.h"
 
+// Uncomment for per-byte debug.
+//#define PER_BYTE_DEBUG
+
 namespace
 {
 #pragma pack(push, 1)
@@ -39,6 +42,7 @@ namespace
 #pragma pack(pop)
 }
 
+
 namespace ignite
 {
     namespace odbc
@@ -218,7 +222,9 @@ namespace ignite
             if (res == OperationResult::FAIL)
                 throw OdbcError(SqlState::S08S01_LINK_FAILURE, "Can not send message due to connection failure");
 
+#ifdef PER_BYTE_DEBUG
             LOG_MSG("message sent: (" <<  msg.GetSize() << " bytes)" << utility::HexDump(msg.GetData(), msg.GetSize()));
+#endif //PER_BYTE_DEBUG
 
             return true;
         }
@@ -285,7 +291,9 @@ namespace ignite
             if (res == OperationResult::FAIL)
                 throw OdbcError(SqlState::S08S01_LINK_FAILURE, "Can not receive message body");
 
+#ifdef PER_BYTE_DEBUG
             LOG_MSG("Message received: " << utility::HexDump(&msg[0], msg.size()));
+#endif //PER_BYTE_DEBUG
 
             return true;
         }
diff --git a/modules/platforms/cpp/odbc/src/message.cpp b/modules/platforms/cpp/odbc/src/message.cpp
index 57b72103ff1a2..93fc6bbc06e2e 100644
--- a/modules/platforms/cpp/odbc/src/message.cpp
+++ b/modules/platforms/cpp/odbc/src/message.cpp
@@ -350,7 +350,6 @@ namespace ignite
 
         QueryExecuteBatchResponse::QueryExecuteBatchResponse():
             affectedRows(0),
-            errorSetIdx(-1),
             errorMessage(),
             errorCode(1)
         {
@@ -370,7 +369,8 @@ namespace ignite
 
             if (!success)
             {
-                errorSetIdx = reader.ReadInt64();
+                // Ignoring error set idx. To be deleted in next major version.
+                reader.ReadInt64();
                 errorMessage = reader.ReadObject();
 
                 if (ver >= ProtocolVersion::VERSION_2_1_5)
diff --git a/modules/platforms/cpp/odbc/src/query/batch_query.cpp b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
index a9db8d81e8394..235daf823feb0 100644
--- a/modules/platforms/cpp/odbc/src/query/batch_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
@@ -72,7 +72,7 @@ namespace ignite
                     res = MakeRequestExecuteBatch(processed, processed + currentPageSize, lastPage);
 
                     processed += currentPageSize;
-                } while (res == SqlResult::AI_SUCCESS && processed < rowNum);
+                } while ((res == SqlResult::AI_SUCCESS || res == SqlResult::AI_SUCCESS_WITH_INFO) && processed < rowNum);
 
                 params.SetParamsProcessed(static_cast(rowsAffected.size()));
 
@@ -187,7 +187,16 @@ namespace ignite
                     return SqlResult::AI_ERROR;
                 }
 
-                rowsAffected.insert(rowsAffected.end(), rsp.GetAffectedRows().begin(), rsp.GetAffectedRows().end());
+                const std::vector& rowsLastTime = rsp.GetAffectedRows();
+
+                for (size_t i = 0; i < rowsLastTime.size(); ++i)
+                {
+                    int64_t idx = static_cast(i + rowsAffected.size());
+
+                    params.SetParamStatus(idx, rowsLastTime[i] < 0 ? SQL_PARAM_ERROR : SQL_PARAM_SUCCESS);
+                }
+
+                rowsAffected.insert(rowsAffected.end(), rowsLastTime.begin(), rowsLastTime.end());
                 LOG_MSG("Affected rows list size: " << rowsAffected.size());
 
                 if (!rsp.GetErrorMessage().empty())
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index 898d44d43c827..332487e5d9c44 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -285,6 +285,12 @@ namespace ignite
                     break;
                 }
 
+                case SQL_ATTR_PARAM_STATUS_PTR:
+                {
+                    parameters.SetParamsStatusPtr(reinterpret_cast(value));
+                    break;
+                }
+
                 case SQL_ATTR_QUERY_TIMEOUT:
                 {
                     SqlUlen uTimeout = reinterpret_cast(value);
@@ -435,6 +441,18 @@ namespace ignite
                     break;
                 }
 
+                case SQL_ATTR_PARAM_STATUS_PTR:
+                {
+                    SQLUSMALLINT** val = reinterpret_cast(buf);
+
+                    *val = parameters.GetParamsStatusPtr();
+
+                    if (valueLen)
+                        *valueLen = SQL_IS_POINTER;
+
+                    break;
+                }
+
                 case SQL_ATTR_QUERY_TIMEOUT:
                 {
                     SqlUlen *uTimeout = reinterpret_cast(buf);

From 6173365e6254f34715b47353ed93228c6b72530c Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Wed, 24 Jan 2018 10:46:34 +0300
Subject: [PATCH 43/65] master Add a method to quickly check if a node is in
 baseline topology

---
 .../ignite/internal/managers/discovery/DiscoCache.java |  8 ++++++++
 .../cache/persistence/file/FilePageStore.java          |  2 +-
 .../processors/cluster/GridClusterStateProcessor.java  | 10 ++++++----
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index f73f4205570f2..7bc0344ebd002 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -275,6 +275,14 @@ public Collection aliveServerNodes() {
 
     }
 
+    /**
+     * @param node Node to check.
+     * @return {@code True} if the node is in baseline or if baseline is not set.
+     */
+    public boolean baselineNode(ClusterNode node) {
+        return nodeIdToConsIdx == null || nodeIdToConsIdx.get(node.id()) != null;
+    }
+
     /**
      * @return Oldest alive server node.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index ddf6062c864f4..1153c88cd77fd 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -126,7 +126,7 @@ public int headerSize() {
     /**
      * Page store version.
      */
-    public int version() {
+    @Override public int version() {
         return VERSION;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 54e0f56f5d90f..73c9d7f0571b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -243,12 +243,14 @@ public boolean compatibilityMode() {
      * @throws IgniteCheckedException If write to metastore has failed.
      */
     public void resetBranchingHistory(long newBranchingHash) throws IgniteCheckedException {
-        globalState.baselineTopology().resetBranchingHistory(newBranchingHash);
+        if (!compatibilityMode()) {
+            globalState.baselineTopology().resetBranchingHistory(newBranchingHash);
 
-        writeBaselineTopology(globalState.baselineTopology(), null);
+            writeBaselineTopology(globalState.baselineTopology(), null);
 
-        U.log(log,
-            String.format("Branching history of current BaselineTopology is reset to the value %d", newBranchingHash));
+            U.log(log,
+                String.format("Branching history of current BaselineTopology is reset to the value %d", newBranchingHash));
+        }
     }
 
     /**

From 8484c191e34c6e835d9e2e499d76866e0aaf10b9 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh 
Date: Wed, 24 Jan 2018 17:54:09 +0300
Subject: [PATCH 44/65] IGNITE-7514 Choose owner primary node after cluster
 restart

---
 .../internal/events/DiscoveryCustomEvent.java |  34 ++++
 .../cache/CacheAffinitySharedManager.java     |  76 +++++---
 .../processors/cache/ClusterCachesInfo.java   |   1 +
 .../GridCachePartitionExchangeManager.java    |   9 +-
 .../dht/GridClientPartitionTopology.java      |   9 +-
 .../dht/GridDhtLocalPartition.java            |   3 -
 .../dht/GridDhtPartitionTopology.java         |  10 +-
 .../dht/GridDhtPartitionTopologyImpl.java     |  47 ++++-
 .../GridDhtPartitionsExchangeFuture.java      | 134 ++++++++------
 .../GridCacheDatabaseSharedManager.java       |   4 +
 .../CacheBaselineTopologyTest.java            | 166 +++++++++++++++++-
 11 files changed, 408 insertions(+), 85 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
index b3c6a2d865b2f..3b12b384f54be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
@@ -21,7 +21,10 @@
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
+import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Custom event.
@@ -85,4 +88,35 @@ public void affinityTopologyVersion(AffinityTopologyVersion affTopVer) {
     @Override public String toString() {
         return S.toString(DiscoveryCustomEvent.class, this, super.toString());
     }
+
+    /**
+     * @param evt Discovery event.
+     * @return {@code True} if event is DiscoveryCustomEvent that requires centralized affinity assignment.
+     */
+    public static boolean requiresCentralizedAffinityAssignment(DiscoveryEvent evt) {
+        if (!(evt instanceof DiscoveryCustomEvent))
+            return false;
+
+        return requiresCentralizedAffinityAssignment(((DiscoveryCustomEvent)evt).customMessage());
+    }
+
+    /**
+     * @param msg Discovery custom message.
+     * @return {@code True} if message belongs to event that requires centralized affinity assignment.
+     */
+    public static boolean requiresCentralizedAffinityAssignment(@Nullable DiscoveryCustomMessage msg) {
+        if (msg == null)
+            return false;
+
+        if (msg instanceof ChangeGlobalStateMessage && ((ChangeGlobalStateMessage)msg).activate())
+            return true;
+
+        if (msg instanceof SnapshotDiscoveryMessage) {
+            SnapshotDiscoveryMessage snapMsg = (SnapshotDiscoveryMessage) msg;
+
+            return snapMsg.needExchange() && snapMsg.needAssignPartitions();
+        }
+
+        return false;
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 63aba9849c820..921701a14d01f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
+import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
@@ -57,7 +58,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
-import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
@@ -161,13 +161,15 @@ void onDiscoveryEvent(int type,
         ClusterNode node,
         AffinityTopologyVersion topVer,
         DiscoveryDataClusterState state) {
-        if (state.transition() || !state.active())
+        if ((state.transition() || !state.active()) &&
+            !DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(customMsg))
             return;
 
         if (type == EVT_NODE_JOINED && node.isLocal())
             lastAffVer = null;
 
-        if (!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) {
+        if ((!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) ||
+            DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(customMsg)) {
             synchronized (mux) {
                 assert lastAffVer == null || topVer.compareTo(lastAffVer) > 0;
 
@@ -1263,7 +1265,7 @@ public GridAffinityAssignmentCache affinity(Integer grpId) {
      * @param fut Current exchange future.
      * @param msg Finish exchange message.
      */
-    public void mergeExchangesOnServerLeft(final GridDhtPartitionsExchangeFuture fut,
+    public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture fut,
         final GridDhtPartitionsFullMessage msg) {
         final Map nodesByOrder = new HashMap<>();
 
@@ -1396,7 +1398,7 @@ public void onServerJoinWithExchangeMergeProtocol(GridDhtPartitionsExchangeFutur
      * @return Computed difference with ideal affinity.
      * @throws IgniteCheckedException If failed.
      */
-    public  Map onServerLeftWithExchangeMergeProtocol(
+    public Map onServerLeftWithExchangeMergeProtocol(
         final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
     {
         final ExchangeDiscoveryEvents evts = fut.context().events();
@@ -1404,6 +1406,32 @@ public  Map onServerLeftWithExchangeMergePro
         assert fut.context().mergeExchanges();
         assert evts.hasServerLeft();
 
+        return onReassignmentEnforced(fut);
+    }
+
+    /**
+     * @param fut Current exchange future.
+     * @return Computed difference with ideal affinity.
+     * @throws IgniteCheckedException If failed.
+     */
+    public Map onCustomEventWithEnforcedAffinityReassignment(
+        final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
+    {
+        assert DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent());
+
+        return onReassignmentEnforced(fut);
+    }
+
+    /**
+     * @param fut Current exchange future.
+     * @return Computed difference with ideal affinity.
+     * @throws IgniteCheckedException If failed.
+     */
+    public Map onReassignmentEnforced(
+        final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
+    {
+        final ExchangeDiscoveryEvents evts = fut.context().events();
+
         forAllRegisteredCacheGroups(new IgniteInClosureX() {
             @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException {
                 AffinityTopologyVersion topVer = evts.topologyVersion();
@@ -1418,7 +1446,7 @@ public  Map onServerLeftWithExchangeMergePro
             }
         });
 
-        Map>> diff = initAffinityOnNodeLeft0(evts.topologyVersion(),
+        Map>> diff = initAffinityBasedOnPartitionsAvailability(evts.topologyVersion(),
             fut,
             NODE_TO_ORDER,
             true);
@@ -1642,17 +1670,16 @@ private GridDhtAffinityAssignmentResponse fetchAffinity(AffinityTopologyVersion
     }
 
     /**
-     * Called on exchange initiated by server node leave.
+     * Called on exchange initiated by server node leave or custom event with centralized affinity assignment.
      *
      * @param fut Exchange future.
      * @param crd Coordinator flag.
      * @throws IgniteCheckedException If failed.
      * @return {@code True} if affinity should be assigned by coordinator.
      */
-    public boolean onServerLeft(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
-        ClusterNode leftNode = fut.firstEvent().eventNode();
-
-        assert !leftNode.isClient() : leftNode;
+    public boolean onCentralizedAffinityChange(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
+        assert (fut.events().hasServerLeft() && !fut.firstEvent().eventNode().isClient()) ||
+            DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent()) : fut.firstEvent();
 
         if (crd) {
             // Need initialize CacheGroupHolders if this node become coordinator on this exchange.
@@ -2066,7 +2093,7 @@ public IgniteInternalFuture>>> initAffinity
             initFut.listen(new IgniteInClosure>() {
                 @Override public void apply(IgniteInternalFuture initFut) {
                     try {
-                        resFut.onDone(initAffinityOnNodeLeft0(fut.initialVersion(), fut, NODE_TO_ID, false));
+                        resFut.onDone(initAffinityBasedOnPartitionsAvailability(fut.initialVersion(), fut, NODE_TO_ID, false));
                     }
                     catch (IgniteCheckedException e) {
                         resFut.onDone(e);
@@ -2077,7 +2104,7 @@ public IgniteInternalFuture>>> initAffinity
             return resFut;
         }
         else
-            return new GridFinishedFuture<>(initAffinityOnNodeLeft0(fut.initialVersion(), fut, NODE_TO_ID, false));
+            return new GridFinishedFuture<>(initAffinityBasedOnPartitionsAvailability(fut.initialVersion(), fut, NODE_TO_ID, false));
     }
 
     /**
@@ -2088,12 +2115,17 @@ public IgniteInternalFuture>>> initAffinity
      * @return Affinity assignment.
      * @throws IgniteCheckedException If failed.
      */
-    private  Map>> initAffinityOnNodeLeft0(final AffinityTopologyVersion topVer,
+    private  Map>> initAffinityBasedOnPartitionsAvailability(final AffinityTopologyVersion topVer,
         final GridDhtPartitionsExchangeFuture fut,
         final IgniteClosure c,
         final boolean initAff)
         throws IgniteCheckedException {
-        final WaitRebalanceInfo waitRebalanceInfo = new WaitRebalanceInfo(fut.context().events().lastServerEventVersion());
+        final boolean enforcedCentralizedAssignment =
+            DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent());
+
+        final WaitRebalanceInfo waitRebalanceInfo = enforcedCentralizedAssignment ?
+            new WaitRebalanceInfo(fut.exchangeId().topologyVersion()) :
+            new WaitRebalanceInfo(fut.context().events().lastServerEventVersion());
 
         final Collection aliveNodes = fut.context().events().discoveryCache().serverNodes();
 
@@ -2103,13 +2135,14 @@ private  Map>> initAffinityOnNodeLeft0(final Af
             @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException {
                 CacheGroupHolder grpHolder = groupHolder(topVer, desc);
 
-                if (!grpHolder.rebalanceEnabled || fut.cacheGroupAddedOnExchange(desc.groupId(), desc.receivedFrom()))
+                if (!grpHolder.rebalanceEnabled ||
+                    (fut.cacheGroupAddedOnExchange(desc.groupId(), desc.receivedFrom()) && !enforcedCentralizedAssignment))
                     return;
 
                 AffinityTopologyVersion affTopVer = grpHolder.affinity().lastVersion();
 
-                assert affTopVer.topologyVersion() > 0 && !affTopVer.equals(topVer) : "Invalid affinity version " +
-                    "[last=" + affTopVer + ", futVer=" + topVer + ", grp=" + desc.cacheOrGroupName() + ']';
+                assert (affTopVer.topologyVersion() > 0 && !affTopVer.equals(topVer)) || enforcedCentralizedAssignment :
+                    "Invalid affinity version [last=" + affTopVer + ", futVer=" + topVer + ", grp=" + desc.cacheOrGroupName() + ']';
 
                 List> curAssignment = grpHolder.affinity().assignments(affTopVer);
                 List> newAssignment = grpHolder.affinity().idealAssignment();
@@ -2141,6 +2174,11 @@ private  Map>> initAffinityOnNodeLeft0(final Af
                         ", node=" + newPrimary +
                         ", topVer=" + topVer + ']';
 
+                    List owners = top.owners(p);
+
+                    if (!owners.isEmpty() && !owners.contains(curPrimary))
+                        curPrimary = owners.get(0);
+
                     if (curPrimary != null && newPrimary != null && !curPrimary.equals(newPrimary)) {
                         if (aliveNodes.contains(curPrimary)) {
                             GridDhtPartitionState state = top.partitionState(newPrimary.id(), p);
@@ -2173,8 +2211,6 @@ private  Map>> initAffinityOnNodeLeft0(final Af
                                 }
 
                                 if (newNodes0 == null) {
-                                    List owners = top.owners(p);
-
                                     for (ClusterNode owner : owners) {
                                         if (aliveNodes.contains(owner)) {
                                             newNodes0 = latePrimaryAssignment(grpHolder.affinity(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 08a910b81603d..2b2fb559c182e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -1186,6 +1186,7 @@ public void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) {
     /**
      * @param msg Message.
      * @param topVer Current topology version.
+     * @param curState Current cluster state.
      * @return Exchange action.
      * @throws IgniteCheckedException If configuration validation failed.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 9b9284f04b10a..0a657c9cdf779 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -253,6 +253,13 @@ public class GridCachePartitionExchangeManager extends GridCacheSharedMana
 
                     return;
                 }
+                if (evt.type() == EVT_DISCOVERY_CUSTOM_EVT &&
+                    (((DiscoveryCustomEvent)evt).customMessage() instanceof CacheAffinityChangeMessage) &&
+                    ((CacheAffinityChangeMessage)((DiscoveryCustomEvent)evt).customMessage()).exchangeId() != null) {
+                    onDiscoveryEvent(evt, cache);
+
+                    return;
+                }
 
                 if (cache.state().transition()) {
                     if (log.isDebugEnabled())
@@ -965,7 +972,7 @@ public void scheduleResendPartitions() {
      * For coordinator causes {@link GridDhtPartitionsFullMessage FullMessages} send,
      * for non coordinator -  {@link GridDhtPartitionsSingleMessage SingleMessages} send
      */
-    private void refreshPartitions() {
+    public void refreshPartitions() {
         // TODO https://issues.apache.org/jira/browse/IGNITE-6857
         if (cctx.snapshot().snapshotOperationInProgress()) {
             scheduleResendPartitions();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index e994113087ec8..def00f3f9ae89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -256,9 +256,9 @@ private String mapString(GridDhtPartitionMap map) {
     }
 
     /** {@inheritDoc} */
-    @Override public void initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
+    @Override public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
         GridDhtPartitionsExchangeFuture exchFut) {
-        // No-op.
+        return false;
     }
 
     /** {@inheritDoc} */
@@ -382,6 +382,11 @@ else if (!node2part.nodeId().equals(loc.id())) {
                 fullMapString() + ']');
     }
 
+    /** {@inheritDoc} */
+    @Override public void afterStateRestored(AffinityTopologyVersion topVer) {
+        // no-op
+    }
+
     /** {@inheritDoc} */
     @Override public boolean afterExchange(GridDhtPartitionsExchangeFuture exchFut) throws IgniteCheckedException {
         AffinityTopologyVersion topVer = exchFut.topologyVersion();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index e1f1d6f43ce1e..e63aab6c0dc78 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -213,9 +213,6 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
             // TODO ignite-db
             throw new IgniteException(e);
         }
-
-        // Todo log moving state
-        casState(state.get(), MOVING);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 4ae68ef739f8c..13564c2af2666 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -117,11 +117,19 @@ public void beforeExchange(GridDhtPartitionsExchangeFuture exchFut,
     /**
      * @param affVer Affinity version.
      * @param exchFut Exchange future.
+     * @return {@code True} if partitions must be refreshed.
      * @throws IgniteInterruptedCheckedException If interrupted.
      */
-    public void initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut)
+    public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut)
         throws IgniteInterruptedCheckedException;
 
+    /**
+     * Initializes local data structures after partitions are restored from persistence.
+     *
+     * @param topVer Topology version.
+     */
+    public void afterStateRestored(AffinityTopologyVersion topVer);
+
     /**
      * Post-initializes this topology.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 0a2c1541bd70d..f3da42a62d6bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -299,10 +299,12 @@ private String mapString(GridDhtPartitionMap map) {
     }
 
     /** {@inheritDoc} */
-    @Override public void initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
+    @Override public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
         GridDhtPartitionsExchangeFuture exchFut)
         throws IgniteInterruptedCheckedException
     {
+        boolean needRefresh;
+
         ctx.database().checkpointReadLock();
 
         try {
@@ -310,11 +312,11 @@ private String mapString(GridDhtPartitionMap map) {
 
             try {
                 if (stopping)
-                    return;
+                    return false;
 
                 long updateSeq = this.updateSeq.incrementAndGet();
 
-                initPartitions0(affVer, exchFut, updateSeq);
+                needRefresh = initPartitions0(affVer, exchFut, updateSeq);
 
                 consistencyCheck();
             }
@@ -325,16 +327,21 @@ private String mapString(GridDhtPartitionMap map) {
         finally {
             ctx.database().checkpointReadUnlock();
         }
+
+        return needRefresh;
     }
 
     /**
      * @param affVer Affinity version to use.
      * @param exchFut Exchange future.
      * @param updateSeq Update sequence.
+     * @return {@code True} if partitions must be refreshed.
      */
-    private void initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
+    private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
         List> aff = grp.affinity().readyAssignments(affVer);
 
+        boolean needRefresh = false;
+
         if (grp.affinityNode()) {
             ClusterNode loc = ctx.localNode();
 
@@ -366,6 +373,8 @@ private void initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsEx
                                 GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)grp.shared().database();
 
                                 locPart.restoreState(db.readPartitionState(grp, locPart.id()));
+
+                                needRefresh = true;
                             }
                             else {
                                 boolean owned = locPart.own();
@@ -423,6 +432,8 @@ else if (belongs) {
         }
 
         updateRebalanceVersion(aff);
+
+        return needRefresh;
     }
 
     /**
@@ -616,6 +627,30 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
         return grp.affinity().nodes(p, topVer).contains(ctx.localNode());
     }
 
+    /** {@inheritDoc} */
+    @Override public void afterStateRestored(AffinityTopologyVersion topVer) {
+        lock.writeLock().lock();
+
+        try {
+            if (node2part == null)
+                return;
+
+            long updateSeq = this.updateSeq.incrementAndGet();
+
+            for (int p = 0; p < grp.affinity().partitions(); p++) {
+                GridDhtLocalPartition locPart = locParts.get(p);
+
+                if (locPart == null)
+                    updateLocal(p, EVICTED, updateSeq, topVer);
+                else
+                    updateLocal(p, locPart.state(), updateSeq, topVer);
+            }
+        }
+        finally {
+            lock.writeLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public boolean afterExchange(GridDhtPartitionsExchangeFuture exchFut) {
         boolean changed = false;
@@ -996,9 +1031,11 @@ else if (loc != null && state == RENTING && !showRenting) {
                 map.put(i, part.state());
             }
 
+            GridDhtPartitionMap locPartMap = node2part != null ? node2part.get(ctx.localNodeId()) : null;
+
             return new GridDhtPartitionMap(ctx.localNodeId(),
                 updateSeq.get(),
-                readyTopVer,
+                locPartMap != null ? locPartMap.topologyVersion() : readyTopVer,
                 map,
                 true);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 6c09b6a4373df..9a247e12cec93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -227,6 +227,12 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
      */
     private boolean centralizedAff;
 
+    /**
+     * Enforce affinity reassignment based on actual partition distribution. This mode should be used when partitions
+     * might be distributed not according to affinity assignment.
+     */
+    private boolean forceAffReassignment;
+
     /** Change global state exception. */
     private Exception changeGlobalStateE;
 
@@ -590,6 +596,8 @@ public void init(boolean newCrd) throws IgniteInterruptedCheckedException {
 
                 DiscoveryCustomMessage msg = ((DiscoveryCustomEvent)firstDiscoEvt).customMessage();
 
+                forceAffReassignment = DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(msg);
+
                 if (msg instanceof ChangeGlobalStateMessage) {
                     assert exchActions != null && !exchActions.empty();
 
@@ -611,6 +619,9 @@ else if (msg instanceof WalStateAbstractMessage)
                     exchange = onAffinityChangeRequest(crdNode);
                 }
 
+                if (forceAffReassignment)
+                    cctx.affinity().onCentralizedAffinityChange(this, crdNode);
+
                 initCoordinatorCaches(newCrd);
             }
             else {
@@ -756,7 +767,7 @@ private void initTopologies() throws IgniteCheckedException {
                     if (grp.isLocal())
                         continue;
 
-                    grp.topology().beforeExchange(this, !centralizedAff, false);
+                    grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
                 }
             }
         }
@@ -899,8 +910,6 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
         else if (req.activate()) {
             // TODO: BLT changes on inactive cluster can't be handled easily because persistent storage hasn't been initialized yet.
             try {
-                cctx.affinity().onBaselineTopologyChanged(this, crd);
-
                 if (CU.isPersistenceEnabled(cctx.kernalContext().config()) && !cctx.kernalContext().clientNode())
                     cctx.kernalContext().state().onBaselineTopologyChanged(req.baselineTopology(),
                         req.prevBaselineTopologyHistoryItem());
@@ -937,7 +946,8 @@ private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedExcep
      * @return Exchange type.
      */
     private ExchangeType onCustomMessageNoAffinityChange(boolean crd) {
-        cctx.affinity().onCustomMessageNoAffinityChange(this, crd, exchActions);
+        if (!forceAffReassignment)
+            cctx.affinity().onCustomMessageNoAffinityChange(this, crd, exchActions);
 
         return cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL;
     }
@@ -992,7 +1002,7 @@ private ExchangeType onServerNodeEvent(boolean crd) throws IgniteCheckedExceptio
 
             exchCtx.events().warnNoAffinityNodes(cctx);
 
-            centralizedAff = cctx.affinity().onServerLeft(this, crd);
+            centralizedAff = cctx.affinity().onCentralizedAffinityChange(this, crd);
         }
         else
             cctx.affinity().onServerJoin(this, crd);
@@ -1072,7 +1082,7 @@ private void distributedExchange() throws IgniteCheckedException {
 
                 // It is possible affinity is not initialized yet if node joins to cluster.
                 if (grp.affinity().lastVersion().topologyVersion() > 0)
-                    grp.topology().beforeExchange(this, !centralizedAff, false);
+                    grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
             }
         }
 
@@ -1509,19 +1519,24 @@ public void finishMerged() {
         }
 
         if (err == null) {
-            if (centralizedAff) {
+            if (centralizedAff || forceAffReassignment) {
                 assert !exchCtx.mergeExchanges();
 
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                     if (grp.isLocal())
                         continue;
 
+                    boolean needRefresh = false;
+
                     try {
-                        grp.topology().initPartitionsWhenAffinityReady(res, this);
+                        needRefresh = grp.topology().initPartitionsWhenAffinityReady(res, this);
                     }
                     catch (IgniteInterruptedCheckedException e) {
                         U.error(log, "Failed to initialize partitions.", e);
                     }
+
+                    if (needRefresh)
+                        cctx.exchange().refreshPartitions();
                 }
             }
 
@@ -2317,7 +2332,7 @@ private void onAllReceived(@Nullable Collection sndResNodes) {
             if (!exchCtx.mergeExchanges() && !crd.equals(events().discoveryCache().serverNodes().get(0))) {
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                     if (!grp.isLocal())
-                        grp.topology().beforeExchange(this, !centralizedAff, false);
+                        grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
                 }
             }
 
@@ -2449,6 +2464,9 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
                     detectLostPartitions(resTopVer);
             }
 
+            if (!exchCtx.mergeExchanges() && forceAffReassignment)
+                idealAffDiff = cctx.affinity().onCustomEventWithEnforcedAffinityReassignment(this);
+
             for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
                 if (!grpCtx.isLocal())
                     grpCtx.topology().applyUpdateCounters();
@@ -2471,6 +2489,8 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
                 if (exchCtx.events().hasServerLeft())
                     msg.idealAffinityDiff(idealAffDiff);
             }
+            else if (forceAffReassignment)
+                msg.idealAffinityDiff(idealAffDiff);
 
             msg.prepareMarshal(cctx);
 
@@ -2524,65 +2544,69 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
                         nodes.addAll(sndResNodes);
                 }
 
-                IgniteCheckedException err = null;
+                if (!nodes.isEmpty())
+                    sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff);
 
-                if (stateChangeExchange()) {
-                    StateChangeRequest req = exchActions.stateChangeRequest();
+                if (!stateChangeExchange())
+                    onDone(exchCtx.events().topologyVersion(), null);
 
-                    assert req != null : exchActions;
+                for (Map.Entry e : pendingSingleMsgs.entrySet()) {
+                    if (log.isInfoEnabled()) {
+                        log.info("Process pending message on coordinator [node=" + e.getKey() +
+                            ", ver=" + initialVersion() +
+                            ", resVer=" + resTopVer + ']');
+                    }
 
-                    boolean stateChangeErr = false;
+                    processSingleMessage(e.getKey(), e.getValue());
+                }
+            }
 
-                    if (!F.isEmpty(changeGlobalStateExceptions)) {
-                        stateChangeErr = true;
+            if (stateChangeExchange()) {
+                IgniteCheckedException err = null;
 
-                        err = new IgniteCheckedException("Cluster state change failed.");
+                StateChangeRequest req = exchActions.stateChangeRequest();
 
-                        cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req);
-                    }
-                    else {
-                        boolean hasMoving = !partsToReload.isEmpty();
+                assert req != null : exchActions;
 
-                        Set waitGrps = cctx.affinity().waitGroups();
+                boolean stateChangeErr = false;
 
-                        if (!hasMoving) {
-                            for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
-                                if (waitGrps.contains(grpCtx.groupId()) && grpCtx.topology().hasMovingPartitions()) {
-                                    hasMoving = true;
+                if (!F.isEmpty(changeGlobalStateExceptions)) {
+                    stateChangeErr = true;
 
-                                    break;
-                                }
+                    err = new IgniteCheckedException("Cluster state change failed.");
 
-                            }
-                        }
+                    cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req);
+                }
+                else {
+                    boolean hasMoving = !partsToReload.isEmpty();
 
-                        cctx.kernalContext().state().onExchangeFinishedOnCoordinator(this, hasMoving);
-                    }
+                    Set waitGrps = cctx.affinity().waitGroups();
+
+                    if (!hasMoving) {
+                        for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
+                            if (waitGrps.contains(grpCtx.groupId()) && grpCtx.topology().hasMovingPartitions()) {
+                                hasMoving = true;
 
-                    boolean active = !stateChangeErr && req.activate();
+                                break;
+                            }
 
-                    ChangeGlobalStateFinishMessage stateFinishMsg = new ChangeGlobalStateFinishMessage(
-                        req.requestId(),
-                        active,
-                        !stateChangeErr);
+                        }
+                    }
 
-                    cctx.discovery().sendCustomEvent(stateFinishMsg);
+                    cctx.kernalContext().state().onExchangeFinishedOnCoordinator(this, hasMoving);
                 }
 
-                if (!nodes.isEmpty())
-                    sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff);
+                boolean active = !stateChangeErr && req.activate();
 
-                onDone(exchCtx.events().topologyVersion(), err);
+                ChangeGlobalStateFinishMessage stateFinishMsg = new ChangeGlobalStateFinishMessage(
+                    req.requestId(),
+                    active,
+                    !stateChangeErr);
 
-                for (Map.Entry e : pendingSingleMsgs.entrySet()) {
-                    if (log.isInfoEnabled()) {
-                        log.info("Process pending message on coordinator [node=" + e.getKey() +
-                            ", ver=" + initialVersion() +
-                            ", resVer=" + resTopVer + ']');
-                    }
+                cctx.discovery().sendCustomEvent(stateFinishMsg);
 
-                    processSingleMessage(e.getKey(), e.getValue());
-                }
+                if (!centralizedAff)
+                    onDone(exchCtx.events().topologyVersion(), err);
             }
         }
         catch (IgniteCheckedException e) {
@@ -2925,7 +2949,7 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
                     cctx.affinity().onLocalJoin(this, msg, resTopVer);
                 else {
                     if (exchCtx.events().hasServerLeft())
-                        cctx.affinity().mergeExchangesOnServerLeft(this, msg);
+                        cctx.affinity().applyAffinityFromFullMessage(this, msg);
                     else
                         cctx.affinity().onServerJoinWithExchangeMergeProtocol(this, false);
 
@@ -2939,6 +2963,8 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
             }
             else if (localJoinExchange() && !exchCtx.fetchAffinityOnJoin())
                 cctx.affinity().onLocalJoin(this, msg, resTopVer);
+            else if (forceAffReassignment)
+                cctx.affinity().applyAffinityFromFullMessage(this, msg);
 
             updatePartitionFullMap(resTopVer, msg);
 
@@ -3048,6 +3074,9 @@ public void onAffinityChangeMessage(final ClusterNode node, final CacheAffinityC
                             crd.isLocal(),
                             msg);
 
+                        IgniteCheckedException err = !F.isEmpty(msg.partitionsMessage().getErrorsMap()) ?
+                            new IgniteCheckedException("Cluster state change failed.") : null;
+
                         if (!crd.isLocal()) {
                             GridDhtPartitionsFullMessage partsMsg = msg.partitionsMessage();
 
@@ -3055,9 +3084,12 @@ public void onAffinityChangeMessage(final ClusterNode node, final CacheAffinityC
                             assert partsMsg.lastVersion() != null : partsMsg;
 
                             updatePartitionFullMap(resTopVer, partsMsg);
+
+                            if (exchActions != null && exchActions.stateChangeRequest() != null && err != null)
+                                cctx.kernalContext().state().onStateChangeError(msg.partitionsMessage().getErrorsMap(), exchActions.stateChangeRequest());
                         }
 
-                        onDone(resTopVer);
+                        onDone(resTopVer, err);
                     }
                     else {
                         if (log.isDebugEnabled()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index a636464a9f3c0..b43827a07e9ed 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -109,6 +109,8 @@
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
@@ -2300,6 +2302,8 @@ else if (restore != null) {
                     updateState(part, restore.get1());
                 }
             }
+
+            grp.topology().afterStateRestored(grp.topology().lastTopologyChangeVersion());
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 5fbd7525b0867..7dc1bfaf15016 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -25,13 +25,16 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
+import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.cache.affinity.AffinityFunction;
+import org.apache.ignite.cache.affinity.AffinityFunctionContext;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -40,9 +43,16 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestDelayingCommunicationSpi;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -63,6 +73,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
     /** */
     private static final int NODE_COUNT = 4;
 
+    /** */
+    private static boolean delayRebalance = false;
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
@@ -98,6 +111,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
         if (client)
             cfg.setClientMode(true);
 
+        if (delayRebalance)
+            cfg.setCommunicationSpi(new DelayRebalanceCommunicationSpi());
+
         return cfg;
     }
 
@@ -646,6 +662,83 @@ public boolean apply() {
         }
     }
 
+    /**
+     * @throws Exception if failed.
+     */
+    public void testAffinityAssignmentChangedAfterRestart() throws Exception {
+        delayRebalance = false;
+
+        int parts = 32;
+
+        final List partMapping = new ArrayList<>();
+
+        for (int p = 0; p < parts; p++)
+            partMapping.add(p);
+
+        final AffinityFunction affFunc = new TestAffinityFunction(new RendezvousAffinityFunction(false, parts));
+
+        TestAffinityFunction.partsAffMapping = partMapping;
+
+        String cacheName = CACHE_NAME + 2;
+
+        startGrids(4);
+
+        IgniteEx ig = grid(0);
+
+        ig.cluster().active(true);
+
+        IgniteCache cache = ig.createCache(
+            new CacheConfiguration()
+                .setName(cacheName)
+                .setCacheMode(PARTITIONED)
+                .setBackups(1)
+                .setPartitionLossPolicy(READ_ONLY_SAFE)
+                .setReadFromBackup(true)
+                .setWriteSynchronizationMode(FULL_SYNC)
+                .setRebalanceDelay(-1)
+                .setAffinity(affFunc));
+
+        Map keyToConsId = new HashMap<>();
+
+        for (int k = 0; k < 1000; k++) {
+            cache.put(k, k);
+
+            keyToConsId.put(k, ig.affinity(cacheName).mapKeyToNode(k).consistentId().toString());
+        }
+
+        stopAllGrids();
+
+        Collections.shuffle(TestAffinityFunction.partsAffMapping, new Random(1));
+
+        delayRebalance = true;
+
+        startGrids(4);
+
+        ig = grid(0);
+
+        ig.active(true);
+
+        cache = ig.cache(cacheName);
+
+        GridDhtPartitionFullMap partMap = ig.cachex(cacheName).context().topology().partitionMap(false);
+
+        for (int i = 1; i < 4; i++) {
+            IgniteEx ig0 = grid(i);
+
+            for (int p = 0; p < 32; p++)
+                assertEqualsCollections(ig.affinity(cacheName).mapPartitionToPrimaryAndBackups(p), ig0.affinity(cacheName).mapPartitionToPrimaryAndBackups(p));
+        }
+
+        for (Map.Entry e : keyToConsId.entrySet()) {
+            int p = ig.affinity(cacheName).partition(e.getKey());
+
+            assertEquals("p=" + p, GridDhtPartitionState.OWNING, partMap.get(ig.affinity(cacheName).mapKeyToNode(e.getKey()).id()).get(p));
+        }
+
+        for (int k = 0; k < 1000; k++)
+            assertEquals("k=" + k, Integer.valueOf(k), cache.get(k));
+    }
+
     /** */
     private Collection baselineNodes(Collection clNodes) {
         Collection res = new ArrayList<>(clNodes.size());
@@ -708,4 +801,73 @@ private TestValue(int a) {
             return result;
         }
     }
+
+    /**
+     *
+     */
+    private static class TestAffinityFunction implements AffinityFunction {
+        /** */
+        private final AffinityFunction delegate;
+
+        /** */
+        private static List partsAffMapping;
+
+        /** */
+        public TestAffinityFunction(AffinityFunction delegate) {
+            this.delegate = delegate;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void reset() {
+            delegate.reset();;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int partitions() {
+            return delegate.partitions();
+        }
+
+        /** {@inheritDoc} */
+        @Override public int partition(Object key) {
+            return delegate.partition(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public List> assignPartitions(AffinityFunctionContext affCtx) {
+            List> res0 = delegate.assignPartitions(affCtx);
+
+            List> res = new ArrayList<>(res0.size());
+
+            for (int p = 0; p < res0.size(); p++)
+                res.add(p, null);
+
+            for (int p = 0; p < res0.size(); p++)
+                res.set(partsAffMapping.get(p), res0.get(p));
+
+            return res;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void removeNode(UUID nodeId) {
+            delegate.removeNode(nodeId);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class DelayRebalanceCommunicationSpi extends TestDelayingCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override protected boolean delayMessage(Message msg, GridIoMessage ioMsg) {
+            if (msg != null && (msg instanceof GridDhtPartitionDemandMessage || msg instanceof GridDhtPartitionSupplyMessage))
+                return true;
+
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected int delayMillis() {
+            return 1_000_000;
+        }
+    }
 }

From 175d0752f3a85ee920f3acb43e7e652b78389574 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko 
Date: Wed, 24 Jan 2018 18:02:09 +0300
Subject: [PATCH 45/65] IGNITE-6832 Properly handle IO errors while
 checkpointing - Fixes #3394.

Signed-off-by: Alexey Goncharuk 
---
 .../DataStorageConfiguration.java             |   6 +-
 .../ignite/internal/GridKernalContext.java    |  14 +
 .../internal/GridKernalContextImpl.java       |  13 +
 .../apache/ignite/internal/IgnitionEx.java    |  33 ++
 .../ignite/internal/NodeInvalidator.java      |  53 +++
 .../pagemem/store/IgnitePageStoreManager.java |   2 +-
 .../pagemem/wal/StorageException.java         |  16 +-
 .../GridCacheDatabaseSharedManager.java       |  12 +-
 .../cache/persistence/file/FilePageStore.java |  60 +--
 .../file/FilePageStoreManager.java            | 114 ++++--
 .../file/PersistentStorageIOException.java    |  47 +++
 .../wal/AbstractWalRecordsIterator.java       |   2 +-
 .../wal/FileWriteAheadLogManager.java         | 119 ++----
 .../reader/StandaloneGridKernalContext.java   |  10 +
 .../IgnitePdsDiskErrorsRecoveringTest.java    | 376 ++++++++++++++++++
 .../file/IgnitePdsThreadInterruptionTest.java | 143 +++++--
 .../db/wal/reader/MockWalIteratorFactory.java |   2 +-
 .../IgnitePdsWithIndexingCoreTestSuite.java   |   2 +
 18 files changed, 844 insertions(+), 180 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/PersistentStorageIOException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
index 30507fec5e025..8d91503f9c127 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
@@ -224,7 +224,7 @@ public class DataStorageConfiguration implements Serializable {
     /** Always write full pages. */
     private boolean alwaysWriteFullPages = DFLT_WAL_ALWAYS_WRITE_FULL_PAGES;
 
-    /** Factory to provide I/O interface for files */
+    /** Factory to provide I/O interface for data storage files */
     private FileIOFactory fileIOFactory =
         IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_ASYNC_FILE_IO_FACTORY, true) ?
             new AsyncFileIOFactory() : new RandomAccessFileIOFactory();
@@ -824,7 +824,7 @@ public DataStorageConfiguration setAlwaysWriteFullPages(boolean alwaysWriteFullP
 
     /**
      * Factory to provide implementation of FileIO interface
-     * which is used for any file read/write operations
+     * which is used for data storage files read/write operations
      *
      * @return File I/O factory
      */
@@ -834,7 +834,7 @@ public FileIOFactory getFileIOFactory() {
 
     /**
      * Sets factory to provide implementation of FileIO interface
-     * which is used for any file read/write operations
+     * which is used for data storage files read/write operations
      *
      * @param fileIOFactory File I/O factory
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index ce12b6166d95a..a260327dbc518 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -655,4 +655,18 @@ public interface GridKernalContext extends Iterable {
      * @return subscription processor to manage internal-only (strict node-local) subscriptions between components.
      */
     public GridInternalSubscriptionProcessor internalSubscriptionProcessor();
+
+    /**
+     * TODO: Should be replaced with proper implementation in https://issues.apache.org/jira/browse/IGNITE-6891
+     *
+     * @return {@code true} if node was invalidated, false in other case.
+     */
+    public boolean invalidated();
+
+    /**
+     * Invalidates node.
+     *
+     * TODO: Should be replaced with proper implementation in https://issues.apache.org/jira/browse/IGNITE-6891
+     */
+    public void invalidate();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 36c623120c413..9a315e754dc5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -385,6 +385,9 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     /** */
     private GridInternalSubscriptionProcessor internalSubscriptionProc;
 
+    /** Node invalidation flag. */
+    private volatile boolean invalidated;
+
     /**
      * No-arg constructor is required by externalization.
      */
@@ -1091,6 +1094,16 @@ void disconnected(boolean disconnected) {
         return pdsFolderRslvr;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean invalidated() {
+        return invalidated;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void invalidate() {
+        invalidated = true;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridKernalContextImpl.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 232476be9e531..ed31f00e6bda3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -37,8 +37,11 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Handler;
 import javax.management.JMException;
@@ -386,6 +389,36 @@ public static boolean stop(@Nullable String name, boolean cancel, boolean stopNo
         return false;
     }
 
+    /**
+     * Behavior of the method is the almost same as {@link IgnitionEx#stop(String, boolean, boolean)}.
+     * If node stopping process will not be finished within {@code timeoutMs} whole JVM will be killed.
+     *
+     * @param timeoutMs Timeout to wait graceful stopping.
+     */
+    public static boolean stop(@Nullable String name, boolean cancel, boolean stopNotStarted, long timeoutMs) {
+        final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+
+        // Schedule delayed node killing if graceful stopping will be not finished within timeout.
+        executor.schedule(new Runnable() {
+            @Override
+            public void run() {
+                if (state(name) == IgniteState.STARTED) {
+                    U.error(null, "Unable to gracefully stop node within timeout " + timeoutMs +
+                            " milliseconds. Killing node...");
+
+                    // We are not able to kill only one grid so whole JVM will be stopped.
+                    System.exit(Ignition.KILL_EXIT_CODE);
+                }
+            }
+        }, timeoutMs, TimeUnit.MILLISECONDS);
+
+        boolean success = stop(name, cancel, stopNotStarted);
+
+        executor.shutdownNow();
+
+        return success;
+    }
+
     /**
      * Stops all started grids. If {@code cancel} flag is set to {@code true} then
      * all jobs currently executing on local node will be interrupted.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java b/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java
new file mode 100644
index 0000000000000..b19ec087c7075
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal;
+
+import org.apache.ignite.IgniteLogger;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Temporary functionality to invalidate and stop the node.
+ * TODO: Should be replaced on proper implementation in https://issues.apache.org/jira/browse/IGNITE-6891
+ */
+public class NodeInvalidator {
+    public static NodeInvalidator INSTANCE = new NodeInvalidator();
+
+    private static final long STOP_TIMEOUT_MS = 60 * 1000;
+
+    private NodeInvalidator() {
+        // Empty
+    }
+
+    public void invalidate(@NotNull GridKernalContext ctx, @NotNull Throwable error) {
+        if (ctx.invalidated())
+            return;
+
+        ctx.invalidate();
+
+        final String gridName = ctx.igniteInstanceName();
+        final IgniteLogger logger = ctx.log(getClass());
+
+        logger.error("Critical error with " + gridName + " is happened. " +
+                "All further operations will be failed and local node will be stopped.", error);
+
+        new Thread("node-stopper") {
+            @Override public void run() {
+                IgnitionEx.stop(gridName, true, true, STOP_TIMEOUT_MS);
+            }
+        }.start();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 2707a5e0db2c6..1b46bf990c540 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -39,7 +39,7 @@ public interface IgnitePageStoreManager extends GridCacheSharedManager, IgniteCh
     /**
      * Invoked after checkpoint recover is finished.
      */
-    public void finishRecover();
+    public void finishRecover() throws IgniteCheckedException;
 
     /**
      * Callback called when a cache is starting.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java
index e38e5f237158c..3aa50c0dffb05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java
@@ -17,11 +17,12 @@
 
 package org.apache.ignite.internal.pagemem.wal;
 
+import java.io.IOException;
 import org.apache.ignite.IgniteCheckedException;
-import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.NotNull;
 
 /**
- *
+ * Exception is needed to distinguish WAL manager critical I/O errors.
  */
 public class StorageException extends IgniteCheckedException {
     /** */
@@ -31,14 +32,21 @@ public class StorageException extends IgniteCheckedException {
      * @param msg Error message.
      * @param cause Error cause.
      */
-    public StorageException(String msg, @Nullable Throwable cause) {
+    public StorageException(String msg, @NotNull IOException cause) {
         super(msg, cause);
     }
 
     /**
      * @param e Cause exception.
      */
-    public StorageException(Exception e) {
+    public StorageException(IOException e) {
         super(e);
     }
+
+    /**
+     * @param msg Error message
+     */
+    public StorageException(String msg) {
+        super(msg);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index b43827a07e9ed..f3b11ccfee0d2 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -76,6 +76,7 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.NodeInvalidator;
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
@@ -2805,7 +2806,16 @@ private void doCheckpoint() {
                         }
 
                         // Wait and check for errors.
-                        doneWriteFut.get();
+                        try {
+                            doneWriteFut.get();
+                        } catch (IgniteCheckedException e) {
+                            chp.progress.cpFinishFut.onDone(e);
+
+                            // In case of writing error node should be invalidated and stopped.
+                            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+
+                            return;
+                        }
 
                         // Must re-check shutdown flag here because threads may have skipped some pages.
                         // If so, we should not put finish checkpoint mark.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 1153c88cd77fd..b5f412e279769 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -25,7 +25,6 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -154,27 +153,28 @@ public ByteBuffer header(byte type, int pageSize) {
     }
 
     /**
+     * Initializes header and writes it into the file store.
      *
+     * @return Next available position in the file to store a data.
+     * @throws IOException If initialization is failed.
      */
-    private long initFile() {
-        try {
-            ByteBuffer hdr = header(type, dbCfg.getPageSize());
+    private long initFile() throws IOException {
+        ByteBuffer hdr = header(type, dbCfg.getPageSize());
 
-            while (hdr.remaining() > 0)
-                fileIO.write(hdr);
-        }
-        catch (IOException e) {
-            throw new IgniteException("Check file failed.", e);
-        }
+        while (hdr.remaining() > 0)
+            fileIO.write(hdr);
 
         //there is 'super' page in every file
         return headerSize() + dbCfg.getPageSize();
     }
 
     /**
+     * Checks that file store has correct header and size.
      *
+     * @return Next available position in the file to store a data.
+     * @throws PersistentStorageIOException If check is failed.
      */
-    private long checkFile() throws IgniteCheckedException {
+    private long checkFile() throws PersistentStorageIOException {
         try {
             ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN);
 
@@ -186,28 +186,28 @@ private long checkFile() throws IgniteCheckedException {
             long signature = hdr.getLong();
 
             if (SIGNATURE != signature)
-                throw new IgniteCheckedException("Failed to verify store file (invalid file signature)" +
+                throw new IOException("Failed to verify store file (invalid file signature)" +
                     " [expectedSignature=" + U.hexLong(SIGNATURE) +
                     ", actualSignature=" + U.hexLong(signature) + ']');
 
             int ver = hdr.getInt();
 
             if (version() != ver)
-                throw new IgniteCheckedException("Failed to verify store file (invalid file version)" +
+                throw new IOException("Failed to verify store file (invalid file version)" +
                     " [expectedVersion=" + version() +
                     ", fileVersion=" + ver + "]");
 
             byte type = hdr.get();
 
             if (this.type != type)
-                throw new IgniteCheckedException("Failed to verify store file (invalid file type)" +
+                throw new IOException("Failed to verify store file (invalid file type)" +
                     " [expectedFileType=" + this.type +
                     ", actualFileType=" + type + "]");
 
             int pageSize = hdr.getInt();
 
             if (dbCfg.getPageSize() != pageSize)
-                throw new IgniteCheckedException("Failed to verify store file (invalid page size)" +
+                throw new IOException("Failed to verify store file (invalid page size)" +
                     " [expectedPageSize=" + dbCfg.getPageSize() +
                     ", filePageSize=" + pageSize + "]");
 
@@ -217,22 +217,22 @@ private long checkFile() throws IgniteCheckedException {
                 fileSize = pageSize + headerSize();
 
             if ((fileSize - headerSize()) % pageSize != 0)
-                throw new IgniteCheckedException("Failed to verify store file (invalid file size)" +
+                throw new IOException("Failed to verify store file (invalid file size)" +
                     " [fileSize=" + U.hexLong(fileSize) +
                     ", pageSize=" + U.hexLong(pageSize) + ']');
 
             return fileSize;
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("File check failed", e);
+            throw new PersistentStorageIOException("File check failed", e);
         }
     }
 
     /**
      * @param cleanFile {@code True} to delete file.
-     * @throws IgniteCheckedException If failed.
+     * @throws PersistentStorageIOException If failed.
      */
-    public void stop(boolean cleanFile) throws IgniteCheckedException {
+    public void stop(boolean cleanFile) throws PersistentStorageIOException {
         lock.writeLock().lock();
 
         try {
@@ -247,7 +247,7 @@ public void stop(boolean cleanFile) throws IgniteCheckedException {
                 cfgFile.delete();
         }
         catch (IOException e) {
-            throw new IgniteCheckedException(e);
+            throw new PersistentStorageIOException(e);
         }
         finally {
             lock.writeLock().unlock();
@@ -257,7 +257,7 @@ public void stop(boolean cleanFile) throws IgniteCheckedException {
     /**
      *
      */
-    public void truncate(int tag) throws IgniteCheckedException {
+    public void truncate(int tag) throws PersistentStorageIOException {
         lock.writeLock().lock();
 
         try {
@@ -277,7 +277,7 @@ public void truncate(int tag) throws IgniteCheckedException {
             allocatedTracker.updateTotalAllocatedPages(delta / pageSize);
         }
         catch (IOException e) {
-            throw new IgniteCheckedException(e);
+            throw new PersistentStorageIOException(e);
         }
         finally {
             lock.writeLock().unlock();
@@ -301,7 +301,7 @@ public void beginRecover() {
     /**
      *
      */
-    public void finishRecover() {
+    public void finishRecover() throws PersistentStorageIOException {
         lock.writeLock().lock();
 
         try {
@@ -320,7 +320,7 @@ public void finishRecover() {
             recover = false;
         }
         catch (IOException e) {
-            throw new RuntimeException(e);
+            throw new PersistentStorageIOException("Unable to finish recover", e);
         }
         finally {
             lock.writeLock().unlock();
@@ -378,7 +378,7 @@ public void finishRecover() {
                 PageIO.setCrc(pageBuf, savedCrc32);
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Read error", e);
+            throw new PersistentStorageIOException("Read error", e);
         }
     }
 
@@ -407,7 +407,7 @@ public void finishRecover() {
             while (len > 0);
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Read error", e);
+            throw new PersistentStorageIOException("Read error", e);
         }
     }
 
@@ -442,7 +442,9 @@ private void init() throws IgniteCheckedException {
                         inited = true;
                     }
                     catch (IOException e) {
-                        throw err = new IgniteCheckedException("Can't open file: " + cfgFile.getName(), e);
+                        err = new PersistentStorageIOException("Could not initialize file: " + cfgFile.getName(), e);
+
+                        throw err;
                     }
                     finally {
                         if (err != null && fileIO != null)
@@ -509,7 +511,7 @@ private void init() throws IgniteCheckedException {
             PageIO.setCrc(pageBuf, 0);
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to write the page to the file store [pageId=" + pageId +
+            throw new PersistentStorageIOException("Failed to write the page to the file store [pageId=" + pageId +
                 ", file=" + cfgFile.getAbsolutePath() + ']', e);
         }
         finally {
@@ -547,7 +549,7 @@ private static int calcCrc32(ByteBuffer pageBuf, int pageSize) {
             fileIO.force();
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Sync error", e);
+            throw new PersistentStorageIOException("Sync error", e);
         }
         finally {
             lock.writeLock().unlock();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 26e46b214848f..9549269d9672e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.NodeInvalidator;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
@@ -51,6 +52,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -190,12 +192,19 @@ public FilePageStoreManager(GridKernalContext ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public void finishRecover() {
-        for (CacheStoreHolder holder : idxCacheStores.values()) {
-            holder.idxStore.finishRecover();
+    @Override public void finishRecover() throws IgniteCheckedException {
+        try {
+            for (CacheStoreHolder holder : idxCacheStores.values()) {
+                holder.idxStore.finishRecover();
 
-            for (FilePageStore partStore : holder.partStores)
-                partStore.finishRecover();
+                for (FilePageStore partStore : holder.partStores)
+                    partStore.finishRecover();
+            }
+        }
+        catch (PersistentStorageIOException e) {
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+
+            throw e;
         }
     }
 
@@ -309,7 +318,14 @@ public FilePageStoreManager(GridKernalContext ctx) {
     public void read(int cacheId, long pageId, ByteBuffer pageBuf, boolean keepCrc) throws IgniteCheckedException {
         PageStore store = getStore(cacheId, PageIdUtils.partId(pageId));
 
-        store.read(pageId, pageBuf, keepCrc);
+        try {
+            store.read(pageId, pageBuf, keepCrc);
+        }
+        catch (PersistentStorageIOException e) {
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+
+            throw e;
+        }
     }
 
     /** {@inheritDoc} */
@@ -323,7 +339,14 @@ public void read(int cacheId, long pageId, ByteBuffer pageBuf, boolean keepCrc)
     @Override public void readHeader(int grpId, int partId, ByteBuffer buf) throws IgniteCheckedException {
         PageStore store = getStore(grpId, partId);
 
-        store.readHeader(buf);
+        try {
+            store.readHeader(buf);
+        }
+        catch (PersistentStorageIOException e) {
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+
+            throw e;
+        }
     }
 
     /** {@inheritDoc} */
@@ -352,7 +375,14 @@ public PageStore writeInternal(int cacheId, long pageId, ByteBuffer pageBuf, int
 
         PageStore store = getStore(cacheId, partId);
 
-        store.write(pageId, pageBuf, tag, calculateCrc);
+        try {
+            store.write(pageId, pageBuf, tag, calculateCrc);
+        }
+        catch (PersistentStorageIOException e) {
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+
+            throw e;
+        }
 
         return store;
     }
@@ -396,35 +426,42 @@ private CacheStoreHolder initDir(File cacheWorkDir,
         int grpId,
         int partitions,
         AllocatedPageTracker allocatedTracker) throws IgniteCheckedException {
-        boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir);
+        try {
+            boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir);
 
-        File idxFile = new File(cacheWorkDir, INDEX_FILE_NAME);
+            File idxFile = new File(cacheWorkDir, INDEX_FILE_NAME);
 
-        if (dirExisted && !idxFile.exists())
-            grpsWithoutIdx.add(grpId);
+            if (dirExisted && !idxFile.exists())
+                grpsWithoutIdx.add(grpId);
 
-        FilePageStoreFactory pageStoreFactory = new FileVersionCheckingFactory(
-            pageStoreFileIoFactory, pageStoreV1FileIoFactory, igniteCfg.getDataStorageConfiguration());
+            FilePageStoreFactory pageStoreFactory = new FileVersionCheckingFactory(
+                pageStoreFileIoFactory, pageStoreV1FileIoFactory, igniteCfg.getDataStorageConfiguration());
 
-        FilePageStore idxStore =
+            FilePageStore idxStore =
             pageStoreFactory.createPageStore(
                 PageMemory.FLAG_IDX,
                 idxFile,
                 allocatedTracker);
 
-        FilePageStore[] partStores = new FilePageStore[partitions];
+            FilePageStore[] partStores = new FilePageStore[partitions];
 
-        for (int partId = 0; partId < partStores.length; partId++) {
-            FilePageStore partStore =
-                pageStoreFactory.createPageStore(
-                    PageMemory.FLAG_DATA,
-                    getPartitionFile(cacheWorkDir, partId),
-                    allocatedTracker);
+            for (int partId = 0; partId < partStores.length; partId++) {
+                FilePageStore partStore =
+                    pageStoreFactory.createPageStore(
+                        PageMemory.FLAG_DATA,
+                        getPartitionFile(cacheWorkDir, partId),
+                        allocatedTracker);
 
-            partStores[partId] = partStore;
+                    partStores[partId] = partStore;
+                }
+
+            return new CacheStoreHolder(idxStore, partStores);
         }
+        catch (PersistentStorageIOException e) {
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
 
-        return new CacheStoreHolder(idxStore, partStores);
+            throw e;
+        }
     }
 
     /**
@@ -509,12 +546,26 @@ else if (lockF.exists()) {
 
     /** {@inheritDoc} */
     @Override public void sync(int grpId, int partId) throws IgniteCheckedException {
-        getStore(grpId, partId).sync();
+        try {
+            getStore(grpId, partId).sync();
+        }
+        catch (PersistentStorageIOException e) {
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+
+            throw e;
+        }
     }
 
     /** {@inheritDoc} */
     @Override public void ensure(int grpId, int partId) throws IgniteCheckedException {
-        getStore(grpId, partId).ensure();
+        try {
+            getStore(grpId, partId).ensure();
+        }
+        catch (PersistentStorageIOException e) {
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+
+            throw e;
+        }
     }
 
     /** {@inheritDoc} */
@@ -523,9 +574,16 @@ else if (lockF.exists()) {
 
         PageStore store = getStore(grpId, partId);
 
-        long pageIdx = store.allocatePage();
+        try {
+            long pageIdx = store.allocatePage();
+
+            return PageIdUtils.pageId(partId, flags, (int)pageIdx);
+        }
+        catch (PersistentStorageIOException e) {
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
 
-        return PageIdUtils.pageId(partId, flags, (int)pageIdx);
+            throw e;
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/PersistentStorageIOException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/PersistentStorageIOException.java
new file mode 100644
index 0000000000000..7b3c30371b9dd
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/PersistentStorageIOException.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.IOException;
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Exception is needed to distinguish persistent storage I/O errors.
+ */
+public class PersistentStorageIOException extends IgniteCheckedException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * Create an instance of exception.
+     *
+     * @param cause Error cause.
+     */
+    public PersistentStorageIOException(IOException cause) {
+        super(cause);
+    }
+
+    /**
+     * Create an instance of exception.
+     *
+     * @param msg Error message.
+     * @param cause Error cause.
+     */
+    public PersistentStorageIOException(String msg, IOException cause) {
+        super(msg, cause);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index 195d1813f695f..bf59c8166c8e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -304,7 +304,7 @@ protected FileWriteAheadLogManager.ReadFileHandle initReadHandle(
                 }
 
                 return new FileWriteAheadLogManager.ReadFileHandle(
-                    fileIO, desc.idx, sharedCtx.igniteInstanceName(), serializerFactory.createSerializer(serVer), in);
+                    fileIO, desc.idx, serializerFactory.createSerializer(serVer), in);
             }
             catch (SegmentEofException | EOFException ignore) {
                 try {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 7b3e93871d2c5..73751c62f51c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -69,7 +69,7 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.NodeInvalidator;
 import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.StorageException;
@@ -115,8 +115,8 @@
 import static java.nio.file.StandardOpenOption.CREATE;
 import static java.nio.file.StandardOpenOption.READ;
 import static java.nio.file.StandardOpenOption.WRITE;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
 import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
 import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.SWITCH_SEGMENT_RECORD;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBuffer.BufferMode.DIRECT;
@@ -212,7 +212,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     private static final int BUF_SIZE = 1024 * 1024;
 
     /** Use mapped byte buffer. */
-    private static boolean mmap = IgniteSystemProperties.getBoolean(IGNITE_WAL_MMAP, true);
+    private final boolean mmap = IgniteSystemProperties.getBoolean(IGNITE_WAL_MMAP, true);
 
     /** {@link FileWriteHandle#written} atomic field updater. */
     private static final AtomicLongFieldUpdater WRITTEN_UPD =
@@ -293,9 +293,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     /** Current log segment handle */
     private volatile FileWriteHandle currHnd;
 
-    /** Environment failure. */
-    private volatile Throwable envFailed;
-
     /**
      * Positive (non-0) value indicates WAL can be archived even if not complete

      * See {@link DataStorageConfiguration#setWalAutoArchiveAfterInactivity(long)}

@@ -666,7 +663,8 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
         }
         catch (IgniteCheckedException e) {
             U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e);
-            handle.invalidateEnvironment(e);
+
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
         }
     }
 
@@ -714,7 +712,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
             else
                 currWrHandle = rollOver(currWrHandle);
 
-            checkEnvironment();
+            checkNode();
 
             if (isStopping())
                 throw new IgniteCheckedException("Stopping.");
@@ -1107,7 +1105,6 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
                 FileWriteHandle hnd = new FileWriteHandle(
                     fileIO,
                     absIdx,
-                    cctx.igniteInstanceName(),
                     off + len,
                     true,
                     ser,
@@ -1161,17 +1158,9 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageE
                         lsnr.apply(fileIO);
 
                     if (mmap) {
-                        try {
-                            MappedByteBuffer buf = fileIO.map((int)maxWalSegmentSize);
+                        MappedByteBuffer buf = fileIO.map((int)maxWalSegmentSize);
 
-                            rbuf = new SegmentedRingByteBuffer(buf, metrics);
-                        }
-                        catch (IOException e) {
-                            if (e instanceof ClosedByInterruptException)
-                                throw e;
-                            else
-                                throw new IgniteCheckedException(e);
-                        }
+                        rbuf = new SegmentedRingByteBuffer(buf, metrics);
                     }
                     else
                         rbuf = cur.buf.reset();
@@ -1179,7 +1168,6 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageE
                     hnd = new FileWriteHandle(
                         fileIO,
                         cur.idx + 1,
-                        cctx.igniteInstanceName(),
                         0,
                         false,
                         serializer,
@@ -1221,7 +1209,11 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageE
             return hnd;
         }
         catch (IOException e) {
-            throw new StorageException(e);
+            StorageException se = new StorageException("Unable to initialize WAL segment", e);
+
+            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+
+            throw se;
         }
     }
 
@@ -1355,12 +1347,12 @@ public static FileDescriptor[] scan(File[] allFiles) {
     }
 
     /**
-     * @throws StorageException If environment is no longer valid and we missed a WAL write.
+     * @throws StorageException If node is no longer valid and we missed a WAL operation.
      */
-    private void checkEnvironment() throws StorageException {
-        if (envFailed != null)
-            throw new StorageException("Failed to flush WAL buffer (environment was invalidated by a " +
-                "previous error)", envFailed);
+    private void checkNode() throws StorageException {
+        if (cctx.kernalContext().invalidated())
+            throw new StorageException("Failed to perform WAL operation (environment was invalidated by a " +
+                    "previous error)");
     }
 
     /**
@@ -1903,10 +1895,7 @@ private void deleteObsoleteRawSegments() {
                 catch (IgniteCheckedException | IOException e) {
                     U.error(log, "Unexpected error during WAL compression", e);
 
-                    FileWriteHandle handle = currentHandle();
-
-                    if (handle != null)
-                        handle.invalidateEnvironment(e);
+                    NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
                 }
                 catch (InterruptedException ignore) {
                     Thread.currentThread().interrupt();
@@ -2058,10 +2047,7 @@ private class FileDecompressor extends Thread {
                 catch (IOException e) {
                     U.error(log, "Unexpected error during WAL decompression", e);
 
-                    FileWriteHandle handle = currentHandle();
-
-                    if (handle != null)
-                        handle.invalidateEnvironment(e);
+                    NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
                 }
             }
         }
@@ -2339,17 +2325,13 @@ private abstract static class FileHandle {
         /** Absolute WAL segment file index (incremental counter) */
         protected final long idx;
 
-        /** */
-        protected String gridName;
-
         /**
          * @param fileIO I/O interface for read/write operations of FileHandle.
          * @param idx Absolute WAL segment file index (incremental counter).
          */
-        private FileHandle(FileIO fileIO, long idx, String gridName) {
+        private FileHandle(FileIO fileIO, long idx) {
             this.fileIO = fileIO;
             this.idx = idx;
-            this.gridName = gridName;
         }
     }
 
@@ -2378,11 +2360,10 @@ public static class ReadFileHandle extends FileHandle {
         ReadFileHandle(
             FileIO fileIO,
             long idx,
-            String gridName,
             RecordSerializer ser,
             FileInput in
         ) {
-            super(fileIO, idx, gridName);
+            super(fileIO, idx);
 
             this.ser = ser;
             this.in = in;
@@ -2427,9 +2408,6 @@ private class FileWriteHandle extends FileHandle {
         /** */
         private final Lock lock = new ReentrantLock();
 
-        /** Condition activated each time writeBuffer() completes. Used to wait previously flushed write to complete */
-        private final Condition writeComplete = lock.newCondition();
-
         /** Condition for timed wait of several threads, see {@link DataStorageConfiguration#getWalFsyncDelayNanos()} */
         private final Condition fsync = lock.newCondition();
 
@@ -2454,13 +2432,12 @@ private class FileWriteHandle extends FileHandle {
         private FileWriteHandle(
             FileIO fileIO,
             long idx,
-            String gridName,
             long pos,
             boolean resume,
             RecordSerializer serializer,
             SegmentedRingByteBuffer buf
         ) throws IOException {
-            super(fileIO, idx, gridName);
+            super(fileIO, idx);
 
             assert serializer != null;
 
@@ -2499,7 +2476,7 @@ public void writeHeader() {
             assert rec.size() > 0 : rec;
 
             for (;;) {
-                checkEnvironment();
+                checkNode();
 
                 SegmentedRingByteBuffer.WriteSegment seg;
 
@@ -2818,7 +2795,7 @@ private void signalNextAvailable() {
             lock.lock();
 
             try {
-                assert envFailed != null || written == lastFsyncPos || mode != WALMode.DEFAULT :
+                assert cctx.kernalContext().invalidated() || written == lastFsyncPos || mode != WALMode.DEFAULT :
                     "fsync [written=" + written + ", lastFsync=" + lastFsyncPos + ", idx=" + idx + ']';
 
                 fileIO = null;
@@ -2845,40 +2822,6 @@ private void awaitNext() {
             }
         }
 
-        /**
-         * @param e Exception to set as a cause for all further operations.
-         */
-        private void invalidateEnvironment(Throwable e) {
-            lock.lock();
-
-            try {
-                invalidateEnvironmentLocked(e);
-            }
-            finally {
-                writeComplete.signalAll();
-
-                lock.unlock();
-            }
-        }
-
-        /**
-         * @param e Exception to set as a cause for all further operations.
-         */
-        private void invalidateEnvironmentLocked(Throwable e) {
-            if (envFailed == null) {
-                envFailed = e;
-
-                U.error(log, "IO error encountered while running WAL flush. All further operations " +
-                    " will be failed and local node will be stopped.", e);
-
-                new Thread() {
-                    @Override public void run() {
-                        IgnitionEx.stop(gridName, true, true);
-                    }
-                }.start();
-            }
-        }
-
         /**
          * @return Safely reads current position of the file channel as String. Will return "null" if channel is null.
          */
@@ -3311,7 +3254,7 @@ void flushBuffer(long expPos) throws StorageException, IgniteCheckedException {
             Throwable err = walWriter.err;
 
             if (err != null)
-                currentHandle().invalidateEnvironment(err);
+                NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), err);
 
             if (expPos == UNCONDITIONAL_FLUSH)
                 expPos = (currentHandle().buf.tail());
@@ -3350,7 +3293,7 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni
 
             assert hdl.fileIO != null : "Writing to a closed segment.";
 
-            checkEnvironment();
+            checkNode();
 
             long lastLogged = U.currentTimeMillis();
 
@@ -3375,7 +3318,7 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni
                     lastLogged = now;
                 }
 
-                checkEnvironment();
+                checkNode();
             }
 
             // Do the write.
@@ -3398,9 +3341,11 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni
                 assert hdl.written == hdl.fileIO.position();
             }
             catch (IOException e) {
-                hdl.invalidateEnvironmentLocked(e);
+                StorageException se = new StorageException("Unable to write", e);
+
+                NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
 
-                throw new StorageException(e);
+                throw se;
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index fa3f7f306ccbb..908d5b7771544 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -624,6 +624,16 @@ private IgniteConfiguration prepareIgniteConfiguration() {
         };
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean invalidated() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void invalidate() {
+
+    }
+
     /** {@inheritDoc} */
     @NotNull @Override public Iterator iterator() {
         return null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
new file mode 100644
index 0000000000000..35115515ddd07
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.GridKernalState;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ * Tests node recovering after disk errors during interaction with persistent storage.
+ */
+public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest {
+    /** */
+    private static final int PAGE_SIZE = DataStorageConfiguration.DFLT_PAGE_SIZE;
+
+    /** */
+    private static final int WAL_SEGMENT_SIZE = 1024 * PAGE_SIZE;
+
+    /** */
+    private static final long DFLT_DISK_SPACE_BYTES = Long.MAX_VALUE;
+
+    /** */
+    private static final long STOP_TIMEOUT_MS = 30 * 1000;
+
+    /** */
+    private static final String CACHE_NAME = "cache";
+
+    /** */
+    private boolean failPageStoreDiskOperations = false;
+
+    /** */
+    private long diskSpaceBytes = DFLT_DISK_SPACE_BYTES;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+
+        failPageStoreDiskOperations = false;
+        diskSpaceBytes = DFLT_DISK_SPACE_BYTES;
+        System.clearProperty(IGNITE_WAL_MMAP);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                        new DataRegionConfiguration().setMaxSize(100 * 1024 * 1024).setPersistenceEnabled(true))
+                .setWalMode(WALMode.LOG_ONLY)
+                .setWalCompactionEnabled(false)
+                .setWalSegmentSize(WAL_SEGMENT_SIZE)
+                .setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
+
+        if (failPageStoreDiskOperations)
+            dsCfg.setFileIOFactory(new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), diskSpaceBytes));
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        CacheConfiguration cacheCfg = new CacheConfiguration(CACHE_NAME)
+            .setRebalanceMode(CacheRebalanceMode.NONE)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setAffinity(new RendezvousAffinityFunction(false, 1));
+
+        cfg.setCacheConfiguration(cacheCfg);
+
+        return cfg;
+    }
+
+    /**
+     *
+     */
+    public void testRecoveringOnCacheInitError() throws Exception {
+        failPageStoreDiskOperations = true;
+
+        // Two pages is enough to initialize MetaStorage.
+        diskSpaceBytes = 2 * PAGE_SIZE;
+
+        final IgniteEx grid = startGrid(0);
+
+        boolean failed = false;
+        try {
+            grid.active(true);
+        } catch (Exception expected) {
+            log.warning("Expected cache error", expected);
+
+            failed = true;
+        }
+
+        Assert.assertTrue("Cache initialization must failed", failed);
+
+        // Grid should be automatically stopped after checkpoint fail.
+        awaitStop(grid);
+
+        // Grid should be successfully recovered after stopping.
+        failPageStoreDiskOperations = false;
+
+        IgniteEx recoveredGrid = startGrid(0);
+        recoveredGrid.active(true);
+    }
+
+    /**
+     *
+     */
+    public void testRecoveringOnCheckpointWritingError() throws Exception {
+        failPageStoreDiskOperations = true;
+        diskSpaceBytes = 1024 * PAGE_SIZE;
+
+        final IgniteEx grid = startGrid(0);
+        grid.active(true);
+
+        for (int i = 0; i < 1000; i++) {
+            byte payload = (byte) i;
+            byte[] data = new byte[2048];
+            Arrays.fill(data, payload);
+
+            grid.cache(CACHE_NAME).put(i, data);
+        }
+
+        boolean checkpointFailed = false;
+        try {
+            forceCheckpoint();
+        }
+        catch (IgniteCheckedException e) {
+            for (Throwable t : e.getSuppressed())
+                if (t.getCause() != null && t.getCause().getMessage().equals("Not enough space!"))
+                    checkpointFailed = true;
+        }
+
+        Assert.assertTrue("Checkpoint must be failed by IOException (Not enough space!)", checkpointFailed);
+
+        // Grid should be automatically stopped after checkpoint fail.
+        awaitStop(grid);
+
+        // Grid should be successfully recovered after stopping.
+        failPageStoreDiskOperations = false;
+
+        IgniteEx recoveredGrid = startGrid(0);
+        recoveredGrid.active(true);
+
+        for (int i = 0; i < 1000; i++) {
+            byte payload = (byte) i;
+            byte[] data = new byte[2048];
+            Arrays.fill(data, payload);
+
+            byte[] actualData = (byte[]) recoveredGrid.cache(CACHE_NAME).get(i);
+            Assert.assertArrayEquals(data, actualData);
+        }
+    }
+
+    /**
+     *
+     */
+    public void testRecoveringOnWALErrorWithMmap() throws Exception {
+        diskSpaceBytes = WAL_SEGMENT_SIZE;
+        System.setProperty(IGNITE_WAL_MMAP, "true");
+        emulateRecoveringOnWALWritingError();
+    }
+
+    /**
+     *
+     */
+    public void testRecoveringOnWALErrorWithoutMmap() throws Exception {
+        diskSpaceBytes = 2 * WAL_SEGMENT_SIZE;
+        System.setProperty(IGNITE_WAL_MMAP, "false");
+        emulateRecoveringOnWALWritingError();
+    }
+
+    /**
+     *
+     */
+    private void emulateRecoveringOnWALWritingError() throws Exception {
+        final IgniteEx grid = startGrid(0);
+
+        FileWriteAheadLogManager wal = (FileWriteAheadLogManager)grid.context().cache().context().wal();
+        wal.setFileIOFactory(new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), diskSpaceBytes));
+
+        grid.active(true);
+
+        int failedPosition = -1;
+
+        for (int i = 0; i < 1000; i++) {
+            byte payload = (byte) i;
+            byte[] data = new byte[2048];
+            Arrays.fill(data, payload);
+
+            try {
+                grid.cache(CACHE_NAME).put(i, data);
+            }
+            catch (Exception e) {
+                failedPosition = i;
+
+                break;
+            }
+        }
+
+        // We must be able to put something into cache before fail.
+        Assert.assertTrue(failedPosition > 0);
+
+        // Grid should be automatically stopped after WAL fail.
+        awaitStop(grid);
+
+        // Grid should be successfully recovered after stopping.
+        IgniteEx recoveredGrid = startGrid(0);
+        recoveredGrid.active(true);
+
+        for (int i = 0; i < failedPosition; i++) {
+            byte payload = (byte) i;
+            byte[] data = new byte[2048];
+            Arrays.fill(data, payload);
+
+            byte[] actualData = (byte[]) recoveredGrid.cache(CACHE_NAME).get(i);
+            Assert.assertArrayEquals(data, actualData);
+        }
+    }
+
+    /**
+     *
+     */
+    private void awaitStop(final IgniteEx grid) throws IgniteInterruptedCheckedException {
+        GridTestUtils.waitForCondition(() -> grid.context().gateway().getState() == GridKernalState.STOPPED, STOP_TIMEOUT_MS);
+    }
+
+    /**
+     *
+     */
+    private void forceCheckpoint() throws Exception {
+        for (Ignite ignite : G.allGrids()) {
+            if (ignite.cluster().localNode().isClient())
+                continue;
+
+            GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+                    .cache().context().database();
+
+            dbMgr.waitForCheckpoint("test");
+        }
+    }
+
+    /**
+     *
+     */
+    private static class LimitedSizeFileIO extends FileIODecorator {
+        /** */
+        private final AtomicLong availableSpaceBytes;
+
+        /**
+         * @param delegate File I/O delegate.
+         * @param availableSpaceBytes Shared counter which indicates the number of available bytes in a FS.
+         */
+        public LimitedSizeFileIO(FileIO delegate, AtomicLong availableSpaceBytes) {
+            super(delegate);
+            this.availableSpaceBytes = availableSpaceBytes;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int write(ByteBuffer srcBuf) throws IOException {
+            int written = super.write(srcBuf);
+            availableSpaceBytes.addAndGet(-written);
+            if (availableSpaceBytes.get() < 0)
+                throw new IOException("Not enough space!");
+            return written;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+            int written = super.write(srcBuf, position);
+            availableSpaceBytes.addAndGet(-written);
+            if (availableSpaceBytes.get() < 0)
+                throw new IOException("Not enough space!");
+            return written;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(byte[] buf, int off, int len) throws IOException {
+            super.write(buf, off, len);
+            availableSpaceBytes.addAndGet(-len);
+            if (availableSpaceBytes.get() < 0)
+                throw new IOException("Not enough space!");
+        }
+
+        /** {@inheritDoc} */
+        @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
+            availableSpaceBytes.addAndGet(-maxWalSegmentSize);
+            if (availableSpaceBytes.get() < 0)
+                throw new IOException("Not enough space!");
+            return super.map(maxWalSegmentSize);
+        }
+    }
+
+    private static class LimitedSizeFileIOFactory implements FileIOFactory {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final FileIOFactory delegate;
+
+        /** */
+        private final AtomicLong availableSpaceBytes;
+
+        /**
+         * @param delegate File I/O factory delegate.
+         * @param fsSpaceBytes Number of available bytes in FS.
+         */
+        private LimitedSizeFileIOFactory(FileIOFactory delegate, long fsSpaceBytes) {
+            this.delegate = delegate;
+            this.availableSpaceBytes = new AtomicLong(fsSpaceBytes);
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            return new LimitedSizeFileIO(delegate.create(file), availableSpaceBytes);
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            return new LimitedSizeFileIO(delegate.create(file, modes), availableSpaceBytes);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsThreadInterruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsThreadInterruptionTest.java
index 6955e32ffd662..6cd3c1f77e77b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsThreadInterruptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsThreadInterruptionTest.java
@@ -21,17 +21,18 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataPageEvictionMode;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jsr166.ThreadLocalRandom8;
 
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
-
 /**
  * Test what interruptions of writing threads do not affect PDS.
  */
@@ -40,12 +41,12 @@ public class IgnitePdsThreadInterruptionTest extends GridCommonAbstractTest {
     private static final int PAGE_SIZE = 1 << 12; // 4096
 
     /** */
-    public static final int THREADS_CNT = 10;
+    public static final int THREADS_CNT = 100;
 
     /**
      * Cache name.
      */
-    private final String cacheName = "cache";
+    private final String CACHE_NAME = "cache";
 
     /** */
     private volatile boolean stop = false;
@@ -54,37 +55,49 @@ public class IgnitePdsThreadInterruptionTest extends GridCommonAbstractTest {
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         final IgniteConfiguration cfg = super.getConfiguration(gridName);
 
-        cfg.setDataStorageConfiguration(memoryConfiguration());
+        cfg.setDataStorageConfiguration(storageConfiguration());
+
+        CacheConfiguration ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+        RendezvousAffinityFunction affinityFunction = new RendezvousAffinityFunction();
+        affinityFunction.setPartitions(1);
 
-        cfg.setCacheConfiguration(new CacheConfiguration<>(cacheName));
+        ccfg.setAffinity(affinityFunction);
+
+        cfg.setCacheConfiguration(ccfg);
 
         return cfg;
     }
 
     /**
-     * @return Memory config.
+     * @return DataStorage configuration.
      */
-    private DataStorageConfiguration memoryConfiguration() {
-        return new DataStorageConfiguration()
-            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
-                .setName("dfltMemPlc")
-                .setPersistenceEnabled(true)
-            )
-            .setPageSize(PAGE_SIZE)
-            .setConcurrencyLevel(1)
-            .setWalMode(WALMode.LOG_ONLY)
-            .setWalFsyncDelayNanos(0);
+    private DataStorageConfiguration storageConfiguration() {
+        DataRegionConfiguration regionCfg = new DataRegionConfiguration()
+                .setInitialSize(10L * 1024L * 1024L)
+                .setMaxSize(10L * 1024L * 1024L)
+                .setPageEvictionMode(DataPageEvictionMode.RANDOM_LRU);
+
+        DataStorageConfiguration cfg = new DataStorageConfiguration()
+                .setWalMode(WALMode.LOG_ONLY)
+                .setWalFsyncDelayNanos(0)
+                .setPageSize(PAGE_SIZE)
+                .setFileIOFactory(new AsyncFileIOFactory());
+
+        cfg.setDefaultDataRegionConfiguration(regionCfg);
+
+        return cfg;
     }
 
     /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
+    @Override protected void beforeTest() throws Exception {
         super.beforeTestsStarted();
 
         deleteWorkFiles();
     }
 
     /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
+    @Override protected void afterTest() throws Exception {
         super.afterTestsStopped();
 
         stopAllGrids();
@@ -93,10 +106,92 @@ private DataStorageConfiguration memoryConfiguration() {
     }
 
     /**
-     * Tests interruptions on WAL write.
+     * Tests interruptions on LFS read.
      *
      * @throws Exception If failed.
      */
+    public void testInterruptsOnLFSRead() throws Exception {
+        final Ignite ignite = startGrid();
+
+        ignite.active(true);
+
+        final int valLen = 8192;
+
+        final byte[] payload = new byte[valLen];
+
+        final int maxKey = 10_000;
+
+        Thread[] workers = new Thread[THREADS_CNT];
+
+
+        final IgniteCache cache = ignite.cache(CACHE_NAME);
+        for (int i=0; i < maxKey; i++) {
+            cache.put(i, payload);
+        }
+
+        final AtomicReference fail = new AtomicReference<>();
+
+
+        Runnable clo = new Runnable() {
+            @Override
+            public void run() {
+                cache.get(ThreadLocalRandom8.current().nextInt(maxKey / 5));
+            }
+        };
+        for (int i = 0; i < workers.length; i++) {
+            workers[i] = new Thread(clo);
+            workers[i].setName("reader-" + i);
+            workers[i].setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+                @Override public void uncaughtException(Thread t, Throwable e) {
+                    fail.compareAndSet(null, e);
+
+                }
+            });
+        }
+
+        for (Thread worker : workers)
+            worker.start();
+
+        //Thread.sleep(3_000);
+
+        // Interrupts should not affect reads.
+        for (int i = 0;i < workers.length / 2; i++)
+            workers[i].interrupt();
+
+        Thread.sleep(3_000);
+
+        stop = true;
+
+        for (Thread worker : workers)
+            worker.join();
+
+        Throwable t = fail.get();
+
+        assert t == null : t;
+
+
+
+        int verifiedKeys = 0;
+
+        // Post check.
+        for (int i = 0; i < maxKey; i++) {
+            byte[] val = (byte[]) cache.get(i);
+
+            if (val != null) {
+                assertEquals("Illegal length", valLen, val.length);
+
+                verifiedKeys++;
+            }
+        }
+
+        log.info("Verified keys: " + verifiedKeys);
+    }
+
+    /**
+     * Tests interruptions on WAL write.
+     *
+     * @throws Exception
+     */
     public void testInterruptsOnWALWrite() throws Exception {
         final Ignite ignite = startGrid();
 
@@ -114,7 +209,7 @@ public void testInterruptsOnWALWrite() throws Exception {
 
         Runnable clo = new Runnable() {
             @Override public void run() {
-                IgniteCache cache = ignite.cache(cacheName);
+                IgniteCache cache = ignite.cache(CACHE_NAME);
 
                 while (!stop)
                     cache.put(ThreadLocalRandom8.current().nextInt(maxKey), payload);
@@ -126,8 +221,6 @@ public void testInterruptsOnWALWrite() throws Exception {
             workers[i].setName("writer-" + i);
             workers[i].setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
                 @Override public void uncaughtException(Thread t, Throwable e) {
-                    log.error("Worker thread error", e);
-
                     fail.compareAndSet(null, e);
                 }
             });
@@ -153,7 +246,7 @@ public void testInterruptsOnWALWrite() throws Exception {
 
         assert t == null : t;
 
-        IgniteCache cache = ignite.cache(cacheName);
+        IgniteCache cache = ignite.cache(CACHE_NAME);
 
         int verifiedKeys = 0;
 
@@ -175,6 +268,6 @@ public void testInterruptsOnWALWrite() throws Exception {
      * @throws IgniteCheckedException If fail.
      */
     private void deleteWorkFiles() throws IgniteCheckedException {
-        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java
index 8068b083b3eb8..df649fa257a44 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java
@@ -30,8 +30,8 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.jetbrains.annotations.Nullable;
 import org.mockito.Mockito;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 8308fd3ff668f..9f86e0db08107 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheIntegrationTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsDiskErrorsRecoveringTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsNoActualWalHistoryTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsThreadInterruptionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryPPCTest;
@@ -57,6 +58,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgnitePdsTxCacheRebalancingTest.class);
 
         suite.addTestSuite(IgniteWalRecoveryPPCTest.class);
+        suite.addTestSuite(IgnitePdsDiskErrorsRecoveringTest.class);
 
         suite.addTestSuite(IgnitePdsBinaryMetadataOnClusterRestartTest.class);
         suite.addTestSuite(IgnitePdsMarshallerMappingRestoreOnNodeStartTest.class);

From d0a119c34eeb77c774d31ea45c50e97f55823f7b Mon Sep 17 00:00:00 2001
From: Sergey Chugunov 
Date: Wed, 24 Jan 2018 18:22:11 +0300
Subject: [PATCH 46/65] IGNITE-7506 Fixed resetting of
 GridClusterStateProcessor#compatibilityMode flag

---
 .../internal/cluster/IgniteClusterImpl.java   | 37 ++++++++++++++++--
 .../managers/discovery/DiscoCache.java        | 19 ++++++++--
 .../discovery/GridDiscoveryManager.java       | 15 ++++++--
 .../cluster/GridClusterStateProcessor.java    | 38 ++++++++++++++++++-
 4 files changed, 99 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
index 8cdb5502318e3..b69923b5bb783 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
@@ -47,6 +47,7 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteComponentType;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -59,10 +60,12 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteProductVersion;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IPS;
@@ -87,6 +90,9 @@ public class IgniteClusterImpl extends ClusterGroupAdapter implements IgniteClus
     /** Client reconnect future. */
     private IgniteFuture reconnecFut;
 
+    /** Minimal IgniteProductVersion supporting BaselineTopology */
+    private static final IgniteProductVersion MIN_BLT_SUPPORTING_VER = IgniteProductVersion.fromString("2.4.0");
+
     /**
      * Required by {@link Externalizable}.
      */
@@ -364,10 +370,38 @@ private boolean isInMemoryMode() {
         return !CU.isPersistenceEnabled(cfg);
     }
 
+    /**
+     * Verifies all nodes in current cluster topology support BaselineTopology feature
+     * so compatibilityMode flag is enabled to reset.
+     *
+     * @param discoCache
+     */
+    private void verifyBaselineTopologySupport(DiscoCache discoCache) {
+        if (discoCache.minimumServerNodeVersion().compareTo(MIN_BLT_SUPPORTING_VER) < 0) {
+            SB sb = new SB("Cluster contains nodes that don't support BaselineTopology: [");
+
+            for (ClusterNode cn : discoCache.serverNodes()) {
+                if (cn.version().compareTo(MIN_BLT_SUPPORTING_VER) < 0)
+                    sb
+                        .a("[")
+                        .a(cn.consistentId())
+                        .a(":")
+                        .a(cn.version())
+                        .a("], ");
+            }
+
+            sb.d(sb.length() - 2, sb.length());
+
+            throw new IgniteException(sb.a("]").toString());
+        }
+    }
+
     /**
      * Executes validation checks of cluster state and BaselineTopology before changing BaselineTopology to new one.
      */
     private void validateBeforeBaselineChange(Collection baselineTop) {
+        verifyBaselineTopologySupport(ctx.discovery().discoCache());
+
         if (!ctx.state().clusterState().active())
             throw new IgniteException("Changing BaselineTopology on inactive cluster is not allowed.");
 
@@ -381,9 +415,6 @@ private void validateBeforeBaselineChange(Collection bas
                 if (!onlineNodes.isEmpty())
                     throw new IgniteException("Removing online nodes from BaselineTopology is not supported: " + onlineNodes);
             }
-            else
-                //should never happen, actually: if cluster was activated, we expect it to have a BaselineTopology.
-                throw new IgniteException("Previous BaselineTopology was not found.");
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index 7bc0344ebd002..c21698f4b71ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -87,9 +87,12 @@ public class DiscoCache {
     /** Alive nodes. */
     final Set alives = new GridConcurrentHashSet<>();
 
-    /** */
+    /** Minimum {@link IgniteProductVersion} across all nodes including client nodes. */
     private final IgniteProductVersion minNodeVer;
 
+    /** Minimum {@link IgniteProductVersion} across alive server nodes. */
+    private final IgniteProductVersion minSrvNodeVer;
+
     /** */
     private final AffinityTopologyVersion topVer;
 
@@ -139,7 +142,8 @@ public class DiscoCache {
         Set alives0,
         @Nullable Map nodeIdToConsIdx,
         @Nullable  Map consIdxToNodeId,
-        IgniteProductVersion minNodeVer
+        IgniteProductVersion minNodeVer,
+        IgniteProductVersion minSrvNodeVer
     ) {
         this.topVer = topVer;
         this.state = state;
@@ -155,6 +159,7 @@ public class DiscoCache {
         this.nodeMap = nodeMap;
         alives.addAll(alives0);
         this.minNodeVer = minNodeVer;
+        this.minSrvNodeVer = minSrvNodeVer;
         this.nodeIdToConsIdx = nodeIdToConsIdx;
         this.consIdxToNodeId = consIdxToNodeId;
 
@@ -187,6 +192,13 @@ public IgniteProductVersion minimumNodeVersion() {
         return minNodeVer;
     }
 
+    /**
+     * @return Minimum server node version.
+     */
+    public IgniteProductVersion minimumServerNodeVersion() {
+        return minSrvNodeVer;
+    }
+
     /**
      * @return Current cluster state.
      */
@@ -425,7 +437,8 @@ public DiscoCache copy(AffinityTopologyVersion ver, @Nullable DiscoveryDataClust
             alives,
             nodeIdToConsIdx,
             consIdxToNodeId,
-            minNodeVer);
+            minNodeVer,
+            minSrvNodeVer);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 65cc6665e8f5b..82d0f66c6249b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2249,6 +2249,7 @@ public void reconnect() {
         List baselineNodes;
 
         IgniteProductVersion minVer = null;
+        IgniteProductVersion minSrvVer = null;
 
         for (ClusterNode node : topSnapshot) {
             if (alive(node))
@@ -2262,8 +2263,14 @@ public void reconnect() {
                 if (!node.isLocal())
                     rmtNodes.add(node);
 
-                if (!CU.clientNode(node))
+                if (!CU.clientNode(node)) {
                     srvNodes.add(node);
+
+                    if (minSrvVer == null)
+                        minSrvVer = node.version();
+                    else if (node.version().compareTo(minSrvVer) < 0)
+                        minSrvVer = node.version();
+                }
             }
 
             nodeMap.put(node.id(), node);
@@ -2341,7 +2348,8 @@ else if (node.version().compareTo(minVer) < 0)
             alives,
             nodeIdToConsIdx == null ? null : Collections.unmodifiableMap(nodeIdToConsIdx),
             consIdxToNodeId == null ? null : Collections.unmodifiableMap(consIdxToNodeId),
-            minVer);
+            minVer,
+            minSrvVer);
     }
 
     /**
@@ -3174,6 +3182,7 @@ public DiscoCache createDiscoCacheOnCacheChange(
             discoCache.alives,
             discoCache.nodeIdToConsIdx,
             discoCache.consIdxToNodeId,
-            discoCache.minimumNodeVersion());
+            discoCache.minimumNodeVersion(),
+            discoCache.minimumServerNodeVersion());
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 73c9d7f0571b6..f28df8ac14f80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -65,10 +65,12 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
@@ -128,6 +130,9 @@ public class GridClusterStateProcessor extends GridProcessorAdapter implements I
     /** */
     private final JdkMarshaller marsh = new JdkMarshaller();
 
+    /** Minimal IgniteProductVersion supporting BaselineTopology */
+    private static final IgniteProductVersion MIN_BLT_SUPPORTING_VER = IgniteProductVersion.fromString("2.4.0");
+
     /** Listener. */
     private final GridLocalEventListener lsr = new GridLocalEventListener() {
         @Override public void onEvent(Event evt) {
@@ -417,6 +422,9 @@ protected void afterStateChangeFinished(IgniteUuid msgId, boolean success) {
                 (msg.baselineTopology() == null ? ": null"
                     : "[id=" + msg.baselineTopology().id() + "]"));
 
+        if (msg.baselineTopology() != null)
+            compatibilityMode = false;
+
         if (state.transition()) {
             if (isApplicable(msg, state)) {
                 GridChangeGlobalStateFuture fut = changeStateFuture(msg);
@@ -713,7 +721,7 @@ protected IgniteCheckedException concurrentStateChangeError(boolean activate) {
         if (inMemoryMode)
             return changeGlobalState0(activate, null, false);
 
-        BaselineTopology newBlt = compatibilityMode ? null :
+        BaselineTopology newBlt = (compatibilityMode && !forceChangeBaselineTopology) ? null :
             calculateNewBaselineTopology(activate, baselineNodes, forceChangeBaselineTopology);
 
         return changeGlobalState0(activate, newBlt, forceChangeBaselineTopology);
@@ -783,12 +791,40 @@ private Collection baselineNodes() {
         return bltNodes;
     }
 
+    /**
+     * Verifies all nodes in current cluster topology support BaselineTopology feature
+     * so compatibilityMode flag is enabled to reset.
+     *
+     * @param discoCache
+     */
+    private void verifyBaselineTopologySupport(DiscoCache discoCache) {
+        if (discoCache.minimumServerNodeVersion().compareTo(MIN_BLT_SUPPORTING_VER) < 0) {
+            SB sb = new SB("Cluster contains nodes that don't support BaselineTopology: [");
+
+            for (ClusterNode cn : discoCache.serverNodes()) {
+                if (cn.version().compareTo(MIN_BLT_SUPPORTING_VER) < 0)
+                    sb
+                        .a("[")
+                        .a(cn.consistentId())
+                        .a(":")
+                        .a(cn.version())
+                        .a("], ");
+            }
+
+            sb.d(sb.length() - 2, sb.length());
+
+            throw new IgniteException(sb.a("]").toString());
+        }
+    }
+
     /** */
     private IgniteInternalFuture changeGlobalState0(final boolean activate,
         BaselineTopology blt, boolean forceChangeBaselineTopology) {
         if (ctx.isDaemon() || ctx.clientNode()) {
             GridFutureAdapter fut = new GridFutureAdapter<>();
 
+            verifyBaselineTopologySupport(ctx.discovery().discoCache());
+
             sendComputeChangeGlobalState(activate, blt, forceChangeBaselineTopology, fut);
 
             return fut;

From a36373459191add85b2e45153b0d7722f54d8c48 Mon Sep 17 00:00:00 2001
From: devozerov 
Date: Wed, 24 Jan 2018 23:11:16 +0300
Subject: [PATCH 47/65] AI 2.4 release notes.

---
 RELEASE_NOTES.txt | 90 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 88 insertions(+), 2 deletions(-)

diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 7af132bd426cf..77db736468574 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -5,9 +5,95 @@ Apache Ignite In-Memory Data Fabric 2.4
 ---------------------------------------
 Ignite:
 * Introduced Baseline Affinity Topology
+* Ability to disable WAL for cache in runtime through IgniteCluster API or ALTER TABLE command
 * Added ability to convert WAL to human-readable form
-* Added support for uninterruptible writes for WAL and storage
-* Local scan performance was improved up to 2 times in certain use-cases
+* Ability to enable/disable JDBC, ODBC and thin client endpoints
+* Project moved to Java 8
+* Java 7 is no longer supported
+* Apache Ignite can be run on Java 9
+* Introduced Apache Ignite packaging with RPM
+* Spark Data Frames support
+* Added integration with Spring 3.x
+* Added handler to stop node when IO error is encountered
+* EvictionPolicyFactory support added to CacheConfiguration
+* Added JMX metrics for memory regions
+* Added detailed memory consumption on start and OOM reporting
+* Added custom thread pools monitoring
+* Support Ignite MR in Kerberized environment without IGFS
+* S3 IP finder: support server side encryption and bucket endpoint configuration
+* Multiple fixes and performance optimizations
+
+Ignite.NET:
+* Thin Client initial release
+* .NET Core / Linux / macOS support
+* Mono platform support
+* Removed C++ JNI layer
+
+SQL:
+* Added DEFAULT support to CREATE TABLE command
+* Added ALTER TABLE DROP COLUMN support
+* Added INLINE_SIZE option to CREATE INDEX command
+* Added PARALLEL option to CREATE INDEX command
+* Added optional on-heap row cache
+* INSERT command now can be executed without explicit column names (INSERT INTO table VALUES ...)
+* Allowed multiple caches to share the same schema
+* Added support for Java 8 Date and Time API
+* Added "notNull" property to QuerySqlField annotation
+* Improved COUNT(*) performance
+* Fixed a problem causing an exception in certain nodes do not have primary partitions for REPLICATED cache
+* Fixed per-thread H2 connection leak.
+* Fixed partition exchange hang due to SQL query running inside a transaction.
+
+JDBC Driver:
+* Optimized batched operations processing
+
+ODBC Driver:
+* Added support of multiple statements execution with one call
+* Added support of SQL_ATTR_CONNECTION_TIMEOUT
+* Added support of SQL_ATTR_QUERY_TIMEOUT
+* Optimized batched operations processing
+* Proper handling of ungraceful TCP disconnects (keep-alive connection)
+* Fixed error preventing close of executed DML statement
+* Fixed table names returned by SQLTables for DDL-created tables
+
+Machine Learning:
+* Implemented Fuzzy c-means algorithm
+* Implemented gradient descent for OLS Linear Regression
+* Implemented K nearest neighbor algorithm
+* Introduced API for datasets
+* Introduced API for ML model and trainers
+* Introduced common mechanism for group training of models
+* Added distributed version of multilayer perceptron
+* Added local version of multilayer perceptron
+* Added basic import/export functionality for ml models
+* Added decision tree algorithm
+* Performance optimizations for distributed operations
+
+Web Console:
+* Implemented component for cluster selection and activation
+* Implemented support for multiple statements on Queries screen
+* Implemented progress indication for 'execute', 'scan' and 'export' actions on Queries screen
+* Added support for ClientConnectorConfiguration
+* Added several SQL examples on Queries screen
+* Added "Connected clusters" component in header
+* Added support for custom SMTP server configuration
+* Added detection of CSV separator based on browser locale
+* Added "Copy to clipboard" action on Queries screen
+* Added duration and node ID in results header and 'Show query' modal
+* Improved support for binary JDBC types on import from RDBMS
+* Fixed Web Console under IE11
+* Fixed configuration generation for imported model in case of Oracle NUMBER(5) data type
+* Fixed output of big numbers in SQL query results
+
+Visor:
+* Added "-quiet" flag for batch mode
+* Added ability to start cache rebalance
+* Added output of data region metrics to "node" command
+* Added dialog to put/get/remove values to/from cache for simple key types
+* Show valid message for caches when cluster is inactive
+* Fixed 'cache -a' in case of node filter
+* Fixed reading last command line in batch mode
+* Updated eviction policy factory in configs
 
 Apache Ignite In-Memory Data Fabric 2.3
 ---------------------------------------

From f6421db988c3b01129357cad02d22e263aadba70 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh 
Date: Thu, 25 Jan 2018 11:00:25 +0300
Subject: [PATCH 48/65] IGNITE-7502 Disabled BLT for non-persistent caches

---
 .../affinity/GridAffinityAssignmentCache.java | 10 +++++--
 .../cache/CacheAffinitySharedManager.java     |  3 +-
 .../processors/cache/CacheGroupContext.java   |  3 +-
 .../CacheBaselineTopologyTest.java            | 28 +++++++++++++++++++
 4 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index dd46246146b26..dd51aede00ee3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -107,6 +107,9 @@ public class GridAffinityAssignmentCache {
     /** */
     private final boolean locCache;
 
+    /** */
+    private final boolean persistentCache;
+
     /** Node stop flag. */
     private volatile IgniteCheckedException stopErr;
 
@@ -137,7 +140,8 @@ public GridAffinityAssignmentCache(GridKernalContext ctx,
         AffinityFunction aff,
         IgnitePredicate nodeFilter,
         int backups,
-        boolean locCache)
+        boolean locCache,
+        boolean persistentCache)
     {
         assert ctx != null;
         assert aff != null;
@@ -151,6 +155,7 @@ public GridAffinityAssignmentCache(GridKernalContext ctx,
         this.grpId = grpId;
         this.backups = backups;
         this.locCache = locCache;
+        this.persistentCache = persistentCache;
 
         log = ctx.log(GridAffinityAssignmentCache.class);
 
@@ -290,7 +295,8 @@ public List> calculate(AffinityTopologyVersion topVer, @Nullab
         boolean changedBaseline = false;
 
         if (discoCache != null) {
-            hasBaseline = discoCache.state().baselineTopology() != null;
+            hasBaseline = discoCache.state().baselineTopology() != null && persistentCache;
+
             changedBaseline = !hasBaseline ? baselineTopology != null :
                 !discoCache.state().baselineTopology().equals(baselineTopology);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 921701a14d01f..9eff56097e939 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -2459,7 +2459,8 @@ static CacheGroupHolder2 create(
                 affFunc,
                 ccfg.getNodeFilter(),
                 ccfg.getBackups(),
-                ccfg.getCacheMode() == LOCAL);
+                ccfg.getCacheMode() == LOCAL,
+                grpDesc.persistenceEnabled());
 
             return new CacheGroupHolder2(ccfg.getRebalanceMode() != NONE, cctx, aff, initAff);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 246f298f5d046..b980d5c17934b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -857,7 +857,8 @@ public void start() throws IgniteCheckedException {
             ccfg.getAffinity(),
             ccfg.getNodeFilter(),
             ccfg.getBackups(),
-            ccfg.getCacheMode() == LOCAL);
+            ccfg.getCacheMode() == LOCAL,
+            persistenceEnabled());
 
         if (ccfg.getCacheMode() != LOCAL) {
             top = new GridDhtPartitionTopologyImpl(ctx, this);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 7dc1bfaf15016..857d3a1dc3c2c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -105,6 +105,13 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
                     .setMaxSize(100 * 1024 * 1024)
                     .setInitialSize(100 * 1024 * 1024)
             )
+            .setDataRegionConfigurations(
+                new DataRegionConfiguration()
+                .setName("memory")
+                .setPersistenceEnabled(false)
+                .setMaxSize(100 * 1024 * 1024)
+                .setInitialSize(100 * 1024 * 1024)
+            )
             .setWalMode(WALMode.LOG_ONLY)
         );
 
@@ -739,6 +746,27 @@ public void testAffinityAssignmentChangedAfterRestart() throws Exception {
             assertEquals("k=" + k, Integer.valueOf(k), cache.get(k));
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNonPersistentCachesIgnoreBaselineTopology() throws Exception {
+        Ignite ig = startGrids(4);
+
+        ig.cluster().active(true);
+
+        IgniteCache persistentCache = ig.createCache(CACHE_NAME);
+
+        IgniteCache inMemoryCache = ig.createCache(
+            new CacheConfiguration<>().setName(CACHE_NAME + 2).setDataRegionName("memory"));
+
+        Ignite newNode = startGrid(4);
+
+        awaitPartitionMapExchange();
+
+        assertEquals(0, ig.affinity(persistentCache.getName()).allPartitions(newNode.cluster().localNode()).length);
+        assertTrue(ig.affinity(inMemoryCache.getName()).allPartitions(newNode.cluster().localNode()).length > 0);
+    }
+
     /** */
     private Collection baselineNodes(Collection clNodes) {
         Collection res = new ArrayList<>(clNodes.size());

From d268eb2573ad46efce8b6d9480b7c9c317b792cd Mon Sep 17 00:00:00 2001
From: Andrey Gura 
Date: Mon, 22 Jan 2018 22:41:54 +0300
Subject: [PATCH 49/65] ignite-6643 Marshalling improvements

---
 .../apache/ignite/IgniteSystemProperties.java |  60 +----
 .../org/apache/ignite/internal/ClassSet.java  | 108 +++++++++
 .../internal/GridKernalContextImpl.java       |   6 +-
 .../apache/ignite/internal/IgniteKernal.java  | 107 ++++++++-
 .../internal/MarshallerContextImpl.java       |  33 ++-
 .../GridClientOptimizedMarshaller.java        |   4 +-
 .../optimized/OptimizedMarshallerUtils.java   |   4 -
 .../optimized/OptimizedObjectInputStream.java |   5 +-
 .../OptimizedObjectOutputStream.java          |   3 +-
 .../reader/StandaloneGridKernalContext.java   |   2 +-
 .../platform/utils/PlatformUtils.java         |   2 +-
 .../rest/protocols/tcp/GridTcpRestParser.java |  15 +-
 .../protocols/tcp/GridTcpRestProtocol.java    |  31 +--
 .../ignite/internal/util/IgniteUtils.java     |  15 ++
 .../ignite/marshaller/MarshallerContext.java  |  14 ++
 .../ignite/marshaller/MarshallerUtils.java    |   6 +
 .../ignite/marshaller/jdk/JdkMarshaller.java  |  20 +-
 .../jdk/JdkMarshallerObjectInputStream.java   |   9 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   8 +-
 .../apache/ignite/stream/StreamAdapter.java   |   2 +-
 .../ignite/stream/socket/SocketStreamer.java  |   8 +-
 .../config/class_list_exploit_excluded.txt    |  18 ++
 .../config/class_list_exploit_included.txt    |  19 ++
 .../apache/ignite/internal/ClassSetTest.java  |  71 ++++++
 .../MarshallerContextLockingSelfTest.java     |   2 +-
 ...idBinaryMarshallerCtxDisabledSelfTest.java |  12 +
 .../GridCacheEntryMemorySizeSelfTest.java     |  11 +-
 .../marshaller/MarshallerContextSelfTest.java |  10 +-
 .../marshaller/MarshallerContextTestImpl.java |   2 +-
 .../DiscoveryUnmarshalVulnerabilityTest.java  | 180 ++++++++++++++
 ...ketStreamerUnmarshalVulnerabilityTest.java | 222 ++++++++++++++++++
 .../junits/GridTestKernalContext.java         |   3 +-
 .../testsuites/IgniteBasicTestSuite.java      |   3 +
 .../IgniteSpiDiscoverySelfTestSuite.java      |   3 +
 .../testsuites/IgniteStreamSelfTestSuite.java |   2 +
 35 files changed, 894 insertions(+), 126 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/ClassSet.java
 create mode 100644 modules/core/src/test/config/class_list_exploit_excluded.txt
 create mode 100644 modules/core/src/test/config/class_list_exploit_included.txt
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/ClassSetTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/DiscoveryUnmarshalVulnerabilityTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/stream/socket/SocketStreamerUnmarshalVulnerabilityTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 833773a15b7bb..43b718bb4b957 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -28,7 +28,6 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.internal.client.GridClient;
 import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller;
-import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.stream.StreamTransformer;
 import org.jetbrains.annotations.Nullable;
 
@@ -221,11 +220,6 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_MAX_COMPLETED_TX_COUNT = "IGNITE_MAX_COMPLETED_TX_COUNT";
 
-    /**
-     * Concurrency level for all concurrent hash maps created by Ignite.
-     */
-    public static final String IGNITE_MAP_CONCURRENCY_LEVEL = "IGNITE_MAP_CONCURRENCY_LEVEL";
-
     /**
      * Transactions that take more time, than value of this property, will be output to log
      * with warning level. {@code 0} (default value) disables warning on slow transactions.
@@ -320,13 +314,6 @@ public final class IgniteSystemProperties {
     /** Ttl of removed cache entries (ms). */
     public static final String IGNITE_CACHE_REMOVED_ENTRIES_TTL = "IGNITE_CACHE_REMOVED_ENTRIES_TTL";
 
-    /** Maximum amount of concurrent updates per system thread in atomic caches in case of PRIMARY_SYNC or FULL_ASYNC
-     * write synchronization mode. If this limit is exceeded then update will be performed with FULL_SYNC
-     * synchronization mode. If value is {@code 0} then limit is unbounded.
-     */
-    public static final String IGNITE_ATOMIC_CACHE_MAX_CONCURRENT_DHT_UPDATES =
-        "IGNITE_ATOMIC_CACHE_MAX_CONCURRENT_DHT_UPDATES";
-
     /**
      * Comma separated list of addresses in format "10.100.22.100:45000,10.100.22.101:45000".
      * Makes sense only for {@link org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder}.
@@ -479,9 +466,6 @@ public final class IgniteSystemProperties {
     /** Number of cache operation retries in case of topology exceptions. */
     public static final String IGNITE_CACHE_RETRIES_COUNT = "IGNITE_CACHE_RETRIES_COUNT";
 
-    /** Number of times pending cache objects will be dumped to the log in case of partition exchange timeout. */
-    public static final String IGNITE_DUMP_PENDING_OBJECTS_THRESHOLD = "IGNITE_DUMP_PENDING_OBJECTS_THRESHOLD";
-
     /** If this property is set to {@code true} then Ignite will log thread dump in case of partition exchange timeout. */
     public static final String IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT = "IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT";
 
@@ -524,6 +508,12 @@ public final class IgniteSystemProperties {
     public static final String IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 =
         "IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2";
 
+    /** Defines path to the file that contains list of classes allowed to safe deserialization.*/
+    public static final String IGNITE_MARSHALLER_WHITELIST = "IGNITE_MARSHALLER_WHITELIST";
+
+    /** Defines path to the file that contains list of classes disallowed to safe deserialization.*/
+    public static final String IGNITE_MARSHALLER_BLACKLIST = "IGNITE_MARSHALLER_BLACKLIST";
+
     /**
      * If set to {@code true}, then default selected keys set is used inside
      * {@code GridNioServer} which lead to some extra garbage generation when
@@ -689,16 +679,7 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_FORCE_START_JAVA7 = "IGNITE_FORCE_START_JAVA7";
 
-    /** Returns true for system properties only avoiding sending sensitive information. */
-    private static final IgnitePredicate> PROPS_FILTER = new IgnitePredicate>() {
-        @Override public boolean apply(final Map.Entry entry) {
-            final String key = entry.getKey();
-
-            return key.startsWith("java.") || key.startsWith("os.") || key.startsWith("user.");
-        }
-    };
-
-     /**
+    /**
      * When set to {@code true}, Ignite switches to compatibility mode with versions that don't
      * support service security permissions. In this case security permissions will be ignored
      * (if they set).
@@ -739,11 +720,6 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_ENABLE_FORCIBLE_NODE_KILL = "IGNITE_ENABLE_FORCIBLE_NODE_KILL";
 
-    /**
-     *
-     */
-    public static final String IGNITE_WAL_ARCHIVE_COMPACT_SKIP_DELTA_RECORD = "IGNITE_WAL_ARCHIVE_COMPACT_SKIP_DELTA_RECORD";
-
     /**
      * Tasks stealing will be started if tasks queue size per data-streamer thread exceeds this threshold.
      * 

@@ -1018,26 +994,4 @@ public static Properties snapshot() {
 
         return sysProps;
     }
-
-    /**
-     * Does the same as {@link #snapshot()} but filters out
-     * possible sensitive user data.
-     *
-     * @return Snapshot of system properties.
-     */
-    @SuppressWarnings("unchecked")
-    public static Properties safeSnapshot() {
-        final Properties props = snapshot();
-
-        final Iterator> iter = props.entrySet().iterator();
-
-        while (iter.hasNext()) {
-            final Map.Entry entry = iter.next();
-
-            if (!PROPS_FILTER.apply(entry))
-                iter.remove();
-        }
-
-        return props;
-    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/ClassSet.java b/modules/core/src/main/java/org/apache/ignite/internal/ClassSet.java
new file mode 100644
index 0000000000000..400984d5f0bed
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/ClassSet.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.IllegalFormatException;
+import java.util.Map;
+
+/**
+ * Set of classes represented as prefix tree.
+ * {@code *} symbol is allowed and indicates that all packages and classes are included.
+ */
+public class ClassSet {
+    /** Corresponds to {@code *} symbol. */
+    private static final Map ALL = Collections.emptyMap();
+
+    /** Root. */
+    private Node root = new Node();
+
+    /**
+     * Adds class name to the set.
+     *
+     * @param clsName Class name.
+     */
+    public void add(String clsName) {
+        String[] tokens = clsName.split("\\.");
+
+        Node cur = root;
+
+        for (int i = 0; i < tokens.length; i++) {
+            if (cur.children == ALL)
+                return;
+
+            if (tokens[i].equals("*")) {
+                if (i != tokens.length - 1)
+                    throw new IllegalArgumentException("Incorrect class name format.");
+
+                cur.children = ALL;
+
+                return;
+            }
+
+            if (cur.children == null)
+                cur.children = new HashMap<>();
+
+            Node n = cur.children.get(tokens[i]);
+
+            if (n == null) {
+                n = new Node();
+
+                cur.children.put(tokens[i], n);
+            }
+
+            cur = n;
+        }
+    }
+
+    /**
+     * @param clsName Class name.
+     */
+    public boolean contains(String clsName) {
+        String[] tokens = clsName.split("\\.");
+
+        Node cur = root;
+
+        for (int i = 0; i < tokens.length; i++) {
+            if (cur.children == ALL)
+                return true;
+
+            if (cur.children == null)
+                return false;
+
+            Node n = cur.children.get(tokens[i]);
+
+            if (n == null)
+                return false;
+
+            if (i == tokens.length - 1)
+                return true;
+
+            cur = n;
+        }
+
+        return false;
+    }
+
+    /** */
+    private static class Node {
+        /** Children. */
+        private Map children;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 9a315e754dc5e..3f064fa3505f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -91,6 +91,7 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.PluginNotFoundException;
 import org.apache.ignite.plugin.PluginProvider;
 import org.apache.ignite.thread.IgniteStripedThreadPoolExecutor;
@@ -441,7 +442,8 @@ protected GridKernalContextImpl(
         ExecutorService qryExecSvc,
         ExecutorService schemaExecSvc,
         @Nullable Map customExecSvcs,
-        List plugins
+        List plugins,
+        IgnitePredicate clsFilter
     ) {
         assert grid != null;
         assert cfg != null;
@@ -467,7 +469,7 @@ protected GridKernalContextImpl(
         this.schemaExecSvc = schemaExecSvc;
         this.customExecSvcs = customExecSvcs;
 
-        marshCtx = new MarshallerContextImpl(plugins);
+        marshCtx = new MarshallerContextImpl(plugins, clsFilter);
 
         try {
             spring = SPRING.create(false);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 16698b5f4a977..30949632ae637 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -17,10 +17,14 @@
 
 package org.apache.ignite.internal;
 
+import java.io.BufferedReader;
 import java.io.Externalizable;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.InvalidObjectException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
@@ -259,6 +263,8 @@
 import static org.apache.ignite.internal.IgniteVersionUtils.VER_STR;
 import static org.apache.ignite.lifecycle.LifecycleEventType.AFTER_NODE_START;
 import static org.apache.ignite.lifecycle.LifecycleEventType.BEFORE_NODE_START;
+import static org.apache.ignite.marshaller.MarshallerUtils.CLS_NAMES_FILE;
+import static org.apache.ignite.marshaller.MarshallerUtils.JDK_CLS_NAMES_FILE;
 
 /**
  * Ignite kernal.
@@ -834,7 +840,8 @@ public void start(
                 qryExecSvc,
                 schemaExecSvc,
                 customExecSvcs,
-                plugins
+                plugins,
+                classNameFilter()
             );
 
             cfg.getMarshaller().setContext(ctx.marshallerContext());
@@ -1684,6 +1691,104 @@ private void startProcessor(GridProcessor proc) throws IgniteCheckedException {
         }
     }
 
+    /**
+     * Returns class name filter for marshaller.
+     * @return Class name filter for marshaller.
+     */
+    private IgnitePredicate classNameFilter() throws IgniteCheckedException {
+        ClassSet whiteList = classWhiteList();
+        ClassSet blackList = classBlackList();
+
+        return new IgnitePredicate() {
+            @Override public boolean apply(String s) {
+                // Allows all primitive arrays and checks arrays' type.
+                if ((blackList != null || whiteList != null) && s.charAt(0) == '[') {
+                    if (s.charAt(1) == 'L' && s.length() > 2)
+                        s = s.substring(2, s.length() - 1);
+                    else
+                        return true;
+                }
+
+                return (blackList == null || !blackList.contains(s)) && (whiteList == null || whiteList.contains(s));
+            }
+        };
+    }
+
+    /**
+     * @return White list of classes.
+     */
+    private ClassSet classWhiteList() throws IgniteCheckedException {
+        ClassSet clsSet = null;
+
+        String fileName = IgniteSystemProperties.getString(IgniteSystemProperties.IGNITE_MARSHALLER_WHITELIST);
+
+        if (fileName != null) {
+            clsSet = new ClassSet();
+
+            addClassNames(JDK_CLS_NAMES_FILE, clsSet);
+            addClassNames(CLS_NAMES_FILE, clsSet);
+            addClassNames(fileName, clsSet);
+        }
+
+        return clsSet;
+    }
+
+    /**
+     * @return Black list of classes.
+     */
+    private ClassSet classBlackList() throws IgniteCheckedException {
+        ClassSet clsSet = null;
+
+        String blackListFileName = IgniteSystemProperties.getString(IgniteSystemProperties.IGNITE_MARSHALLER_BLACKLIST);
+
+        if (blackListFileName != null)
+            addClassNames(blackListFileName, clsSet = new ClassSet());
+
+        return clsSet;
+    }
+
+
+    /**
+     * Reads class names from resource referred by given system property name and returns set of classes.
+     * @param fileName File name containing list of classes.
+     * @param clsSet Class set for update.
+     * @return Set of classes.
+     */
+    private void addClassNames(String fileName, ClassSet clsSet) throws IgniteCheckedException {
+        InputStream is = this.getClass().getClassLoader().getResourceAsStream(fileName);
+
+        if (is == null) {
+            try {
+                is = new FileInputStream(new File(fileName));
+            }
+            catch (FileNotFoundException e) {
+                throw new IgniteCheckedException("File " + fileName + " not found.");
+            }
+        }
+
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
+            String line;
+
+            for (int i = 1; (line = reader.readLine()) != null; i++) {
+                String s = line.trim();
+
+                if (!s.isEmpty() && s.charAt(0) != '#' && s.charAt(0) != '[') {
+                    try {
+                        clsSet.add(s);
+                    }
+                    catch (IllegalArgumentException e) {
+                        throw new IgniteCheckedException("Exception occurred while reading list of classes" +
+                        "[path=" + fileName + ", row=" + i + ", line=" + s + ']', e);
+                    }
+                }
+            }
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Exception occurred while reading and creating list of classes " +
+                "[path=" + fileName + ']', e);
+        }
+    }
+
     /**
      * Add helper.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java
index 08661a3b11414..d82ab17629a97 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java
@@ -50,24 +50,22 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.marshaller.MarshallerContext;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.plugin.PluginProvider;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentHashMap8;
 
 import static org.apache.ignite.internal.MarshallerPlatformIds.JAVA_ID;
+import static org.apache.ignite.marshaller.MarshallerUtils.CLS_NAMES_FILE;
+import static org.apache.ignite.marshaller.MarshallerUtils.JDK_CLS_NAMES_FILE;
 
 /**
  * Marshaller context implementation.
  */
 public class MarshallerContextImpl implements MarshallerContext {
-    /** */
-    private static final String CLS_NAMES_FILE = "META-INF/classnames.properties";
-
-    /** */
-    private static final String JDK_CLS_NAMES_FILE = "META-INF/classnames-jdk.properties";
-
     /** */
     private final Map sysTypesMap = new HashMap<>();
 
@@ -89,6 +87,12 @@ public class MarshallerContextImpl implements MarshallerContext {
     /** */
     private boolean clientNode;
 
+    /** Class name filter. */
+    private final IgnitePredicate clsFilter;
+
+    /** JDK marshaller. */
+    private final JdkMarshaller jdkMarsh;
+
     /**
      * Marshaller mapping file store directory. {@code null} used for standard folder, in this case folder is calculated
      * from work directory. Non null value may be used to setup custom directory from outside
@@ -100,7 +104,10 @@ public class MarshallerContextImpl implements MarshallerContext {
      *
      * @param plugins Plugins.
      */
-    public MarshallerContextImpl(@Nullable Collection plugins) {
+    public MarshallerContextImpl(@Nullable Collection plugins, IgnitePredicate clsFilter) {
+        this.clsFilter = clsFilter;
+        this.jdkMarsh = new JdkMarshaller(clsFilter);
+
         initializeCaches();
 
         try {
@@ -330,7 +337,7 @@ public void onMappingAccepted(final MarshallerMappingItem item) {
         if (clsName == null)
             throw new ClassNotFoundException("Unknown type ID: " + typeId);
 
-        return U.forName(clsName, ldr);
+        return U.forName(clsName, ldr, clsFilter);
     }
 
     /** {@inheritDoc} */
@@ -385,6 +392,16 @@ public void onMappingAccepted(final MarshallerMappingItem item) {
         return clsName;
     }
 
+    /** {@inheritDoc} */
+    @Override public IgnitePredicate classNameFilter() {
+        return clsFilter;
+    }
+
+    /** {@inheritDoc} */
+    @Override public JdkMarshaller jdkMarshaller() {
+        return jdkMarsh;
+    }
+
     /**
      * @param platformId Platform id.
      * @param typeId Type id.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/marshaller/optimized/GridClientOptimizedMarshaller.java b/modules/core/src/main/java/org/apache/ignite/internal/client/marshaller/optimized/GridClientOptimizedMarshaller.java
index 257b34cde6834..2af1841b71582 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/marshaller/optimized/GridClientOptimizedMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/marshaller/optimized/GridClientOptimizedMarshaller.java
@@ -117,14 +117,14 @@ public GridClientOptimizedMarshaller(boolean requireSer, int poolSize) throws IO
     private static class ClientMarshallerContext extends MarshallerContextImpl {
         /** */
         public ClientMarshallerContext() {
-            super(null);
+            super(null, null);
         }
 
         /**
          * @param plugins Plugins.
          */
         public ClientMarshallerContext(@Nullable List plugins) {
-            super(plugins);
+            super(plugins, null);
         }
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerUtils.java
index f9a6df9b907c2..aa4bfd6a2cd66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerUtils.java
@@ -33,7 +33,6 @@
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.marshaller.MarshallerContext;
-import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 
 import static org.apache.ignite.internal.MarshallerPlatformIds.JAVA_ID;
 
@@ -152,9 +151,6 @@ class OptimizedMarshallerUtils {
     /** UTF-8 character name. */
     static final Charset UTF_8 = Charset.forName("UTF-8");
 
-    /** JDK marshaller. */
-    static final JdkMarshaller JDK_MARSH = new JdkMarshaller();
-
     static {
         long mapOff;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
index 93a6f6f107d75..0bd7cc01c7c80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectInputStream.java
@@ -70,7 +70,6 @@
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.INT;
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.INT_ARR;
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.JDK;
-import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.JDK_MARSH;
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.LINKED_HASH_MAP;
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.LINKED_HASH_SET;
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.LINKED_LIST;
@@ -227,7 +226,7 @@ private Object readObject0() throws ClassNotFoundException, IOException {
 
             case JDK:
                 try {
-                    return JDK_MARSH.unmarshal(this, clsLdr);
+                    return ctx.jdkMarshaller().unmarshal(this, clsLdr);
                 }
                 catch (IgniteCheckedException e) {
                     IOException ioEx = e.getCause(IOException.class);
@@ -338,7 +337,7 @@ private Object readObject0() throws ClassNotFoundException, IOException {
                 int typeId = readInt();
 
                 OptimizedClassDescriptor desc = typeId == 0 ?
-                    classDescriptor(clsMap, U.forName(readUTF(), clsLdr), ctx, mapper):
+                    classDescriptor(clsMap, U.forName(readUTF(), clsLdr, ctx.classNameFilter()), ctx, mapper):
                     classDescriptor(clsMap, typeId, clsLdr, ctx, mapper);
 
                 curCls = desc.describedClass();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
index 092d6a4f1bf5d..8a323a7fefe20 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedObjectOutputStream.java
@@ -47,7 +47,6 @@
 
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.HANDLE;
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.JDK;
-import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.JDK_MARSH;
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.NULL;
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.classDescriptor;
 import static org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerUtils.getBoolean;
@@ -185,7 +184,7 @@ private void writeObject0(Object obj) throws IOException {
                 writeByte(JDK);
 
                 try {
-                    JDK_MARSH.marshal(obj, this);
+                    ctx.jdkMarshaller().marshal(obj, this);
                 }
                 catch (IgniteCheckedException e) {
                     IOException ioEx = e.getCause(IOException.class);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index 908d5b7771544..2c1d91eff1c6e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -137,7 +137,7 @@ public class StandaloneGridKernalContext implements GridKernalContext {
             throw new IllegalStateException("Must not fail on empty providers list.", e);
         }
 
-        this.marshallerCtx = new MarshallerContextImpl(null);
+        this.marshallerCtx = new MarshallerContextImpl(null, null);
         this.cfg = prepareIgniteConfiguration();
 
         // Fake folder provided to perform processor startup on empty folder.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
index 2954a9e6c0e87..12f70dd95c366 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformUtils.java
@@ -857,7 +857,7 @@ public static GridBinaryMarshaller marshaller() {
 
         BinaryMarshaller marsh = new BinaryMarshaller();
 
-        marsh.setContext(new MarshallerContextImpl(null));
+        marsh.setContext(new MarshallerContextImpl(null, null));
 
         ctx.configure(marsh, new IgniteConfiguration());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestParser.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestParser.java
index b4c8b7cb8b8b5..65eb3c317e50e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestParser.java
@@ -72,7 +72,7 @@ public class GridTcpRestParser implements GridNioParser {
     private static final Charset UTF_8 = Charset.forName("UTF-8");
 
     /** JDK marshaller. */
-    private final Marshaller jdkMarshaller = new JdkMarshaller();
+    private final Marshaller marsh;
 
     /** Router client flag. */
     private final boolean routerClient;
@@ -81,7 +81,16 @@ public class GridTcpRestParser implements GridNioParser {
      * @param routerClient Router client flag.
      */
     public GridTcpRestParser(boolean routerClient) {
+        this(routerClient, new JdkMarshaller());
+    }
+
+    /**
+     * @param routerClient Router client flag.
+     * @param marsh Marshaller.
+     */
+    public GridTcpRestParser(boolean routerClient, Marshaller marsh) {
         this.routerClient = routerClient;
+        this.marsh = marsh;
     }
 
     /** {@inheritDoc} */
@@ -735,7 +744,7 @@ private Object decodeObj(short flags, byte[] bytes) throws IgniteCheckedExceptio
         assert bytes != null;
 
         if ((flags & SERIALIZED_FLAG) != 0)
-            return U.unmarshal(jdkMarshaller, bytes, null);
+            return U.unmarshal(marsh, bytes, null);
 
         int masked = flags & 0xff00;
 
@@ -817,7 +826,7 @@ else if (obj instanceof byte[]) {
             flags |= BYTE_ARR_FLAG;
         }
         else {
-            U.marshal(jdkMarshaller, obj, out);
+            U.marshal(marsh, obj, out);
 
             flags |= SERIALIZED_FLAG;
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
index df0e569f5e1ff..0049dbc5d72f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
@@ -44,17 +44,12 @@
 import org.apache.ignite.internal.util.nio.GridNioParser;
 import org.apache.ignite.internal.util.nio.GridNioServer;
 import org.apache.ignite.internal.util.nio.GridNioServerListener;
-import org.apache.ignite.internal.util.nio.GridNioSession;
 import org.apache.ignite.internal.util.nio.ssl.GridNioSslFilter;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.marshaller.Marshaller;
-import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.plugin.PluginProvider;
 import org.apache.ignite.spi.IgnitePortProtocol;
 import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.MARSHALLER;
-
 /**
  * TCP binary protocol implementation.
  */
@@ -62,9 +57,6 @@ public class GridTcpRestProtocol extends GridRestProtocolAdapter {
     /** Server. */
     private GridNioServer srv;
 
-    /** JDK marshaller. */
-    private final Marshaller jdkMarshaller = new JdkMarshaller();
-
     /** NIO server listener. */
     private GridTcpRestNioListener lsnr;
 
@@ -73,27 +65,6 @@ public GridTcpRestProtocol(GridKernalContext ctx) {
         super(ctx);
     }
 
-    /**
-     * @return JDK marshaller.
-     */
-    Marshaller jdkMarshaller() {
-        return jdkMarshaller;
-    }
-
-    /**
-     * Returns marshaller.
-     *
-     * @param ses Session.
-     * @return Marshaller.
-     */
-    GridClientMarshaller marshaller(GridNioSession ses) {
-        GridClientMarshaller marsh = ses.meta(MARSHALLER.ordinal());
-
-        assert marsh != null;
-
-        return marsh;
-    }
-
     /** {@inheritDoc} */
     @Override public String name() {
         return "TCP binary";
@@ -110,7 +81,7 @@ GridClientMarshaller marshaller(GridNioSession ses) {
 
         lsnr = new GridTcpRestNioListener(log, this, hnd, ctx);
 
-        GridNioParser parser = new GridTcpRestParser(false);
+        GridNioParser parser = new GridTcpRestParser(false, ctx.marshallerContext().jdkMarshaller());
 
         try {
             host = resolveRestTcpHost(ctx.config());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index c5225725bf42e..6e295dda48a7c 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -8504,6 +8504,18 @@ public static long safeAbs(long i) {
      * @throws ClassNotFoundException If class not found.
      */
     public static Class forName(String clsName, @Nullable ClassLoader ldr) throws ClassNotFoundException {
+        return U.forName(clsName, ldr, null);
+    }
+
+    /**
+     * Gets class for provided name. Accepts primitive types names.
+     *
+     * @param clsName Class name.
+     * @param ldr Class loader.
+     * @return Class.
+     * @throws ClassNotFoundException If class not found.
+     */
+    public static Class forName(String clsName, @Nullable ClassLoader ldr, IgnitePredicate clsFilter) throws ClassNotFoundException {
         assert clsName != null;
 
         Class cls = primitiveMap.get(clsName);
@@ -8530,6 +8542,9 @@ public static Class forName(String clsName, @Nullable ClassLoader ldr) throws
         cls = ldrMap.get(clsName);
 
         if (cls == null) {
+            if (clsFilter != null && !clsFilter.apply(clsName))
+                throw new RuntimeException("Deserialization of class " + clsName + " is disallowed.");
+
             Class old = ldrMap.putIfAbsent(clsName, cls = Class.forName(clsName, true, ldr));
 
             if (old != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/MarshallerContext.java b/modules/core/src/main/java/org/apache/ignite/marshaller/MarshallerContext.java
index 8d6cd77df2920..987e999fc1e55 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/MarshallerContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/MarshallerContext.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.marshaller;
 
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 
 /**
  * Marshaller context.
@@ -83,4 +85,16 @@ public interface MarshallerContext {
      * @return {@code true} if the type is a system one, {@code false} otherwise.
      */
     public boolean isSystemType(String typeName);
+
+    /**
+     * Returns class name filter.
+     *
+     * @return Class name filter.
+     */
+    public IgnitePredicate classNameFilter();
+
+    /**
+     * Returns JDK marshaller instance.
+     */
+    public JdkMarshaller jdkMarshaller();
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/MarshallerUtils.java b/modules/core/src/main/java/org/apache/ignite/marshaller/MarshallerUtils.java
index ad63702de00d5..bec1f57ccd9f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/MarshallerUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/MarshallerUtils.java
@@ -25,6 +25,12 @@
  * Utility marshaller methods.
  */
 public class MarshallerUtils {
+    /** Jdk class names file. */
+    public static final String JDK_CLS_NAMES_FILE = "META-INF/classnames-jdk.properties";
+
+    /** Class names file. */
+    public static final String CLS_NAMES_FILE = "META-INF/classnames.properties";
+
     /** Job sender node version. */
     private static final ThreadLocal JOB_SND_NODE_VER = new ThreadLocal<>();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshaller.java b/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshaller.java
index 6759c40bf008a..ae4033b5c30a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshaller.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.util.io.GridByteArrayOutputStream;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.marshaller.AbstractNodeNameAwareMarshaller;
 import org.jetbrains.annotations.Nullable;
 
@@ -67,6 +68,23 @@
  * For information about Spring framework visit www.springframework.org
  */
 public class JdkMarshaller extends AbstractNodeNameAwareMarshaller {
+    /** Class name filter. */
+    private final IgnitePredicate clsFilter;
+
+    /**
+     * Default constructor.
+     */
+    public JdkMarshaller() {
+        this(null);
+    }
+
+    /**
+     * @param clsFilter Class name filter.
+     */
+    public JdkMarshaller(IgnitePredicate clsFilter) {
+        this.clsFilter = clsFilter;
+    }
+
     /** {@inheritDoc} */
     @Override protected void marshal0(@Nullable Object obj, OutputStream out) throws IgniteCheckedException {
         assert out != null;
@@ -116,7 +134,7 @@ public class JdkMarshaller extends AbstractNodeNameAwareMarshaller {
         ObjectInputStream objIn = null;
 
         try {
-            objIn = new JdkMarshallerObjectInputStream(new JdkMarshallerInputStreamWrapper(in), clsLdr);
+            objIn = new JdkMarshallerObjectInputStream(new JdkMarshallerInputStreamWrapper(in), clsLdr, clsFilter);
 
             return (T)objIn.readObject();
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshallerObjectInputStream.java b/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshallerObjectInputStream.java
index 03256e0e6471c..d9fdd3d2f1c3d 100644
--- a/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshallerObjectInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/marshaller/jdk/JdkMarshallerObjectInputStream.java
@@ -22,6 +22,7 @@
 import java.io.ObjectInputStream;
 import java.io.ObjectStreamClass;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
 
 /**
  * This class defines custom JDK object input stream.
@@ -30,17 +31,21 @@ class JdkMarshallerObjectInputStream extends ObjectInputStream {
     /** */
     private final ClassLoader clsLdr;
 
+    /** Class name filter. */
+    private final IgnitePredicate clsFilter;
+
     /**
      * @param in Parent input stream.
      * @param clsLdr Custom class loader.
      * @throws IOException If initialization failed.
      */
-    JdkMarshallerObjectInputStream(InputStream in, ClassLoader clsLdr) throws IOException {
+    JdkMarshallerObjectInputStream(InputStream in, ClassLoader clsLdr, IgnitePredicate clsFilter) throws IOException {
         super(in);
 
         assert clsLdr != null;
 
         this.clsLdr = clsLdr;
+        this.clsFilter = clsFilter;
 
         enableResolveObject(true);
     }
@@ -51,7 +56,7 @@ class JdkMarshallerObjectInputStream extends ObjectInputStream {
         // Must have 'Class.forName()' instead of clsLoader.loadClass()
         // due to weird ClassNotFoundExceptions for arrays of classes
         // in certain cases.
-        return U.forName(desc.getName(), clsLdr);
+        return U.forName(desc.getName(), clsLdr, clsFilter);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 713bbab575702..51c5adf9c5366 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.configuration.AddressResolver;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
@@ -333,7 +334,7 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
     protected volatile long gridStartTime;
 
     /** Marshaller. */
-    private final Marshaller marsh = new JdkMarshaller();
+    private Marshaller marsh;
 
     /** Statistics. */
     protected final TcpDiscoveryStatistics stats = new TcpDiscoveryStatistics();
@@ -551,6 +552,11 @@ public void setClientReconnectDisabled(boolean clientReconnectDisabled) {
         if (ignite != null) {
             setLocalAddress(ignite.configuration().getLocalHost());
             setAddressResolver(ignite.configuration().getAddressResolver());
+
+            if (ignite instanceof IgniteKernal) // IgniteMock instance can be injected from tests.
+                marsh = ((IgniteKernal)ignite).context().marshallerContext().jdkMarshaller();
+            else
+                marsh = new JdkMarshaller();
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/stream/StreamAdapter.java b/modules/core/src/main/java/org/apache/ignite/stream/StreamAdapter.java
index 3f1dfadc6a0c8..49f0c1535d7b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/stream/StreamAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/stream/StreamAdapter.java
@@ -46,7 +46,7 @@ public abstract class StreamAdapter {
     private IgniteDataStreamer stmr;
 
     /** Ignite. */
-    private Ignite ignite;
+    protected Ignite ignite;
 
     /**
      * Empty constructor.
diff --git a/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java b/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
index f45423bd7e5f5..1a68206ed2b85 100644
--- a/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
+++ b/modules/core/src/main/java/org/apache/ignite/stream/socket/SocketStreamer.java
@@ -24,6 +24,7 @@
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.util.nio.GridBufferedParser;
 import org.apache.ignite.internal.util.nio.GridDelimitedParser;
 import org.apache.ignite.internal.util.nio.GridNioCodecFilter;
@@ -38,6 +39,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.marshaller.MarshallerUtils;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.stream.StreamAdapter;
 import org.apache.ignite.stream.StreamTupleExtractor;
 import org.jetbrains.annotations.Nullable;
@@ -217,7 +219,7 @@ public void stop() {
     /**
      * Converts message to Java object using Jdk marshaller.
      */
-    private static class DefaultConverter implements SocketMessageConverter {
+    private class DefaultConverter implements SocketMessageConverter {
         /** Marshaller. */
         private final Marshaller marsh;
 
@@ -227,7 +229,9 @@ private static class DefaultConverter implements SocketMessageConverter {
          * @param igniteInstanceName Ignite instance name.
          */
         private DefaultConverter(@Nullable String igniteInstanceName) {
-            marsh = MarshallerUtils.jdkMarshaller(igniteInstanceName);
+            marsh = new JdkMarshaller(((IgniteKernal)ignite).context().marshallerContext().classNameFilter());
+
+            MarshallerUtils.setNodeName(marsh, igniteInstanceName);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/config/class_list_exploit_excluded.txt b/modules/core/src/test/config/class_list_exploit_excluded.txt
new file mode 100644
index 0000000000000..d2657e8e0580c
--- /dev/null
+++ b/modules/core/src/test/config/class_list_exploit_excluded.txt
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Intentionally empty.
\ No newline at end of file
diff --git a/modules/core/src/test/config/class_list_exploit_included.txt b/modules/core/src/test/config/class_list_exploit_included.txt
new file mode 100644
index 0000000000000..9a07d541cbfd0
--- /dev/null
+++ b/modules/core/src/test/config/class_list_exploit_included.txt
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.ignite.spi.discovery.tcp.DiscoveryUnmarshalVulnerabilityTest$Exploit
+org.apache.ignite.stream.socket.SocketStreamerUnmarshalVulnerabilityTest$Exploit
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClassSetTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClassSetTest.java
new file mode 100644
index 0000000000000..c51957a0e7082
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClassSetTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import junit.framework.TestCase;
+
+/**
+ * Tests for {@link ClassSet} class.
+ */
+public class ClassSetTest extends TestCase {
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAddAndContains() throws Exception {
+        ClassSet clsSet = new ClassSet();
+
+        clsSet.add("org.apache.ignite.Ignite");
+
+        assertTrue(clsSet.contains("org.apache.ignite.Ignite"));
+        assertFalse(clsSet.contains("org.apache.ignite.NotIgnite"));
+        assertFalse(clsSet.contains("org.apache.Ignite"));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAddWithMaskAndContains() throws Exception {
+        ClassSet clsSet = new ClassSet();
+
+        clsSet.add("org.apache.ignite.*");
+
+        assertTrue(clsSet.contains("org.apache.ignite.Ignite"));
+        assertTrue(clsSet.contains("org.apache.ignite.NotIgnite"));
+        assertFalse(clsSet.contains("org.apache.Ignite"));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReduceOnAddWithMask() throws Exception {
+        ClassSet clsSet = new ClassSet();
+
+        clsSet.add("org.apache.ignite.Ignite");
+        clsSet.add("org.apache.ignite.Ignition");
+
+        assertTrue(clsSet.contains("org.apache.ignite.Ignite"));
+        assertTrue(clsSet.contains("org.apache.ignite.Ignition"));
+        assertFalse(clsSet.contains("org.apache.ignite.NotIgnite"));
+
+        clsSet.add("org.apache.ignite.*");
+
+        assertTrue(clsSet.contains("org.apache.ignite.Ignite"));
+        assertTrue(clsSet.contains("org.apache.ignite.Ignition"));
+        assertTrue(clsSet.contains("org.apache.ignite.NotIgnite"));
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/MarshallerContextLockingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/MarshallerContextLockingSelfTest.java
index 522b9419f6c5b..f31a56da46975 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/MarshallerContextLockingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/MarshallerContextLockingSelfTest.java
@@ -140,7 +140,7 @@ public static class InternalExecutor {
         public void executeTest(GridTestLog4jLogger log, GridKernalContext ctx) throws Exception {
             counter.incrementAndGet();
 
-            MarshallerContextImpl marshallerContext = new MarshallerContextImpl(null);
+            MarshallerContextImpl marshallerContext = new MarshallerContextImpl(null, null);
             marshallerContext.onMarshallerProcessorStarted(ctx, null);
 
             MarshallerMappingItem item = new MarshallerMappingItem(JAVA_ID, 1, String.class.getName());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/GridBinaryMarshallerCtxDisabledSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/GridBinaryMarshallerCtxDisabledSelfTest.java
index 35aa9b9a08189..bb7c65d6d2136 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/GridBinaryMarshallerCtxDisabledSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/GridBinaryMarshallerCtxDisabledSelfTest.java
@@ -29,8 +29,10 @@
 import org.apache.ignite.binary.Binarylizable;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.logger.NullLogger;
 import org.apache.ignite.marshaller.MarshallerContext;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -122,6 +124,16 @@ private static class MarshallerContextWithNoStorage implements MarshallerContext
         @Override public boolean isSystemType(String typeName) {
             return false;
         }
+
+        /** {@inheritDoc} */
+        @Override public IgnitePredicate classNameFilter() {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public JdkMarshaller jdkMarshaller() {
+            return new JdkMarshaller();
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryMemorySizeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryMemorySizeSelfTest.java
index f18afb2544e19..8092d9f307d1b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryMemorySizeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheEntryMemorySizeSelfTest.java
@@ -31,9 +31,10 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.marshaller.MarshallerContext;
-import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -166,6 +167,14 @@ protected Marshaller createMarshaller() throws IgniteCheckedException {
             @Override public boolean isSystemType(String typeName) {
                 return false;
             }
+
+            @Override public IgnitePredicate classNameFilter() {
+                return null;
+            }
+
+            @Override public JdkMarshaller jdkMarshaller() {
+                return new JdkMarshaller();
+            }
         });
 
         return marsh;
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextSelfTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextSelfTest.java
index 9df56ecb783e8..da474df82365e 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextSelfTest.java
@@ -72,7 +72,7 @@ public class MarshallerContextSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testClassName() throws Exception {
-        MarshallerContextImpl marshCtx = new MarshallerContextImpl(null);
+        MarshallerContextImpl marshCtx = new MarshallerContextImpl(null, null);
 
         marshCtx.onMarshallerProcessorStarted(ctx, null);
 
@@ -99,7 +99,7 @@ public void testClassName() throws Exception {
     public void testMultiplatformMappingsCollecting() throws Exception {
         String nonJavaClassName = "random.platform.Mapping";
 
-        MarshallerContextImpl marshCtx = new MarshallerContextImpl(null);
+        MarshallerContextImpl marshCtx = new MarshallerContextImpl(null, null);
 
         marshCtx.onMarshallerProcessorStarted(ctx, null);
 
@@ -157,7 +157,7 @@ public void testMultiplatformMappingsDistributing() throws Exception {
      */
     public void testOnUpdated() throws Exception {
         File workDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false);
-        MarshallerContextImpl ctx = new MarshallerContextImpl(null);
+        MarshallerContextImpl ctx = new MarshallerContextImpl(null, null);
 
         ctx.onMarshallerProcessorStarted(this.ctx, null);
 
@@ -188,7 +188,7 @@ public void testOnUpdated() throws Exception {
      * if platform ids passed to marshaller cache were not sequential (like 0, 2).
      */
     public void testCacheStructure0() throws Exception {
-        MarshallerContextImpl ctx = new MarshallerContextImpl(null);
+        MarshallerContextImpl ctx = new MarshallerContextImpl(null, null);
 
         ctx.onMarshallerProcessorStarted(this.ctx, null);
 
@@ -221,7 +221,7 @@ public void testCacheStructure0() throws Exception {
      * if platform ids passed to marshaller context were sequential.
      */
     public void testCacheStructure1() throws Exception {
-        MarshallerContextImpl ctx = new MarshallerContextImpl(null);
+        MarshallerContextImpl ctx = new MarshallerContextImpl(null, null);
 
         ctx.onMarshallerProcessorStarted(this.ctx, null);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextTestImpl.java b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextTestImpl.java
index 0a59bbaf25041..2d6383c409c3e 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextTestImpl.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextTestImpl.java
@@ -43,7 +43,7 @@ public class MarshallerContextTestImpl extends MarshallerContextImpl {
      * @param excluded Excluded classes.
      */
     public MarshallerContextTestImpl(@Nullable List plugins, Collection excluded) {
-        super(plugins);
+        super(plugins, null);
 
         this.excluded = excluded;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/DiscoveryUnmarshalVulnerabilityTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/DiscoveryUnmarshalVulnerabilityTest.java
new file mode 100644
index 0000000000000..448c9af9223cd
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/DiscoveryUnmarshalVulnerabilityTest.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.Socket;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_MARSHALLER_BLACKLIST;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_MARSHALLER_WHITELIST;
+
+/**
+ * Tests for whitelist and blacklist ot avoiding deserialization vulnerability.
+ */
+public class DiscoveryUnmarshalVulnerabilityTest extends GridCommonAbstractTest {
+    /** Marshaller. */
+    private static final JdkMarshaller MARSH = new JdkMarshaller();
+
+    /** Shared value. */
+    private static final AtomicBoolean SHARED = new AtomicBoolean();
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        SHARED.set(false);
+
+        System.clearProperty(IGNITE_MARSHALLER_WHITELIST);
+        System.clearProperty(IGNITE_MARSHALLER_BLACKLIST);
+
+        IgniteUtils.clearClassCache();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoLists() throws Exception {
+        testExploit(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWhiteListIncluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_included.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_WHITELIST, path);
+
+        testExploit(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWhiteListExcluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_excluded.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_WHITELIST, path);
+
+        testExploit(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBlackListIncluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_included.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_BLACKLIST, path);
+
+        testExploit(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBlackListExcluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_excluded.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_BLACKLIST, path);
+
+        testExploit(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBothListIncluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_included.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_WHITELIST, path);
+        System.setProperty(IGNITE_MARSHALLER_BLACKLIST, path);
+
+        testExploit(false);
+    }
+
+    /**
+     * @param positive Positive.
+     */
+    private void testExploit(boolean positive) throws Exception {
+        try {
+            startGrid();
+
+            attack(marshal(new Exploit()));
+
+            boolean res = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return SHARED.get();
+                }
+            }, 3000L);
+
+            if (positive)
+                assertTrue(res);
+            else
+                assertFalse(res);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @param obj Object.
+     */
+    private static byte[] marshal(Object obj) throws IgniteCheckedException {
+        return MARSH.marshal(obj);
+    }
+
+    /**
+     * @param data Data.
+     */
+    private void attack(byte[] data) throws IOException {
+        InetAddress addr = InetAddress.getLoopbackAddress();
+
+        try (
+            Socket sock = new Socket(addr, 47500);
+            OutputStream oos = new BufferedOutputStream(sock.getOutputStream())
+        ) {
+            oos.write(U.IGNITE_HEADER);
+            oos.write(data);
+        }
+    }
+
+    /** */
+    private static class Exploit implements Serializable {
+        /**
+         * @param is Input stream.
+         */
+        private void readObject(ObjectInputStream is) throws ClassNotFoundException, IOException {
+            SHARED.set(true);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/stream/socket/SocketStreamerUnmarshalVulnerabilityTest.java b/modules/core/src/test/java/org/apache/ignite/stream/socket/SocketStreamerUnmarshalVulnerabilityTest.java
new file mode 100644
index 0000000000000..dadc5b6f31c19
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/stream/socket/SocketStreamerUnmarshalVulnerabilityTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.stream.socket;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.apache.ignite.stream.StreamSingleTupleExtractor;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_MARSHALLER_BLACKLIST;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_MARSHALLER_WHITELIST;
+
+/**
+ * Tests for whitelist and blacklist ot avoiding deserialization vulnerability.
+ */
+public class SocketStreamerUnmarshalVulnerabilityTest extends GridCommonAbstractTest {
+    /** Shared value. */
+    private static final AtomicBoolean SHARED = new AtomicBoolean();
+
+    /** Port. */
+    private static int port;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        try (ServerSocket sock = new ServerSocket(0)) {
+            port = sock.getLocalPort();
+        }
+
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setCacheConfiguration(defaultCacheConfiguration());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        SHARED.set(false);
+
+        System.clearProperty(IGNITE_MARSHALLER_WHITELIST);
+        System.clearProperty(IGNITE_MARSHALLER_BLACKLIST);
+
+        IgniteUtils.clearClassCache();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoLists() throws Exception {
+        testExploit(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWhiteListIncluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_included.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_WHITELIST, path);
+
+        testExploit(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWhiteListExcluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_excluded.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_WHITELIST, path);
+
+        testExploit(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBlackListIncluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_included.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_BLACKLIST, path);
+
+        testExploit(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBlackListExcluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_excluded.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_BLACKLIST, path);
+
+        testExploit(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBothListIncluded() throws Exception {
+        String path = U.resolveIgnitePath("modules/core/src/test/config/class_list_exploit_included.txt").getPath();
+
+        System.setProperty(IGNITE_MARSHALLER_WHITELIST, path);
+        System.setProperty(IGNITE_MARSHALLER_BLACKLIST, path);
+
+        testExploit(false);
+    }
+
+    /**
+     * @param positive Positive.
+     */
+    private void testExploit(boolean positive) throws Exception {
+        try {
+            Ignite ignite = startGrid();
+
+            SocketStreamer sockStmr = null;
+
+            try (IgniteDataStreamer stmr = ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
+                stmr.allowOverwrite(true);
+                stmr.autoFlushFrequency(10);
+
+                sockStmr = new SocketStreamer<>();
+
+                sockStmr.setIgnite(ignite);
+
+                sockStmr.setStreamer(stmr);
+
+                sockStmr.setPort(port);
+
+                sockStmr.setSingleTupleExtractor(new StreamSingleTupleExtractor() {
+                    @Override public Map.Entry extract(Exploit msg) {
+                        return new IgniteBiTuple<>(1, "val");
+                    }
+                });
+
+                sockStmr.start();
+
+                try (Socket sock = new Socket(InetAddress.getLocalHost(), port);
+                     OutputStream os = new BufferedOutputStream(sock.getOutputStream())) {
+                    Marshaller marsh = new JdkMarshaller();
+
+                    byte[] msg = marsh.marshal(new Exploit());
+
+                    os.write(msg.length >>> 24);
+                    os.write(msg.length >>> 16);
+                    os.write(msg.length >>> 8);
+                    os.write(msg.length);
+
+                    os.write(msg);
+                }
+                catch (IOException | IgniteCheckedException e) {
+                    throw new IgniteException(e);
+                }
+
+                boolean res = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                    @Override public boolean apply() {
+                        return SHARED.get();
+                    }
+                }, 3000L);
+
+                if (positive)
+                    assertTrue(res);
+                else
+                    assertFalse(res);
+            }
+            finally {
+                if (sockStmr != null)
+                    sockStmr.stop();
+            }
+        }
+        finally {
+                stopAllGrids();
+        }
+    }
+
+    /** */
+    private static class Exploit implements Serializable {
+        /**
+         * @param is Input stream.
+         */
+        private void readObject(ObjectInputStream is) throws ClassNotFoundException, IOException {
+            SHARED.set(true);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
index 6b39faaf01992..5b853856bcde5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
@@ -78,7 +78,8 @@ public GridTestKernalContext(IgniteLogger log, IgniteConfiguration cfg) {
                 null,
                 null,
                 null,
-                U.allPluginProviders()
+                U.allPluginProviders(),
+                null
         );
 
         GridTestUtils.setFieldValue(grid(), "cfg", config());
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 1b4e2da47b669..9fee224ba0930 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -20,6 +20,7 @@
 import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.GridSuppressedExceptionSelfTest;
+import org.apache.ignite.internal.ClassSetTest;
 import org.apache.ignite.internal.ClusterGroupHostsSelfTest;
 import org.apache.ignite.internal.ClusterGroupSelfTest;
 import org.apache.ignite.internal.GridFailFastNodeFailureDetectionSelfTest;
@@ -191,6 +192,8 @@ public static TestSuite suite(@Nullable final Set ignoredTests) throws Ex
 
         suite.addTestSuite(GridCleanerTest.class);
 
+        suite.addTestSuite(ClassSetTest.class);
+
         return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 626875c57ceeb..6e51c36b0fd80 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.spi.GridTcpSpiForwardingSelfTest;
 import org.apache.ignite.spi.discovery.AuthenticationRestartTest;
 import org.apache.ignite.spi.discovery.IgniteDiscoveryCacheReuseSelfTest;
+import org.apache.ignite.spi.discovery.tcp.DiscoveryUnmarshalVulnerabilityTest;
 import org.apache.ignite.spi.discovery.tcp.IgniteClientConnectTest;
 import org.apache.ignite.spi.discovery.tcp.IgniteClientReconnectMassiveShutdownTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelfTest;
@@ -111,6 +112,8 @@ public static TestSuite suite() throws Exception {
         // Disco cache reuse.
         suite.addTest(new TestSuite(IgniteDiscoveryCacheReuseSelfTest.class));
 
+        suite.addTest(new TestSuite(DiscoveryUnmarshalVulnerabilityTest.class));
+
         return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStreamSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStreamSelfTestSuite.java
index d7c677c854caf..9eac2773166f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStreamSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStreamSelfTestSuite.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.stream.socket.SocketStreamerSelfTest;
+import org.apache.ignite.stream.socket.SocketStreamerUnmarshalVulnerabilityTest;
 
 /**
  * Stream test suite.
@@ -32,6 +33,7 @@ public static TestSuite suite() throws Exception {
         TestSuite suite = new TestSuite("Ignite Stream Test Suite");
 
         suite.addTest(new TestSuite(SocketStreamerSelfTest.class));
+        suite.addTest(new TestSuite(SocketStreamerUnmarshalVulnerabilityTest.class));
 
         return suite;
     }

From cde2176a744c8622044a3802e8b2503d071efe3a Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Thu, 25 Jan 2018 17:10:09 +0300
Subject: [PATCH 50/65] Revert "IGNITE-7514 Choose owner primary node after
 cluster restart"

This reverts commit 8484c191e34c6e835d9e2e499d76866e0aaf10b9
---
 .../internal/events/DiscoveryCustomEvent.java |  34 ----
 .../cache/CacheAffinitySharedManager.java     |  76 +++-----
 .../processors/cache/ClusterCachesInfo.java   |   1 -
 .../GridCachePartitionExchangeManager.java    |   9 +-
 .../dht/GridClientPartitionTopology.java      |   9 +-
 .../dht/GridDhtLocalPartition.java            |   3 +
 .../dht/GridDhtPartitionTopology.java         |  10 +-
 .../dht/GridDhtPartitionTopologyImpl.java     |  47 +----
 .../GridDhtPartitionsExchangeFuture.java      | 134 ++++++--------
 .../GridCacheDatabaseSharedManager.java       |   4 -
 .../CacheBaselineTopologyTest.java            | 166 +-----------------
 11 files changed, 85 insertions(+), 408 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
index 3b12b384f54be..b3c6a2d865b2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
@@ -21,10 +21,7 @@
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
-import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.jetbrains.annotations.Nullable;
 
 /**
  * Custom event.
@@ -88,35 +85,4 @@ public void affinityTopologyVersion(AffinityTopologyVersion affTopVer) {
     @Override public String toString() {
         return S.toString(DiscoveryCustomEvent.class, this, super.toString());
     }
-
-    /**
-     * @param evt Discovery event.
-     * @return {@code True} if event is DiscoveryCustomEvent that requires centralized affinity assignment.
-     */
-    public static boolean requiresCentralizedAffinityAssignment(DiscoveryEvent evt) {
-        if (!(evt instanceof DiscoveryCustomEvent))
-            return false;
-
-        return requiresCentralizedAffinityAssignment(((DiscoveryCustomEvent)evt).customMessage());
-    }
-
-    /**
-     * @param msg Discovery custom message.
-     * @return {@code True} if message belongs to event that requires centralized affinity assignment.
-     */
-    public static boolean requiresCentralizedAffinityAssignment(@Nullable DiscoveryCustomMessage msg) {
-        if (msg == null)
-            return false;
-
-        if (msg instanceof ChangeGlobalStateMessage && ((ChangeGlobalStateMessage)msg).activate())
-            return true;
-
-        if (msg instanceof SnapshotDiscoveryMessage) {
-            SnapshotDiscoveryMessage snapMsg = (SnapshotDiscoveryMessage) msg;
-
-            return snapMsg.needExchange() && snapMsg.needAssignPartitions();
-        }
-
-        return false;
-    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 9eff56097e939..4119f2357c896 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -40,7 +40,6 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
-import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
@@ -58,6 +57,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
+import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
@@ -161,15 +161,13 @@ void onDiscoveryEvent(int type,
         ClusterNode node,
         AffinityTopologyVersion topVer,
         DiscoveryDataClusterState state) {
-        if ((state.transition() || !state.active()) &&
-            !DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(customMsg))
+        if (state.transition() || !state.active())
             return;
 
         if (type == EVT_NODE_JOINED && node.isLocal())
             lastAffVer = null;
 
-        if ((!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) ||
-            DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(customMsg)) {
+        if (!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) {
             synchronized (mux) {
                 assert lastAffVer == null || topVer.compareTo(lastAffVer) > 0;
 
@@ -1265,7 +1263,7 @@ public GridAffinityAssignmentCache affinity(Integer grpId) {
      * @param fut Current exchange future.
      * @param msg Finish exchange message.
      */
-    public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture fut,
+    public void mergeExchangesOnServerLeft(final GridDhtPartitionsExchangeFuture fut,
         final GridDhtPartitionsFullMessage msg) {
         final Map nodesByOrder = new HashMap<>();
 
@@ -1398,7 +1396,7 @@ public void onServerJoinWithExchangeMergeProtocol(GridDhtPartitionsExchangeFutur
      * @return Computed difference with ideal affinity.
      * @throws IgniteCheckedException If failed.
      */
-    public Map onServerLeftWithExchangeMergeProtocol(
+    public  Map onServerLeftWithExchangeMergeProtocol(
         final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
     {
         final ExchangeDiscoveryEvents evts = fut.context().events();
@@ -1406,32 +1404,6 @@ public Map onServerLeftWithExchangeMergeProt
         assert fut.context().mergeExchanges();
         assert evts.hasServerLeft();
 
-        return onReassignmentEnforced(fut);
-    }
-
-    /**
-     * @param fut Current exchange future.
-     * @return Computed difference with ideal affinity.
-     * @throws IgniteCheckedException If failed.
-     */
-    public Map onCustomEventWithEnforcedAffinityReassignment(
-        final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
-    {
-        assert DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent());
-
-        return onReassignmentEnforced(fut);
-    }
-
-    /**
-     * @param fut Current exchange future.
-     * @return Computed difference with ideal affinity.
-     * @throws IgniteCheckedException If failed.
-     */
-    public Map onReassignmentEnforced(
-        final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
-    {
-        final ExchangeDiscoveryEvents evts = fut.context().events();
-
         forAllRegisteredCacheGroups(new IgniteInClosureX() {
             @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException {
                 AffinityTopologyVersion topVer = evts.topologyVersion();
@@ -1446,7 +1418,7 @@ public Map onReassignmentEnforced(
             }
         });
 
-        Map>> diff = initAffinityBasedOnPartitionsAvailability(evts.topologyVersion(),
+        Map>> diff = initAffinityOnNodeLeft0(evts.topologyVersion(),
             fut,
             NODE_TO_ORDER,
             true);
@@ -1670,16 +1642,17 @@ private GridDhtAffinityAssignmentResponse fetchAffinity(AffinityTopologyVersion
     }
 
     /**
-     * Called on exchange initiated by server node leave or custom event with centralized affinity assignment.
+     * Called on exchange initiated by server node leave.
      *
      * @param fut Exchange future.
      * @param crd Coordinator flag.
      * @throws IgniteCheckedException If failed.
      * @return {@code True} if affinity should be assigned by coordinator.
      */
-    public boolean onCentralizedAffinityChange(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
-        assert (fut.events().hasServerLeft() && !fut.firstEvent().eventNode().isClient()) ||
-            DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent()) : fut.firstEvent();
+    public boolean onServerLeft(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
+        ClusterNode leftNode = fut.firstEvent().eventNode();
+
+        assert !leftNode.isClient() : leftNode;
 
         if (crd) {
             // Need initialize CacheGroupHolders if this node become coordinator on this exchange.
@@ -2093,7 +2066,7 @@ public IgniteInternalFuture>>> initAffinity
             initFut.listen(new IgniteInClosure>() {
                 @Override public void apply(IgniteInternalFuture initFut) {
                     try {
-                        resFut.onDone(initAffinityBasedOnPartitionsAvailability(fut.initialVersion(), fut, NODE_TO_ID, false));
+                        resFut.onDone(initAffinityOnNodeLeft0(fut.initialVersion(), fut, NODE_TO_ID, false));
                     }
                     catch (IgniteCheckedException e) {
                         resFut.onDone(e);
@@ -2104,7 +2077,7 @@ public IgniteInternalFuture>>> initAffinity
             return resFut;
         }
         else
-            return new GridFinishedFuture<>(initAffinityBasedOnPartitionsAvailability(fut.initialVersion(), fut, NODE_TO_ID, false));
+            return new GridFinishedFuture<>(initAffinityOnNodeLeft0(fut.initialVersion(), fut, NODE_TO_ID, false));
     }
 
     /**
@@ -2115,17 +2088,12 @@ public IgniteInternalFuture>>> initAffinity
      * @return Affinity assignment.
      * @throws IgniteCheckedException If failed.
      */
-    private  Map>> initAffinityBasedOnPartitionsAvailability(final AffinityTopologyVersion topVer,
+    private  Map>> initAffinityOnNodeLeft0(final AffinityTopologyVersion topVer,
         final GridDhtPartitionsExchangeFuture fut,
         final IgniteClosure c,
         final boolean initAff)
         throws IgniteCheckedException {
-        final boolean enforcedCentralizedAssignment =
-            DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent());
-
-        final WaitRebalanceInfo waitRebalanceInfo = enforcedCentralizedAssignment ?
-            new WaitRebalanceInfo(fut.exchangeId().topologyVersion()) :
-            new WaitRebalanceInfo(fut.context().events().lastServerEventVersion());
+        final WaitRebalanceInfo waitRebalanceInfo = new WaitRebalanceInfo(fut.context().events().lastServerEventVersion());
 
         final Collection aliveNodes = fut.context().events().discoveryCache().serverNodes();
 
@@ -2135,14 +2103,13 @@ private  Map>> initAffinityBasedOnPartitionsAva
             @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException {
                 CacheGroupHolder grpHolder = groupHolder(topVer, desc);
 
-                if (!grpHolder.rebalanceEnabled ||
-                    (fut.cacheGroupAddedOnExchange(desc.groupId(), desc.receivedFrom()) && !enforcedCentralizedAssignment))
+                if (!grpHolder.rebalanceEnabled || fut.cacheGroupAddedOnExchange(desc.groupId(), desc.receivedFrom()))
                     return;
 
                 AffinityTopologyVersion affTopVer = grpHolder.affinity().lastVersion();
 
-                assert (affTopVer.topologyVersion() > 0 && !affTopVer.equals(topVer)) || enforcedCentralizedAssignment :
-                    "Invalid affinity version [last=" + affTopVer + ", futVer=" + topVer + ", grp=" + desc.cacheOrGroupName() + ']';
+                assert affTopVer.topologyVersion() > 0 && !affTopVer.equals(topVer) : "Invalid affinity version " +
+                    "[last=" + affTopVer + ", futVer=" + topVer + ", grp=" + desc.cacheOrGroupName() + ']';
 
                 List> curAssignment = grpHolder.affinity().assignments(affTopVer);
                 List> newAssignment = grpHolder.affinity().idealAssignment();
@@ -2174,11 +2141,6 @@ private  Map>> initAffinityBasedOnPartitionsAva
                         ", node=" + newPrimary +
                         ", topVer=" + topVer + ']';
 
-                    List owners = top.owners(p);
-
-                    if (!owners.isEmpty() && !owners.contains(curPrimary))
-                        curPrimary = owners.get(0);
-
                     if (curPrimary != null && newPrimary != null && !curPrimary.equals(newPrimary)) {
                         if (aliveNodes.contains(curPrimary)) {
                             GridDhtPartitionState state = top.partitionState(newPrimary.id(), p);
@@ -2211,6 +2173,8 @@ private  Map>> initAffinityBasedOnPartitionsAva
                                 }
 
                                 if (newNodes0 == null) {
+                                    List owners = top.owners(p);
+
                                     for (ClusterNode owner : owners) {
                                         if (aliveNodes.contains(owner)) {
                                             newNodes0 = latePrimaryAssignment(grpHolder.affinity(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 2b2fb559c182e..08a910b81603d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -1186,7 +1186,6 @@ public void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) {
     /**
      * @param msg Message.
      * @param topVer Current topology version.
-     * @param curState Current cluster state.
      * @return Exchange action.
      * @throws IgniteCheckedException If configuration validation failed.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 0a657c9cdf779..9b9284f04b10a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -253,13 +253,6 @@ public class GridCachePartitionExchangeManager extends GridCacheSharedMana
 
                     return;
                 }
-                if (evt.type() == EVT_DISCOVERY_CUSTOM_EVT &&
-                    (((DiscoveryCustomEvent)evt).customMessage() instanceof CacheAffinityChangeMessage) &&
-                    ((CacheAffinityChangeMessage)((DiscoveryCustomEvent)evt).customMessage()).exchangeId() != null) {
-                    onDiscoveryEvent(evt, cache);
-
-                    return;
-                }
 
                 if (cache.state().transition()) {
                     if (log.isDebugEnabled())
@@ -972,7 +965,7 @@ public void scheduleResendPartitions() {
      * For coordinator causes {@link GridDhtPartitionsFullMessage FullMessages} send,
      * for non coordinator -  {@link GridDhtPartitionsSingleMessage SingleMessages} send
      */
-    public void refreshPartitions() {
+    private void refreshPartitions() {
         // TODO https://issues.apache.org/jira/browse/IGNITE-6857
         if (cctx.snapshot().snapshotOperationInProgress()) {
             scheduleResendPartitions();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index def00f3f9ae89..e994113087ec8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -256,9 +256,9 @@ private String mapString(GridDhtPartitionMap map) {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
+    @Override public void initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
         GridDhtPartitionsExchangeFuture exchFut) {
-        return false;
+        // No-op.
     }
 
     /** {@inheritDoc} */
@@ -382,11 +382,6 @@ else if (!node2part.nodeId().equals(loc.id())) {
                 fullMapString() + ']');
     }
 
-    /** {@inheritDoc} */
-    @Override public void afterStateRestored(AffinityTopologyVersion topVer) {
-        // no-op
-    }
-
     /** {@inheritDoc} */
     @Override public boolean afterExchange(GridDhtPartitionsExchangeFuture exchFut) throws IgniteCheckedException {
         AffinityTopologyVersion topVer = exchFut.topologyVersion();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index e63aab6c0dc78..e1f1d6f43ce1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -213,6 +213,9 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
             // TODO ignite-db
             throw new IgniteException(e);
         }
+
+        // Todo log moving state
+        casState(state.get(), MOVING);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 13564c2af2666..4ae68ef739f8c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -117,19 +117,11 @@ public void beforeExchange(GridDhtPartitionsExchangeFuture exchFut,
     /**
      * @param affVer Affinity version.
      * @param exchFut Exchange future.
-     * @return {@code True} if partitions must be refreshed.
      * @throws IgniteInterruptedCheckedException If interrupted.
      */
-    public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut)
+    public void initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut)
         throws IgniteInterruptedCheckedException;
 
-    /**
-     * Initializes local data structures after partitions are restored from persistence.
-     *
-     * @param topVer Topology version.
-     */
-    public void afterStateRestored(AffinityTopologyVersion topVer);
-
     /**
      * Post-initializes this topology.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index f3da42a62d6bd..0a2c1541bd70d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -299,12 +299,10 @@ private String mapString(GridDhtPartitionMap map) {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
+    @Override public void initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
         GridDhtPartitionsExchangeFuture exchFut)
         throws IgniteInterruptedCheckedException
     {
-        boolean needRefresh;
-
         ctx.database().checkpointReadLock();
 
         try {
@@ -312,11 +310,11 @@ private String mapString(GridDhtPartitionMap map) {
 
             try {
                 if (stopping)
-                    return false;
+                    return;
 
                 long updateSeq = this.updateSeq.incrementAndGet();
 
-                needRefresh = initPartitions0(affVer, exchFut, updateSeq);
+                initPartitions0(affVer, exchFut, updateSeq);
 
                 consistencyCheck();
             }
@@ -327,21 +325,16 @@ private String mapString(GridDhtPartitionMap map) {
         finally {
             ctx.database().checkpointReadUnlock();
         }
-
-        return needRefresh;
     }
 
     /**
      * @param affVer Affinity version to use.
      * @param exchFut Exchange future.
      * @param updateSeq Update sequence.
-     * @return {@code True} if partitions must be refreshed.
      */
-    private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
+    private void initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
         List> aff = grp.affinity().readyAssignments(affVer);
 
-        boolean needRefresh = false;
-
         if (grp.affinityNode()) {
             ClusterNode loc = ctx.localNode();
 
@@ -373,8 +366,6 @@ private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartition
                                 GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)grp.shared().database();
 
                                 locPart.restoreState(db.readPartitionState(grp, locPart.id()));
-
-                                needRefresh = true;
                             }
                             else {
                                 boolean owned = locPart.own();
@@ -432,8 +423,6 @@ else if (belongs) {
         }
 
         updateRebalanceVersion(aff);
-
-        return needRefresh;
     }
 
     /**
@@ -627,30 +616,6 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
         return grp.affinity().nodes(p, topVer).contains(ctx.localNode());
     }
 
-    /** {@inheritDoc} */
-    @Override public void afterStateRestored(AffinityTopologyVersion topVer) {
-        lock.writeLock().lock();
-
-        try {
-            if (node2part == null)
-                return;
-
-            long updateSeq = this.updateSeq.incrementAndGet();
-
-            for (int p = 0; p < grp.affinity().partitions(); p++) {
-                GridDhtLocalPartition locPart = locParts.get(p);
-
-                if (locPart == null)
-                    updateLocal(p, EVICTED, updateSeq, topVer);
-                else
-                    updateLocal(p, locPart.state(), updateSeq, topVer);
-            }
-        }
-        finally {
-            lock.writeLock().unlock();
-        }
-    }
-
     /** {@inheritDoc} */
     @Override public boolean afterExchange(GridDhtPartitionsExchangeFuture exchFut) {
         boolean changed = false;
@@ -1031,11 +996,9 @@ else if (loc != null && state == RENTING && !showRenting) {
                 map.put(i, part.state());
             }
 
-            GridDhtPartitionMap locPartMap = node2part != null ? node2part.get(ctx.localNodeId()) : null;
-
             return new GridDhtPartitionMap(ctx.localNodeId(),
                 updateSeq.get(),
-                locPartMap != null ? locPartMap.topologyVersion() : readyTopVer,
+                readyTopVer,
                 map,
                 true);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 9a247e12cec93..6c09b6a4373df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -227,12 +227,6 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
      */
     private boolean centralizedAff;
 
-    /**
-     * Enforce affinity reassignment based on actual partition distribution. This mode should be used when partitions
-     * might be distributed not according to affinity assignment.
-     */
-    private boolean forceAffReassignment;
-
     /** Change global state exception. */
     private Exception changeGlobalStateE;
 
@@ -596,8 +590,6 @@ public void init(boolean newCrd) throws IgniteInterruptedCheckedException {
 
                 DiscoveryCustomMessage msg = ((DiscoveryCustomEvent)firstDiscoEvt).customMessage();
 
-                forceAffReassignment = DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(msg);
-
                 if (msg instanceof ChangeGlobalStateMessage) {
                     assert exchActions != null && !exchActions.empty();
 
@@ -619,9 +611,6 @@ else if (msg instanceof WalStateAbstractMessage)
                     exchange = onAffinityChangeRequest(crdNode);
                 }
 
-                if (forceAffReassignment)
-                    cctx.affinity().onCentralizedAffinityChange(this, crdNode);
-
                 initCoordinatorCaches(newCrd);
             }
             else {
@@ -767,7 +756,7 @@ private void initTopologies() throws IgniteCheckedException {
                     if (grp.isLocal())
                         continue;
 
-                    grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
+                    grp.topology().beforeExchange(this, !centralizedAff, false);
                 }
             }
         }
@@ -910,6 +899,8 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
         else if (req.activate()) {
             // TODO: BLT changes on inactive cluster can't be handled easily because persistent storage hasn't been initialized yet.
             try {
+                cctx.affinity().onBaselineTopologyChanged(this, crd);
+
                 if (CU.isPersistenceEnabled(cctx.kernalContext().config()) && !cctx.kernalContext().clientNode())
                     cctx.kernalContext().state().onBaselineTopologyChanged(req.baselineTopology(),
                         req.prevBaselineTopologyHistoryItem());
@@ -946,8 +937,7 @@ private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedExcep
      * @return Exchange type.
      */
     private ExchangeType onCustomMessageNoAffinityChange(boolean crd) {
-        if (!forceAffReassignment)
-            cctx.affinity().onCustomMessageNoAffinityChange(this, crd, exchActions);
+        cctx.affinity().onCustomMessageNoAffinityChange(this, crd, exchActions);
 
         return cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL;
     }
@@ -1002,7 +992,7 @@ private ExchangeType onServerNodeEvent(boolean crd) throws IgniteCheckedExceptio
 
             exchCtx.events().warnNoAffinityNodes(cctx);
 
-            centralizedAff = cctx.affinity().onCentralizedAffinityChange(this, crd);
+            centralizedAff = cctx.affinity().onServerLeft(this, crd);
         }
         else
             cctx.affinity().onServerJoin(this, crd);
@@ -1082,7 +1072,7 @@ private void distributedExchange() throws IgniteCheckedException {
 
                 // It is possible affinity is not initialized yet if node joins to cluster.
                 if (grp.affinity().lastVersion().topologyVersion() > 0)
-                    grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
+                    grp.topology().beforeExchange(this, !centralizedAff, false);
             }
         }
 
@@ -1519,24 +1509,19 @@ public void finishMerged() {
         }
 
         if (err == null) {
-            if (centralizedAff || forceAffReassignment) {
+            if (centralizedAff) {
                 assert !exchCtx.mergeExchanges();
 
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                     if (grp.isLocal())
                         continue;
 
-                    boolean needRefresh = false;
-
                     try {
-                        needRefresh = grp.topology().initPartitionsWhenAffinityReady(res, this);
+                        grp.topology().initPartitionsWhenAffinityReady(res, this);
                     }
                     catch (IgniteInterruptedCheckedException e) {
                         U.error(log, "Failed to initialize partitions.", e);
                     }
-
-                    if (needRefresh)
-                        cctx.exchange().refreshPartitions();
                 }
             }
 
@@ -2332,7 +2317,7 @@ private void onAllReceived(@Nullable Collection sndResNodes) {
             if (!exchCtx.mergeExchanges() && !crd.equals(events().discoveryCache().serverNodes().get(0))) {
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                     if (!grp.isLocal())
-                        grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
+                        grp.topology().beforeExchange(this, !centralizedAff, false);
                 }
             }
 
@@ -2464,9 +2449,6 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
                     detectLostPartitions(resTopVer);
             }
 
-            if (!exchCtx.mergeExchanges() && forceAffReassignment)
-                idealAffDiff = cctx.affinity().onCustomEventWithEnforcedAffinityReassignment(this);
-
             for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
                 if (!grpCtx.isLocal())
                     grpCtx.topology().applyUpdateCounters();
@@ -2489,8 +2471,6 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
                 if (exchCtx.events().hasServerLeft())
                     msg.idealAffinityDiff(idealAffDiff);
             }
-            else if (forceAffReassignment)
-                msg.idealAffinityDiff(idealAffDiff);
 
             msg.prepareMarshal(cctx);
 
@@ -2544,69 +2524,65 @@ else if (forceAffReassignment)
                         nodes.addAll(sndResNodes);
                 }
 
-                if (!nodes.isEmpty())
-                    sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff);
-
-                if (!stateChangeExchange())
-                    onDone(exchCtx.events().topologyVersion(), null);
-
-                for (Map.Entry e : pendingSingleMsgs.entrySet()) {
-                    if (log.isInfoEnabled()) {
-                        log.info("Process pending message on coordinator [node=" + e.getKey() +
-                            ", ver=" + initialVersion() +
-                            ", resVer=" + resTopVer + ']');
-                    }
-
-                    processSingleMessage(e.getKey(), e.getValue());
-                }
-            }
-
-            if (stateChangeExchange()) {
                 IgniteCheckedException err = null;
 
-                StateChangeRequest req = exchActions.stateChangeRequest();
+                if (stateChangeExchange()) {
+                    StateChangeRequest req = exchActions.stateChangeRequest();
 
-                assert req != null : exchActions;
+                    assert req != null : exchActions;
 
-                boolean stateChangeErr = false;
+                    boolean stateChangeErr = false;
 
-                if (!F.isEmpty(changeGlobalStateExceptions)) {
-                    stateChangeErr = true;
+                    if (!F.isEmpty(changeGlobalStateExceptions)) {
+                        stateChangeErr = true;
 
-                    err = new IgniteCheckedException("Cluster state change failed.");
+                        err = new IgniteCheckedException("Cluster state change failed.");
 
-                    cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req);
-                }
-                else {
-                    boolean hasMoving = !partsToReload.isEmpty();
+                        cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req);
+                    }
+                    else {
+                        boolean hasMoving = !partsToReload.isEmpty();
 
-                    Set waitGrps = cctx.affinity().waitGroups();
+                        Set waitGrps = cctx.affinity().waitGroups();
 
-                    if (!hasMoving) {
-                        for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
-                            if (waitGrps.contains(grpCtx.groupId()) && grpCtx.topology().hasMovingPartitions()) {
-                                hasMoving = true;
+                        if (!hasMoving) {
+                            for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
+                                if (waitGrps.contains(grpCtx.groupId()) && grpCtx.topology().hasMovingPartitions()) {
+                                    hasMoving = true;
 
-                                break;
-                            }
+                                    break;
+                                }
 
+                            }
                         }
+
+                        cctx.kernalContext().state().onExchangeFinishedOnCoordinator(this, hasMoving);
                     }
 
-                    cctx.kernalContext().state().onExchangeFinishedOnCoordinator(this, hasMoving);
+                    boolean active = !stateChangeErr && req.activate();
+
+                    ChangeGlobalStateFinishMessage stateFinishMsg = new ChangeGlobalStateFinishMessage(
+                        req.requestId(),
+                        active,
+                        !stateChangeErr);
+
+                    cctx.discovery().sendCustomEvent(stateFinishMsg);
                 }
 
-                boolean active = !stateChangeErr && req.activate();
+                if (!nodes.isEmpty())
+                    sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff);
 
-                ChangeGlobalStateFinishMessage stateFinishMsg = new ChangeGlobalStateFinishMessage(
-                    req.requestId(),
-                    active,
-                    !stateChangeErr);
+                onDone(exchCtx.events().topologyVersion(), err);
 
-                cctx.discovery().sendCustomEvent(stateFinishMsg);
+                for (Map.Entry e : pendingSingleMsgs.entrySet()) {
+                    if (log.isInfoEnabled()) {
+                        log.info("Process pending message on coordinator [node=" + e.getKey() +
+                            ", ver=" + initialVersion() +
+                            ", resVer=" + resTopVer + ']');
+                    }
 
-                if (!centralizedAff)
-                    onDone(exchCtx.events().topologyVersion(), err);
+                    processSingleMessage(e.getKey(), e.getValue());
+                }
             }
         }
         catch (IgniteCheckedException e) {
@@ -2949,7 +2925,7 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
                     cctx.affinity().onLocalJoin(this, msg, resTopVer);
                 else {
                     if (exchCtx.events().hasServerLeft())
-                        cctx.affinity().applyAffinityFromFullMessage(this, msg);
+                        cctx.affinity().mergeExchangesOnServerLeft(this, msg);
                     else
                         cctx.affinity().onServerJoinWithExchangeMergeProtocol(this, false);
 
@@ -2963,8 +2939,6 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
             }
             else if (localJoinExchange() && !exchCtx.fetchAffinityOnJoin())
                 cctx.affinity().onLocalJoin(this, msg, resTopVer);
-            else if (forceAffReassignment)
-                cctx.affinity().applyAffinityFromFullMessage(this, msg);
 
             updatePartitionFullMap(resTopVer, msg);
 
@@ -3074,9 +3048,6 @@ public void onAffinityChangeMessage(final ClusterNode node, final CacheAffinityC
                             crd.isLocal(),
                             msg);
 
-                        IgniteCheckedException err = !F.isEmpty(msg.partitionsMessage().getErrorsMap()) ?
-                            new IgniteCheckedException("Cluster state change failed.") : null;
-
                         if (!crd.isLocal()) {
                             GridDhtPartitionsFullMessage partsMsg = msg.partitionsMessage();
 
@@ -3084,12 +3055,9 @@ public void onAffinityChangeMessage(final ClusterNode node, final CacheAffinityC
                             assert partsMsg.lastVersion() != null : partsMsg;
 
                             updatePartitionFullMap(resTopVer, partsMsg);
-
-                            if (exchActions != null && exchActions.stateChangeRequest() != null && err != null)
-                                cctx.kernalContext().state().onStateChangeError(msg.partitionsMessage().getErrorsMap(), exchActions.stateChangeRequest());
                         }
 
-                        onDone(resTopVer, err);
+                        onDone(resTopVer);
                     }
                     else {
                         if (log.isDebugEnabled()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index f3b11ccfee0d2..f833911135b7d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -110,8 +110,6 @@
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
@@ -2303,8 +2301,6 @@ else if (restore != null) {
                     updateState(part, restore.get1());
                 }
             }
-
-            grp.topology().afterStateRestored(grp.topology().lastTopologyChangeVersion());
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 857d3a1dc3c2c..e70773d205274 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -25,16 +25,13 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
-import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.affinity.AffinityFunction;
-import org.apache.ignite.cache.affinity.AffinityFunctionContext;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.PartitionLossPolicy;
 import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -43,16 +40,9 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.TestDelayingCommunicationSpi;
-import org.apache.ignite.internal.managers.communication.GridIoMessage;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.PA;
-import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -73,9 +63,6 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
     /** */
     private static final int NODE_COUNT = 4;
 
-    /** */
-    private static boolean delayRebalance = false;
-
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
@@ -118,9 +105,6 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
         if (client)
             cfg.setClientMode(true);
 
-        if (delayRebalance)
-            cfg.setCommunicationSpi(new DelayRebalanceCommunicationSpi());
-
         return cfg;
     }
 
@@ -669,83 +653,6 @@ public boolean apply() {
         }
     }
 
-    /**
-     * @throws Exception if failed.
-     */
-    public void testAffinityAssignmentChangedAfterRestart() throws Exception {
-        delayRebalance = false;
-
-        int parts = 32;
-
-        final List partMapping = new ArrayList<>();
-
-        for (int p = 0; p < parts; p++)
-            partMapping.add(p);
-
-        final AffinityFunction affFunc = new TestAffinityFunction(new RendezvousAffinityFunction(false, parts));
-
-        TestAffinityFunction.partsAffMapping = partMapping;
-
-        String cacheName = CACHE_NAME + 2;
-
-        startGrids(4);
-
-        IgniteEx ig = grid(0);
-
-        ig.cluster().active(true);
-
-        IgniteCache cache = ig.createCache(
-            new CacheConfiguration()
-                .setName(cacheName)
-                .setCacheMode(PARTITIONED)
-                .setBackups(1)
-                .setPartitionLossPolicy(READ_ONLY_SAFE)
-                .setReadFromBackup(true)
-                .setWriteSynchronizationMode(FULL_SYNC)
-                .setRebalanceDelay(-1)
-                .setAffinity(affFunc));
-
-        Map keyToConsId = new HashMap<>();
-
-        for (int k = 0; k < 1000; k++) {
-            cache.put(k, k);
-
-            keyToConsId.put(k, ig.affinity(cacheName).mapKeyToNode(k).consistentId().toString());
-        }
-
-        stopAllGrids();
-
-        Collections.shuffle(TestAffinityFunction.partsAffMapping, new Random(1));
-
-        delayRebalance = true;
-
-        startGrids(4);
-
-        ig = grid(0);
-
-        ig.active(true);
-
-        cache = ig.cache(cacheName);
-
-        GridDhtPartitionFullMap partMap = ig.cachex(cacheName).context().topology().partitionMap(false);
-
-        for (int i = 1; i < 4; i++) {
-            IgniteEx ig0 = grid(i);
-
-            for (int p = 0; p < 32; p++)
-                assertEqualsCollections(ig.affinity(cacheName).mapPartitionToPrimaryAndBackups(p), ig0.affinity(cacheName).mapPartitionToPrimaryAndBackups(p));
-        }
-
-        for (Map.Entry e : keyToConsId.entrySet()) {
-            int p = ig.affinity(cacheName).partition(e.getKey());
-
-            assertEquals("p=" + p, GridDhtPartitionState.OWNING, partMap.get(ig.affinity(cacheName).mapKeyToNode(e.getKey()).id()).get(p));
-        }
-
-        for (int k = 0; k < 1000; k++)
-            assertEquals("k=" + k, Integer.valueOf(k), cache.get(k));
-    }
-
     /**
      * @throws Exception If failed.
      */
@@ -829,73 +736,4 @@ private TestValue(int a) {
             return result;
         }
     }
-
-    /**
-     *
-     */
-    private static class TestAffinityFunction implements AffinityFunction {
-        /** */
-        private final AffinityFunction delegate;
-
-        /** */
-        private static List partsAffMapping;
-
-        /** */
-        public TestAffinityFunction(AffinityFunction delegate) {
-            this.delegate = delegate;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void reset() {
-            delegate.reset();;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int partitions() {
-            return delegate.partitions();
-        }
-
-        /** {@inheritDoc} */
-        @Override public int partition(Object key) {
-            return delegate.partition(key);
-        }
-
-        /** {@inheritDoc} */
-        @Override public List> assignPartitions(AffinityFunctionContext affCtx) {
-            List> res0 = delegate.assignPartitions(affCtx);
-
-            List> res = new ArrayList<>(res0.size());
-
-            for (int p = 0; p < res0.size(); p++)
-                res.add(p, null);
-
-            for (int p = 0; p < res0.size(); p++)
-                res.set(partsAffMapping.get(p), res0.get(p));
-
-            return res;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void removeNode(UUID nodeId) {
-            delegate.removeNode(nodeId);
-        }
-    }
-
-    /**
-     *
-     */
-    private static class DelayRebalanceCommunicationSpi extends TestDelayingCommunicationSpi {
-        /** {@inheritDoc} */
-        @Override protected boolean delayMessage(Message msg, GridIoMessage ioMsg) {
-            if (msg != null && (msg instanceof GridDhtPartitionDemandMessage || msg instanceof GridDhtPartitionSupplyMessage))
-                return true;
-
-            return false;
-        }
-
-        /** {@inheritDoc} */
-        @Override protected int delayMillis() {
-            return 1_000_000;
-        }
-    }
 }

From 80736ccd7e13b0d3438d2ffed25fed989edaf204 Mon Sep 17 00:00:00 2001
From: devozerov 
Date: Fri, 26 Jan 2018 11:20:06 +0300
Subject: [PATCH 51/65] IGNITE-7003: Fixed SQL WAL enable/disable command
 usability: throw exception in case of change failure.

---
 .../thin/JdbcThinWalModeChangeSelfTest.java   | 37 ++++++++++++++++---
 .../WalModeChangeCommonAbstractSelfTest.java  |  3 +-
 .../query/h2/ddl/DdlStatementsProcessor.java  | 16 ++++++--
 .../config/benchmark-atomic.properties        |  2 +-
 4 files changed, 45 insertions(+), 13 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java
index 6a3ac52986c1a..2497d685d918b 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java
@@ -26,6 +26,7 @@
 
 import java.sql.Connection;
 import java.sql.DriverManager;
+import java.sql.SQLException;
 import java.sql.Statement;
 
 /**
@@ -64,20 +65,44 @@ public JdbcThinWalModeChangeSelfTest() {
 
     /** {@inheritDoc} */
     @Override protected boolean walEnable(Ignite node, String cacheName) {
-        String cmd = "ALTER TABLE " + cacheName + " LOGGING";
+        try {
+            String cmd = "ALTER TABLE " + cacheName + " LOGGING";
 
-        execute(node, cmd);
+            execute(node, cmd);
+
+            return true;
+        }
+        catch (RuntimeException e) {
+            if (e.getCause() != null && e.getCause() instanceof SQLException) {
+                SQLException e0 = (SQLException)e.getCause();
 
-        return false;
+                if (e0.getMessage().startsWith("Logging already enabled"))
+                    return false;
+            }
+
+            throw e;
+        }
     }
 
     /** {@inheritDoc} */
     @Override protected boolean walDisable(Ignite node, String cacheName) {
-        String cmd = "ALTER TABLE " + cacheName + " NOLOGGING";
+        try {
+            String cmd = "ALTER TABLE " + cacheName + " NOLOGGING";
 
-        execute(node, cmd);
+            execute(node, cmd);
+
+            return true;
+        }
+        catch (RuntimeException e) {
+            if (e.getCause() != null && e.getCause() instanceof SQLException) {
+                SQLException e0 = (SQLException)e.getCause();
 
-        return false;
+                if (e0.getMessage().startsWith("Logging already disabled"))
+                    return false;
+            }
+
+            throw e;
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java
index 4467f7e97eb9e..2e458f28fbb0e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java
@@ -161,8 +161,7 @@ protected boolean walDisable(Ignite node, String cacheName) {
     protected void assertWalEnable(Ignite node, String cacheName, boolean expRes) {
         boolean res = walEnable(node, cacheName);
 
-        if (!jdbc)
-            assertEquals(expRes, res);
+        assertEquals(expRes, res);
     }
 
     /**
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index 2630239dac24c..ca7680ae272ae 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -188,10 +188,18 @@ else if (cmd instanceof SqlAlterTableCommand) {
 
                 IgniteCluster cluster = ctx.grid().cluster();
 
-                if (logging)
-                    cluster.enableWal(tbl.cacheName());
-                else
-                    cluster.disableWal(tbl.cacheName());
+                if (logging) {
+                    boolean res = cluster.enableWal(tbl.cacheName());
+
+                    if (!res)
+                        throw new IgniteSQLException("Logging already enabled for table: " + cmd0.tableName());
+                }
+                else {
+                    boolean res = cluster.disableWal(tbl.cacheName());
+
+                    if (!res)
+                        throw new IgniteSQLException("Logging already disabled for table: " + cmd0.tableName());
+                }
 
                 fut = new GridFinishedFuture();
             }
diff --git a/modules/yardstick/config/benchmark-atomic.properties b/modules/yardstick/config/benchmark-atomic.properties
index 8e2413bd3279b..1f4e0c3c9cfc6 100644
--- a/modules/yardstick/config/benchmark-atomic.properties
+++ b/modules/yardstick/config/benchmark-atomic.properties
@@ -74,7 +74,7 @@ sm=PRIMARY_SYNC
 # Run configuration.
 # Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
 CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteGetBenchmark -sn IgniteNode -ds ${ver}atomic-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteGetBenchmark -sn IgniteNode -ds ${ver}atomic-get-${b}-backup --single-shot ,\
 -cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-${b}-backup,\
 -cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-${b}-backup,\
 "

From 915dd2966084d78f7b4f3d482e6bd25f860c1e23 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Wed, 31 Jan 2018 11:22:26 +0300
Subject: [PATCH 52/65] IGNITE-7569 Fixed index rebuild future - Fixes #3454.

---
 .../GridDhtPartitionsExchangeFuture.java      | 29 +++++++++-
 .../GridCacheDatabaseSharedManager.java       | 57 ++++++++++++++-----
 2 files changed, 69 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 6c09b6a4373df..a45c9b99cd85d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -168,7 +168,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     /** */
     private AtomicBoolean added = new AtomicBoolean(false);
 
-    /** Event latch. */
+    /**
+     * Discovery event receive latch. There is a race between discovery event processing and single message
+     * processing, so it is possible to create an exchange future before the actual discovery event is received.
+     * This latch is notified when the discovery event arrives.
+     */
     @GridToStringExclude
     private final CountDownLatch evtLatch = new CountDownLatch(1);
 
@@ -344,6 +348,10 @@ public ExchangeContext context() {
     }
 
     /**
+     * Sets exchange actions associated with the exchange future (such as cache start or stop).
+     * Exchange actions is created from discovery event, so the actions must be set before the event is processed,
+     * thus the setter requires that {@code evtLatch} be armed.
+     *
      * @param exchActions Exchange actions.
      */
     public void exchangeActions(ExchangeActions exchActions) {
@@ -354,6 +362,20 @@ public void exchangeActions(ExchangeActions exchActions) {
     }
 
     /**
+     * Gets exchanges actions (such as cache start or stop) associated with the exchange future.
+     * Exchange actions can be {@code null} (for example, if the exchange is created for topology
+     * change event).
+     *
+     * @return Exchange actions.
+     */
+    @Nullable public ExchangeActions exchangeActions() {
+        return exchActions;
+    }
+
+    /**
+     * Sets affinity change message associated with the exchange. Affinity change message is required when
+     * centralized affinity change is performed.
+     *
      * @param affChangeMsg Affinity change message.
      */
     public void affinityChangeMessage(CacheAffinityChangeMessage affChangeMsg) {
@@ -361,9 +383,12 @@ public void affinityChangeMessage(CacheAffinityChangeMessage affChangeMsg) {
     }
 
     /**
+     * Gets the affinity topology version for which this exchange was created. If several exchanges
+     * were merged, initial version is the version of the earliest merged exchange.
+     *
      * @return Initial exchange version.
      */
-    public AffinityTopologyVersion initialVersion() {
+    @Override public AffinityTopologyVersion initialVersion() {
         return exchId.topologyVersion();
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index f833911135b7d..0b35f1830a912 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -105,6 +105,7 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.ExchangeActions;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
@@ -320,8 +321,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Thread local with buffers for the checkpoint threads. Each buffer represent one page for durable memory. */
     private ThreadLocal threadBuf;
 
-    /** */
-    private final ConcurrentMap idxRebuildFuts = new ConcurrentHashMap<>();
+    /** Map from a cacheId to a future indicating that there is an in-progress index rebuild for the given cache. */
+    private final ConcurrentMap> idxRebuildFuts = new ConcurrentHashMap<>();
 
     /**
      * Lock holder for compatible folders mode. Null if lock holder was created at start node. 

@@ -1127,33 +1128,59 @@ private void shutdownCheckpointer(boolean cancel) {
         // Before local node join event.
         if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode))
             restoreState();
+
+        if (cctx.kernalContext().query().moduleEnabled()) {
+            ExchangeActions acts = fut.exchangeActions();
+
+            if (acts != null && !F.isEmpty(acts.cacheStartRequests())) {
+                for (ExchangeActions.CacheActionData actionData : acts.cacheStartRequests()) {
+                    int cacheId = CU.cacheId(actionData.request().cacheName());
+
+                    GridFutureAdapter old = idxRebuildFuts.put(cacheId, new GridFutureAdapter<>());
+
+                    if (old != null)
+                        old.onDone();
+                }
+            }
+        }
     }
 
     /** {@inheritDoc} */
     @Override public void rebuildIndexesIfNeeded(GridDhtPartitionsExchangeFuture fut) {
         if (cctx.kernalContext().query().moduleEnabled()) {
             for (final GridCacheContext cacheCtx : (Collection)cctx.cacheContexts()) {
-                if (cacheCtx.startTopologyVersion().equals(fut.initialVersion()) &&
-                    !cctx.pageStore().hasIndexStore(cacheCtx.groupId()) && cacheCtx.affinityNode()) {
+                if (cacheCtx.startTopologyVersion().equals(fut.initialVersion())) {
                     final int cacheId = cacheCtx.cacheId();
+                    final GridFutureAdapter usrFut = idxRebuildFuts.get(cacheId);
+
+                    if (!cctx.pageStore().hasIndexStore(cacheCtx.groupId()) && cacheCtx.affinityNode()) {
+                        IgniteInternalFuture rebuildFut = cctx.kernalContext().query()
+                            .rebuildIndexesFromHash(Collections.singletonList(cacheCtx.cacheId()));
 
-                    final IgniteInternalFuture rebuildFut = cctx.kernalContext().query()
-                        .rebuildIndexesFromHash(Collections.singletonList(cacheCtx.cacheId()));
+                        assert usrFut != null : "Missing user future for cache: " + cacheCtx.name();
 
-                    idxRebuildFuts.put(cacheId, rebuildFut);
+                        rebuildFut.listen(new CI1() {
+                            @Override public void apply(IgniteInternalFuture igniteInternalFut) {
+                                idxRebuildFuts.remove(cacheId, usrFut);
 
-                    rebuildFut.listen(new CI1() {
-                        @Override public void apply(IgniteInternalFuture igniteInternalFut) {
-                            idxRebuildFuts.remove(cacheId, rebuildFut);
+                                usrFut.onDone(igniteInternalFut.error());
 
-                            CacheConfiguration ccfg = cacheCtx.config();
+                                CacheConfiguration ccfg = cacheCtx.config();
 
-                            if (ccfg != null) {
-                                log().info("Finished indexes rebuilding for cache [name=" + ccfg.getName()
-                                    + ", grpName=" + ccfg.getGroupName() + ']');
+                                if (ccfg != null) {
+                                    log().info("Finished indexes rebuilding for cache [name=" + ccfg.getName()
+                                        + ", grpName=" + ccfg.getGroupName() + ']');
+                                }
                             }
+                        });
+                    }
+                    else {
+                        if (usrFut != null) {
+                            idxRebuildFuts.remove(cacheId, usrFut);
+
+                            usrFut.onDone();
                         }
-                    });
+                    }
                 }
             }
         }

From 8ea8609259039852ab0c26f26ac528c1ffae7c94 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Wed, 31 Jan 2018 11:24:57 +0300
Subject: [PATCH 53/65] IGNITE-7577 Fixing public API active flag on baseline
 changes - Fixes #3455.

---
 .../cluster/DiscoveryDataClusterState.java    |   9 +-
 .../cluster/GridClusterStateProcessor.java    |   2 +-
 .../CacheBaselineTopologyTest.java            | 102 +++++++++++-------
 3 files changed, 71 insertions(+), 42 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index 1c8e830710f8d..dea2ce79c311e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -146,12 +146,19 @@ public UUID transitionRequestId() {
     }
 
     /**
-     * @return {@code True} if state change is in progress.
+     * @return {@code True} if any cluster state change is in progress (e.g. active state change, baseline change).
      */
     public boolean transition() {
         return transitionReqId != null;
     }
 
+    /**
+     * @return {@code True} if cluster active state change is in progress, {@code false} otherwise.
+     */
+    public boolean activeStateChanging() {
+        return transition() && active != prevState.active;
+    }
+
     /**
      * @return State change exchange version.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index f28df8ac14f80..aa23b61a3a653 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -180,7 +180,7 @@ public boolean compatibilityMode() {
 
         assert globalState != null;
 
-        if (globalState.transition()) {
+        if (globalState.transition() && globalState.activeStateChanging()) {
             Boolean transitionRes = globalState.transitionResult();
 
             if (transitionRes != null)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index e70773d205274..7b40b03cae393 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -30,8 +30,6 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.PartitionLossPolicy;
 import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -40,9 +38,10 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.util.lang.GridAbsPredicate;
-import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -85,6 +84,8 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
+        cfg.setConsistentId(igniteInstanceName);
+
         cfg.setDataStorageConfiguration(
             new DataStorageConfiguration().setDefaultDataRegionConfiguration(
                 new DataRegionConfiguration()
@@ -116,7 +117,7 @@ public void testTopologyChangesWithFixedBaseline() throws Exception {
 
         IgniteEx ignite = grid(0);
 
-        ignite.active(true);
+        ignite.cluster().active(true);
 
         awaitPartitionMapExchange();
 
@@ -257,6 +258,42 @@ public void testBaselineTopologyChangesFromClient() throws Exception {
         testBaselineTopologyChanges(true);
     }
 
+    /**
+     * @throws Exception if failed.
+     */
+    public void testClusterActiveWhileBaselineChanging() throws Exception {
+        startGrids(NODE_COUNT);
+
+        IgniteEx ig = grid(0);
+
+        ig.cluster().active(true);
+
+        assertTrue(ig.cluster().active());
+
+        startGrid(NODE_COUNT);
+
+        IgniteInternalFuture fut = GridTestUtils.runAsync(() -> {
+            try {
+                U.sleep(100);
+            }
+            catch (IgniteInterruptedCheckedException e) {
+                e.printStackTrace();
+            }
+            ig.cluster().setBaselineTopology(NODE_COUNT + 1);
+        });
+
+        while (!fut.isDone()) {
+            assertTrue(grid(0).cluster().active());
+            assertTrue(grid(0).context().state().publicApiActiveState(false));
+            assertTrue(grid(NODE_COUNT).cluster().active());
+            assertTrue(grid(NODE_COUNT).context().state().publicApiActiveState(false));
+        }
+
+        assertNull(String.valueOf(fut.error()), fut.error());
+
+        assertEquals(NODE_COUNT + 1, ig.cluster().currentBaselineTopology().size());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -275,7 +312,7 @@ private void testBaselineTopologyChanges(boolean fromClient) throws Exception {
         else
             ignite = grid(0);
 
-        ignite.active(true);
+        ignite.cluster().active(true);
 
         awaitPartitionMapExchange();
 
@@ -287,14 +324,13 @@ private void testBaselineTopologyChanges(boolean fromClient) throws Exception {
             nodes.put(ig.cluster().localNode(), ig);
         }
 
-        IgniteCache cache =
-            ignite.createCache(
-                new CacheConfiguration()
-                    .setName(CACHE_NAME)
-                    .setCacheMode(PARTITIONED)
-                    .setBackups(1)
-                    .setPartitionLossPolicy(READ_ONLY_SAFE)
-            );
+        ignite.createCache(
+            new CacheConfiguration()
+                .setName(CACHE_NAME)
+                .setCacheMode(PARTITIONED)
+                .setBackups(1)
+                .setPartitionLossPolicy(READ_ONLY_SAFE)
+        );
 
         int key = -1;
 
@@ -307,8 +343,6 @@ private void testBaselineTopologyChanges(boolean fromClient) throws Exception {
 
         assert key >= 0;
 
-        int part = ignite.affinity(CACHE_NAME).partition(key);
-
         Collection initialMapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
 
         assert initialMapping.size() == 2 : initialMapping;
@@ -474,7 +508,7 @@ public void testPrimaryLeftAndClusterRestart() throws Exception {
 
         IgniteEx ig = grid(0);
 
-        ig.active(true);
+        ig.cluster().active(true);
 
         IgniteCache cache =
             ig.createCache(
@@ -544,14 +578,13 @@ else if (grid(i).localNode().equals(affNodes.get(1))) {
         primary = grid(primaryIdx);
         backup = grid(backupIdx);
 
-        boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override public boolean apply() {
-                for (int i = 0; i < NODE_COUNT; i++)
-                  if (!grid(i).active())
-                      return false;
-
-                return true;
+        boolean activated = GridTestUtils.waitForCondition(() -> {
+            for (int i = 0; i < NODE_COUNT; i++) {
+                if (!grid(i).cluster().active())
+                    return false;
             }
+
+            return true;
         }, 10_000);
 
         assert activated;
@@ -580,7 +613,7 @@ public void testMetadataUpdate() throws Exception {
 
         Ignite ignite3 = grid(3);
 
-        ignite3.active(true);
+        ignite3.cluster().active(true);
 
         CacheConfiguration repCacheCfg = new CacheConfiguration<>("replicated")
             .setCacheMode(CacheMode.REPLICATED)
@@ -600,12 +633,7 @@ public void testMetadataUpdate() throws Exception {
 
         startGrids(5);
 
-        GridTestUtils.waitForCondition(new PA() {
-            @Override
-            public boolean apply() {
-                return grid(0).cluster().active();
-            }
-        }, getTestTimeout());
+        GridTestUtils.waitForCondition(() -> grid(0).cluster().active(), getTestTimeout());
 
         for (int g = 0; g < 5; g++) {
             for (int i = 0; i < 100; i++)
@@ -621,7 +649,7 @@ public void testClusterRestoredOnRestart() throws Exception {
 
         Ignite ignite3 = grid(3);
 
-        ignite3.active(true);
+        ignite3.cluster().active(true);
 
         stopGrid(0);
 
@@ -640,12 +668,7 @@ public void testClusterRestoredOnRestart() throws Exception {
 
         startGrids(5);
 
-        GridTestUtils.waitForCondition(new PA() {
-            @Override
-            public boolean apply() {
-                return grid(0).cluster().active();
-            }
-        }, getTestTimeout());
+        GridTestUtils.waitForCondition(() -> grid(0).cluster().active(), getTestTimeout());
 
         for (int g = 0; g < 5; g++) {
             for (int i = 0; i < 2048; i++)
@@ -678,8 +701,7 @@ public void testNonPersistentCachesIgnoreBaselineTopology() throws Exception {
     private Collection baselineNodes(Collection clNodes) {
         Collection res = new ArrayList<>(clNodes.size());
 
-        for (ClusterNode clN : clNodes)
-            res.add(clN);
+        res.addAll(clNodes);
 
         return res;
     }

From c8ce1f66e98b3174d771a3b801a2538499dc2c3d Mon Sep 17 00:00:00 2001
From: Ivan Rakov 
Date: Wed, 31 Jan 2018 12:51:09 +0300
Subject: [PATCH 54/65] IGNITE-7475 Improved VerifyBackupPartitionsTask to
 calculate partition hashes in parallel - Fixes #3407.

Signed-off-by: Alexey Goncharuk 
---
 .../verify/VerifyBackupPartitionsTask.java    | 157 +++++++++++++-----
 1 file changed, 118 insertions(+), 39 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
index 23aa0e1bac2a2..b884cb01ac934 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
@@ -19,13 +19,22 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
@@ -162,6 +171,9 @@ public static class VerifyBackupPartitionsJob extends ComputeJobAdapter {
         /** Cache names. */
         private Set cacheNames;
 
+        /** Counter of processed partitions. */
+        private final AtomicInteger completionCntr = new AtomicInteger(0);
+
         /**
          * @param names Names.
          */
@@ -208,7 +220,9 @@ private VerifyBackupPartitionsJob(Set names) {
                 }
             }
 
-            Map res = new HashMap<>();
+            List>> partHashCalcFutures = new ArrayList<>();
+
+            completionCntr.set(0);
 
             for (Integer grpId : grpIds) {
                 CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
@@ -218,62 +232,127 @@ private VerifyBackupPartitionsJob(Set names) {
 
                 List parts = grpCtx.topology().localPartitions();
 
-                for (GridDhtLocalPartition part : parts) {
-                    if (!part.reserve())
-                        continue;
+                for (GridDhtLocalPartition part : parts)
+                    partHashCalcFutures.add(calculatePartitionHashAsync(grpCtx, part));
+            }
 
-                    int partHash = 0;
-                    long partSize;
-                    long updateCntrBefore;
+            Map res = new HashMap<>();
 
-                    try {
-                        if (part.state() != GridDhtPartitionState.OWNING)
-                            continue;
+            long lastProgressLogTs = U.currentTimeMillis();
 
-                        updateCntrBefore = part.updateCounter();
+            for (int i = 0; i < partHashCalcFutures.size(); ) {
+                Future> fut = partHashCalcFutures.get(i);
 
-                        partSize = part.dataStore().fullSize();
+                try {
+                    Map partHash = fut.get(10, TimeUnit.SECONDS);
 
-                        GridIterator it = grpCtx.offheap().partitionIterator(part.id());
+                    res.putAll(partHash);
 
-                        while (it.hasNextX()) {
-                            CacheDataRow row = it.nextX();
+                    i++;
+                }
+                catch (InterruptedException | ExecutionException e) {
+                    for (int j = i + 1; j < partHashCalcFutures.size(); j++)
+                        partHashCalcFutures.get(j).cancel(false);
+
+                    if (e instanceof InterruptedException)
+                        throw new IgniteInterruptedException((InterruptedException)e);
+                    else if (e.getCause() instanceof IgniteException)
+                        throw (IgniteException)e.getCause();
+                    else
+                        throw new IgniteException(e.getCause());
+                }
+                catch (TimeoutException e) {
+                    if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 * 1000L) {
+                        lastProgressLogTs = U.currentTimeMillis();
 
-                            partHash += row.key().hashCode();
+                        log.warning("idle_verify is still running, processed " + completionCntr.get() + " of " +
+                            partHashCalcFutures.size() + " local partitions");
+                    }
+                }
+            }
 
-                            partHash += Arrays.hashCode(row.value().valueBytes(grpCtx.cacheObjectContext()));
-                        }
+            return res;
+        }
 
-                        long updateCntrAfter = part.updateCounter();
+        /**
+         * @param grpCtx Group context.
+         * @param part Local partition.
+         */
+        private Future> calculatePartitionHashAsync(
+            final CacheGroupContext grpCtx,
+            final GridDhtLocalPartition part
+        ) {
+            return ForkJoinPool.commonPool().submit(new Callable>() {
+                @Override public Map call() throws Exception {
+                    return calculatePartitionHash(grpCtx, part);
+                }
+            });
+        }
 
-                        if (updateCntrBefore != updateCntrAfter) {
-                            throw new IgniteException("Cluster is not idle: update counter of partition [grpId=" +
-                                grpId + ", partId=" + part.id() + "] changed during hash calculation [before=" +
-                                updateCntrBefore + ", after=" + updateCntrAfter + "]");
-                        }
-                    }
-                    catch (IgniteCheckedException e) {
-                        U.error(log, "Can't calculate partition hash [grpId=" + grpId +
-                            ", partId=" + part.id() + "]", e);
 
-                        continue;
-                    }
-                    finally {
-                        part.release();
-                    }
+        /**
+         * @param grpCtx Group context.
+         * @param part Local partition.
+         */
+        private Map calculatePartitionHash(
+            CacheGroupContext grpCtx,
+            GridDhtLocalPartition part
+        ) {
+            if (!part.reserve())
+                return Collections.emptyMap();
+
+            int partHash = 0;
+            long partSize;
+            long updateCntrBefore;
+
+            try {
+                if (part.state() != GridDhtPartitionState.OWNING)
+                    return Collections.emptyMap();
 
-                    Object consId = ignite.context().discovery().localNode().consistentId();
+                updateCntrBefore = part.updateCounter();
 
-                    boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
+                partSize = part.dataStore().fullSize();
+
+                GridIterator it = grpCtx.offheap().partitionIterator(part.id());
+
+                while (it.hasNextX()) {
+                    CacheDataRow row = it.nextX();
+
+                    partHash += row.key().hashCode();
+
+                    partHash += Arrays.hashCode(row.value().valueBytes(grpCtx.cacheObjectContext()));
+                }
 
-                    PartitionKey partKey = new PartitionKey(grpId, part.id(), grpCtx.cacheOrGroupName());
+                long updateCntrAfter = part.updateCounter();
 
-                    res.put(partKey, new PartitionHashRecord(
-                        partKey, isPrimary, consId, partHash, updateCntrBefore, partSize));
+                if (updateCntrBefore != updateCntrAfter) {
+                    throw new IgniteException("Cluster is not idle: update counter of partition [grpId=" +
+                        grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation [before=" +
+                        updateCntrBefore + ", after=" + updateCntrAfter + "]");
                 }
             }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Can't calculate partition hash [grpId=" + grpCtx.groupId() +
+                    ", partId=" + part.id() + "]", e);
 
-            return res;
+                return Collections.emptyMap();
+            }
+            finally {
+                part.release();
+            }
+
+            Object consId = ignite.context().discovery().localNode().consistentId();
+
+            boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
+
+            PartitionKey partKey = new PartitionKey(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName());
+
+            PartitionHashRecord partRec = new PartitionHashRecord(
+                partKey, isPrimary, consId, partHash, updateCntrBefore, partSize);
+
+            completionCntr.incrementAndGet();
+
+            return Collections.singletonMap(partKey, partRec);
         }
     }
 

From 258ff4299da20122d7c387cb8579264035c93c18 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Wed, 31 Jan 2018 16:52:24 +0300
Subject: [PATCH 55/65] IGNITE-7573 Fixed full API tests to be compliant with
 baseline topology

---
 .../cache/GridCacheAbstractFullApiSelfTest.java       | 11 ++++-------
 .../IgniteBaselineAbstractFullApiSelfTest.java        |  4 +++-
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
index e6c9589a5ec30..2e6a19cb5903a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
@@ -463,14 +463,11 @@ public void testSize() throws Exception {
         for (int i = 0; i < gridCount(); i++)
             assertEquals(globalPrimarySize, jcache(i).size(PRIMARY));
 
-        int times = 1;
+        // Check how many instances of any given key there is in the cluster.
+        int globalSize = 0;
 
-        if (cacheMode() == REPLICATED)
-            times = gridCount();
-        else if (cacheMode() == PARTITIONED)
-            times = Math.min(gridCount(), jcache().getConfiguration(CacheConfiguration.class).getBackups() + 1);
-
-        int globalSize = globalPrimarySize * times;
+        for (String key : map.keySet())
+            globalSize += affinity(jcache()).mapKeyToPrimaryAndBackups(key).size();
 
         for (int i = 0; i < gridCount(); i++)
             assertEquals(globalSize, jcache(i).size(ALL));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
index 8dcfc0b979926..d78c289f2e9e5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
@@ -19,6 +19,7 @@
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
 
 /**
@@ -33,7 +34,8 @@ public abstract class IgniteBaselineAbstractFullApiSelfTest extends GridCacheAbs
             .setDefaultDataRegionConfiguration(
                 new DataRegionConfiguration()
                     .setMaxSize(200 * 1024 * 1024)
-                    .setPersistenceEnabled(true)));
+                    .setPersistenceEnabled(true))
+            .setWalMode(WALMode.LOG_ONLY));
 
         return cfg;
     }

From 254ed3a9c32d092702a0461509bf867cbd7cdee6 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov 
Date: Thu, 1 Feb 2018 15:22:53 +0700
Subject: [PATCH 56/65] ignite-2.4.0 Update version.

(cherry picked from commit 2e43749)
---
 modules/web-console/frontend/app/services/Version.service.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/services/Version.service.js b/modules/web-console/frontend/app/services/Version.service.js
index 22d0732cff835..9bd9c7dfc302d 100644
--- a/modules/web-console/frontend/app/services/Version.service.js
+++ b/modules/web-console/frontend/app/services/Version.service.js
@@ -73,7 +73,7 @@ const compare = (a, b) => {
 
 export default class IgniteVersion {
     constructor() {
-        this.webConsole = '2.2.0';
+        this.webConsole = '2.4.0';
 
         this.supportedVersions = [
             {

From c1a9c0a404d77fba08170bedf14844f87abe3028 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Thu, 1 Feb 2018 13:17:28 +0300
Subject: [PATCH 57/65] IGNITE-7569 Fixing index rebuild future

---
 .../GridCacheDatabaseSharedManager.java       | 32 ++++++++++++++-----
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 0b35f1830a912..5dc81c5e7ad48 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -71,6 +71,7 @@
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.GridKernalContext;
@@ -1132,19 +1133,34 @@ private void shutdownCheckpointer(boolean cancel) {
         if (cctx.kernalContext().query().moduleEnabled()) {
             ExchangeActions acts = fut.exchangeActions();
 
-            if (acts != null && !F.isEmpty(acts.cacheStartRequests())) {
-                for (ExchangeActions.CacheActionData actionData : acts.cacheStartRequests()) {
-                    int cacheId = CU.cacheId(actionData.request().cacheName());
-
-                    GridFutureAdapter old = idxRebuildFuts.put(cacheId, new GridFutureAdapter<>());
-
-                    if (old != null)
-                        old.onDone();
+            if (acts != null) {
+                if (!F.isEmpty(acts.cacheStartRequests())) {
+                    for (ExchangeActions.CacheActionData actionData : acts.cacheStartRequests())
+                        prepareIndexRebuildFuture(CU.cacheId(actionData.request().cacheName()));
+                }
+                else if (acts.localJoinContext() != null && !F.isEmpty(acts.localJoinContext().caches())) {
+                    for (T2 tup : acts.localJoinContext().caches())
+                        prepareIndexRebuildFuture(tup.get1().cacheId());
                 }
             }
         }
     }
 
+    /**
+     * Creates a new index rebuild future that should be completed later after exchange is done. The future
+     * has to be created before exchange is initialized to guarantee that we will capture a correct future
+     * after activation or restore completes.
+     * If there was an old future for the given ID, it will be completed.
+     *
+     * @param cacheId Cache ID.
+     */
+    private void prepareIndexRebuildFuture(int cacheId) {
+        GridFutureAdapter old = idxRebuildFuts.put(cacheId, new GridFutureAdapter<>());
+
+        if (old != null)
+            old.onDone();
+    }
+
     /** {@inheritDoc} */
     @Override public void rebuildIndexesIfNeeded(GridDhtPartitionsExchangeFuture fut) {
         if (cctx.kernalContext().query().moduleEnabled()) {

From e43799ce70cdbe03d9e206381d1d5138b820b075 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Thu, 1 Feb 2018 16:39:17 +0300
Subject: [PATCH 58/65] IGNITE-7520 Provide util-methods to get baseline from
 context - Fixes #3431.

---
 .../ignite/internal/util/IgniteUtils.java     | 63 +++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 6e295dda48a7c..59430b0b9917b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -194,7 +194,9 @@
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
 import org.apache.ignite.internal.mxbean.IgniteStandardMXBean;
 import org.apache.ignite.internal.processors.cache.GridCacheAttributes;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
@@ -10294,6 +10296,67 @@ public static LockTracer lockTracer(Lock lock) {
         return new LockTracer(lock);
     }
 
+    /**
+     * @param ctx Context.
+     *
+     * @return instance of current baseline topology if it exists
+     */
+    public static BaselineTopology getBaselineTopology(@NotNull GridKernalContext ctx) {
+        return ctx.state().clusterState().baselineTopology();
+    }
+
+
+    /**
+     * @param cctx Context.
+     *
+     * @return instance of current baseline topology if it exists
+     */
+    public static BaselineTopology getBaselineTopology(@NotNull GridCacheSharedContext cctx) {
+        return getBaselineTopology(cctx.kernalContext());
+    }
+
+    /**
+     * @param cctx Context.
+     *
+     * @return instance of current baseline topology if it exists
+     */
+    public static BaselineTopology getBaselineTopology(@NotNull GridCacheContext cctx) {
+        return getBaselineTopology(cctx.kernalContext());
+    }
+
+    /**
+     * @param addr pointer in memory
+     * @param len how much byte to read (should divide 8)
+     *
+     * @return hex representation of memory region
+     */
+    public static String toHexString(long addr, int len) {
+        assert (len & 0b111) == 0 && len > 0;
+
+        StringBuilder sb = new StringBuilder(len * 2);
+
+        for (int i = 0; i < len; i += 8)
+            sb.append(U.hexLong(GridUnsafe.getLong(addr + i)));
+
+        return sb.toString();
+    }
+
+    /**
+     * @param buf which content should be converted to string
+     *
+     * @return hex representation of memory region
+     */
+    public static String toHexString(ByteBuffer buf) {
+        assert (buf.capacity() & 0b111) == 0;
+
+        StringBuilder sb = new StringBuilder(buf.capacity() * 2);
+
+        for (int i = 0; i < buf.capacity(); i += 8)
+            sb.append(U.hexLong(buf.getLong(i)));
+
+        return sb.toString();
+    }
+
     /**
      *
      */

From 8f5fc7cfb0624cf2048efad38dfff32f782116e8 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov 
Date: Fri, 2 Feb 2018 11:24:29 +0300
Subject: [PATCH 59/65] IGNITE-7580 Fix compatibilityMode flag consistency

This closes #3466

(cherry picked from commit 8f2045e)
---
 .../cluster/GridClusterStateProcessor.java    | 31 +------------------
 1 file changed, 1 insertion(+), 30 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index aa23b61a3a653..eaceb6923a230 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -65,7 +65,6 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteFuture;
@@ -643,7 +642,7 @@ protected IgniteCheckedException concurrentStateChangeError(boolean activate) {
             BaselineTopologyHistory historyToSend = null;
 
             if (joiningNodeData != null) {
-                if (!joiningNodeData.hasJoiningNodeData()) {
+                if (!joiningNodeData.hasJoiningNodeData() || compatibilityMode) {
                     //compatibility mode: old nodes don't send any data on join, so coordinator of new version
                     //doesn't send BaselineTopology history, only its current globalState
                     dataBag.addGridCommonData(STATE_PROC.ordinal(), globalState);
@@ -791,40 +790,12 @@ private Collection baselineNodes() {
         return bltNodes;
     }
 
-    /**
-     * Verifies all nodes in current cluster topology support BaselineTopology feature
-     * so compatibilityMode flag is enabled to reset.
-     *
-     * @param discoCache
-     */
-    private void verifyBaselineTopologySupport(DiscoCache discoCache) {
-        if (discoCache.minimumServerNodeVersion().compareTo(MIN_BLT_SUPPORTING_VER) < 0) {
-            SB sb = new SB("Cluster contains nodes that don't support BaselineTopology: [");
-
-            for (ClusterNode cn : discoCache.serverNodes()) {
-                if (cn.version().compareTo(MIN_BLT_SUPPORTING_VER) < 0)
-                    sb
-                        .a("[")
-                        .a(cn.consistentId())
-                        .a(":")
-                        .a(cn.version())
-                        .a("], ");
-            }
-
-            sb.d(sb.length() - 2, sb.length());
-
-            throw new IgniteException(sb.a("]").toString());
-        }
-    }
-
     /** */
     private IgniteInternalFuture changeGlobalState0(final boolean activate,
         BaselineTopology blt, boolean forceChangeBaselineTopology) {
         if (ctx.isDaemon() || ctx.clientNode()) {
             GridFutureAdapter fut = new GridFutureAdapter<>();
 
-            verifyBaselineTopologySupport(ctx.discovery().discoCache());
-
             sendComputeChangeGlobalState(activate, blt, forceChangeBaselineTopology, fut);
 
             return fut;

From d3ddd50cb2b889173176b6c47c9ff61410e1d909 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh 
Date: Wed, 7 Feb 2018 13:33:28 +0300
Subject: [PATCH 60/65] IGNITE-7514 Affinity assignment should be recalculated
 when primary node is not OWNER

(cherry picked from commit faf50f1)
---
 .../internal/events/DiscoveryCustomEvent.java |  34 ++++
 .../cache/CacheAffinitySharedManager.java     |  86 ++++++---
 .../processors/cache/ClusterCachesInfo.java   |   1 +
 .../GridCachePartitionExchangeManager.java    |   2 +-
 .../dht/GridClientPartitionTopology.java      |   9 +-
 .../dht/GridDhtLocalPartition.java            |   3 -
 .../dht/GridDhtPartitionTopology.java         |  10 +-
 .../dht/GridDhtPartitionTopologyImpl.java     |  47 ++++-
 .../GridDhtPartitionsExchangeFuture.java      | 144 +++++++++------
 .../GridCacheDatabaseSharedManager.java       |   5 +
 .../CacheBaselineTopologyTest.java            | 169 +++++++++++++++++-
 11 files changed, 424 insertions(+), 86 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
index b3c6a2d865b2f..3b12b384f54be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/events/DiscoveryCustomEvent.java
@@ -21,7 +21,10 @@
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
+import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Custom event.
@@ -85,4 +88,35 @@ public void affinityTopologyVersion(AffinityTopologyVersion affTopVer) {
     @Override public String toString() {
         return S.toString(DiscoveryCustomEvent.class, this, super.toString());
     }
+
+    /**
+     * @param evt Discovery event.
+     * @return {@code True} if event is DiscoveryCustomEvent that requires centralized affinity assignment.
+     */
+    public static boolean requiresCentralizedAffinityAssignment(DiscoveryEvent evt) {
+        if (!(evt instanceof DiscoveryCustomEvent))
+            return false;
+
+        return requiresCentralizedAffinityAssignment(((DiscoveryCustomEvent)evt).customMessage());
+    }
+
+    /**
+     * @param msg Discovery custom message.
+     * @return {@code True} if message belongs to event that requires centralized affinity assignment.
+     */
+    public static boolean requiresCentralizedAffinityAssignment(@Nullable DiscoveryCustomMessage msg) {
+        if (msg == null)
+            return false;
+
+        if (msg instanceof ChangeGlobalStateMessage && ((ChangeGlobalStateMessage)msg).activate())
+            return true;
+
+        if (msg instanceof SnapshotDiscoveryMessage) {
+            SnapshotDiscoveryMessage snapMsg = (SnapshotDiscoveryMessage) msg;
+
+            return snapMsg.needExchange() && snapMsg.needAssignPartitions();
+        }
+
+        return false;
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 4119f2357c896..7bf793c2f0b05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
+import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
@@ -57,7 +58,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
-import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
@@ -161,13 +161,15 @@ void onDiscoveryEvent(int type,
         ClusterNode node,
         AffinityTopologyVersion topVer,
         DiscoveryDataClusterState state) {
-        if (state.transition() || !state.active())
+        if ((state.transition() || !state.active()) &&
+            !DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(customMsg))
             return;
 
         if (type == EVT_NODE_JOINED && node.isLocal())
             lastAffVer = null;
 
-        if (!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) {
+        if ((!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) ||
+            DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(customMsg)) {
             synchronized (mux) {
                 assert lastAffVer == null || topVer.compareTo(lastAffVer) > 0;
 
@@ -1260,10 +1262,12 @@ public GridAffinityAssignmentCache affinity(Integer grpId) {
     }
 
     /**
+     * Applies affinity diff from the received full message.
+     *
      * @param fut Current exchange future.
      * @param msg Finish exchange message.
      */
-    public void mergeExchangesOnServerLeft(final GridDhtPartitionsExchangeFuture fut,
+    public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture fut,
         final GridDhtPartitionsFullMessage msg) {
         final Map nodesByOrder = new HashMap<>();
 
@@ -1396,7 +1400,7 @@ public void onServerJoinWithExchangeMergeProtocol(GridDhtPartitionsExchangeFutur
      * @return Computed difference with ideal affinity.
      * @throws IgniteCheckedException If failed.
      */
-    public  Map onServerLeftWithExchangeMergeProtocol(
+    public Map onServerLeftWithExchangeMergeProtocol(
         final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
     {
         final ExchangeDiscoveryEvents evts = fut.context().events();
@@ -1404,6 +1408,36 @@ public  Map onServerLeftWithExchangeMergePro
         assert fut.context().mergeExchanges();
         assert evts.hasServerLeft();
 
+        return onReassignmentEnforced(fut);
+    }
+
+    /**
+     * Calculates affinity on coordinator for custom event types that require centralized assignment.
+     *
+     * @param fut Current exchange future.
+     * @return Computed difference with ideal affinity.
+     * @throws IgniteCheckedException If failed.
+     */
+    public Map onCustomEventWithEnforcedAffinityReassignment(
+        final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
+    {
+        assert DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent());
+
+        return onReassignmentEnforced(fut);
+    }
+
+    /**
+     * Calculates new affinity assignment on coordinator and creates affinity diff messages for other nodes.
+     *
+     * @param fut Current exchange future.
+     * @return Computed difference with ideal affinity.
+     * @throws IgniteCheckedException If failed.
+     */
+    private Map onReassignmentEnforced(
+        final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
+    {
+        final ExchangeDiscoveryEvents evts = fut.context().events();
+
         forAllRegisteredCacheGroups(new IgniteInClosureX() {
             @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException {
                 AffinityTopologyVersion topVer = evts.topologyVersion();
@@ -1418,7 +1452,7 @@ public  Map onServerLeftWithExchangeMergePro
             }
         });
 
-        Map>> diff = initAffinityOnNodeLeft0(evts.topologyVersion(),
+        Map>> diff = initAffinityBasedOnPartitionsAvailability(evts.topologyVersion(),
             fut,
             NODE_TO_ORDER,
             true);
@@ -1642,17 +1676,16 @@ private GridDhtAffinityAssignmentResponse fetchAffinity(AffinityTopologyVersion
     }
 
     /**
-     * Called on exchange initiated by server node leave.
+     * Called on exchange initiated by server node leave or custom event with centralized affinity assignment.
      *
      * @param fut Exchange future.
      * @param crd Coordinator flag.
      * @throws IgniteCheckedException If failed.
      * @return {@code True} if affinity should be assigned by coordinator.
      */
-    public boolean onServerLeft(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
-        ClusterNode leftNode = fut.firstEvent().eventNode();
-
-        assert !leftNode.isClient() : leftNode;
+    public boolean onCentralizedAffinityChange(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
+        assert (fut.events().hasServerLeft() && !fut.firstEvent().eventNode().isClient()) ||
+            DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent()) : fut.firstEvent();
 
         if (crd) {
             // Need initialize CacheGroupHolders if this node become coordinator on this exchange.
@@ -2066,7 +2099,7 @@ public IgniteInternalFuture>>> initAffinity
             initFut.listen(new IgniteInClosure>() {
                 @Override public void apply(IgniteInternalFuture initFut) {
                     try {
-                        resFut.onDone(initAffinityOnNodeLeft0(fut.initialVersion(), fut, NODE_TO_ID, false));
+                        resFut.onDone(initAffinityBasedOnPartitionsAvailability(fut.initialVersion(), fut, NODE_TO_ID, false));
                     }
                     catch (IgniteCheckedException e) {
                         resFut.onDone(e);
@@ -2077,10 +2110,13 @@ public IgniteInternalFuture>>> initAffinity
             return resFut;
         }
         else
-            return new GridFinishedFuture<>(initAffinityOnNodeLeft0(fut.initialVersion(), fut, NODE_TO_ID, false));
+            return new GridFinishedFuture<>(initAffinityBasedOnPartitionsAvailability(fut.initialVersion(), fut, NODE_TO_ID, false));
     }
 
     /**
+     * Initializes current affinity assignment based on partitions availability.
+     * Nodes that have most recent data will be considered affinity nodes.
+     *
      * @param topVer Topology version.
      * @param fut Exchange future.
      * @param c Closure converting affinity diff.
@@ -2088,12 +2124,17 @@ public IgniteInternalFuture>>> initAffinity
      * @return Affinity assignment.
      * @throws IgniteCheckedException If failed.
      */
-    private  Map>> initAffinityOnNodeLeft0(final AffinityTopologyVersion topVer,
+    private  Map>> initAffinityBasedOnPartitionsAvailability(final AffinityTopologyVersion topVer,
         final GridDhtPartitionsExchangeFuture fut,
         final IgniteClosure c,
         final boolean initAff)
         throws IgniteCheckedException {
-        final WaitRebalanceInfo waitRebalanceInfo = new WaitRebalanceInfo(fut.context().events().lastServerEventVersion());
+        final boolean enforcedCentralizedAssignment =
+            DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent());
+
+        final WaitRebalanceInfo waitRebalanceInfo = enforcedCentralizedAssignment ?
+            new WaitRebalanceInfo(fut.exchangeId().topologyVersion()) :
+            new WaitRebalanceInfo(fut.context().events().lastServerEventVersion());
 
         final Collection aliveNodes = fut.context().events().discoveryCache().serverNodes();
 
@@ -2103,13 +2144,14 @@ private  Map>> initAffinityOnNodeLeft0(final Af
             @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException {
                 CacheGroupHolder grpHolder = groupHolder(topVer, desc);
 
-                if (!grpHolder.rebalanceEnabled || fut.cacheGroupAddedOnExchange(desc.groupId(), desc.receivedFrom()))
+                if (!grpHolder.rebalanceEnabled ||
+                    (fut.cacheGroupAddedOnExchange(desc.groupId(), desc.receivedFrom()) && !enforcedCentralizedAssignment))
                     return;
 
                 AffinityTopologyVersion affTopVer = grpHolder.affinity().lastVersion();
 
-                assert affTopVer.topologyVersion() > 0 && !affTopVer.equals(topVer) : "Invalid affinity version " +
-                    "[last=" + affTopVer + ", futVer=" + topVer + ", grp=" + desc.cacheOrGroupName() + ']';
+                assert (affTopVer.topologyVersion() > 0 && !affTopVer.equals(topVer)) || enforcedCentralizedAssignment :
+                    "Invalid affinity version [last=" + affTopVer + ", futVer=" + topVer + ", grp=" + desc.cacheOrGroupName() + ']';
 
                 List> curAssignment = grpHolder.affinity().assignments(affTopVer);
                 List> newAssignment = grpHolder.affinity().idealAssignment();
@@ -2141,6 +2183,12 @@ private  Map>> initAffinityOnNodeLeft0(final Af
                         ", node=" + newPrimary +
                         ", topVer=" + topVer + ']';
 
+                    List owners = top.owners(p);
+
+                    // It is essential that curPrimary node has partition in OWNING state.
+                    if (!owners.isEmpty() && !owners.contains(curPrimary))
+                        curPrimary = owners.get(0);
+
                     if (curPrimary != null && newPrimary != null && !curPrimary.equals(newPrimary)) {
                         if (aliveNodes.contains(curPrimary)) {
                             GridDhtPartitionState state = top.partitionState(newPrimary.id(), p);
@@ -2173,8 +2221,6 @@ private  Map>> initAffinityOnNodeLeft0(final Af
                                 }
 
                                 if (newNodes0 == null) {
-                                    List owners = top.owners(p);
-
                                     for (ClusterNode owner : owners) {
                                         if (aliveNodes.contains(owner)) {
                                             newNodes0 = latePrimaryAssignment(grpHolder.affinity(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 08a910b81603d..2b2fb559c182e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -1186,6 +1186,7 @@ public void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) {
     /**
      * @param msg Message.
      * @param topVer Current topology version.
+     * @param curState Current cluster state.
      * @return Exchange action.
      * @throws IgniteCheckedException If configuration validation failed.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 9b9284f04b10a..8aa6db933a5d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -965,7 +965,7 @@ public void scheduleResendPartitions() {
      * For coordinator causes {@link GridDhtPartitionsFullMessage FullMessages} send,
      * for non coordinator -  {@link GridDhtPartitionsSingleMessage SingleMessages} send
      */
-    private void refreshPartitions() {
+    public void refreshPartitions() {
         // TODO https://issues.apache.org/jira/browse/IGNITE-6857
         if (cctx.snapshot().snapshotOperationInProgress()) {
             scheduleResendPartitions();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index e994113087ec8..def00f3f9ae89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -256,9 +256,9 @@ private String mapString(GridDhtPartitionMap map) {
     }
 
     /** {@inheritDoc} */
-    @Override public void initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
+    @Override public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
         GridDhtPartitionsExchangeFuture exchFut) {
-        // No-op.
+        return false;
     }
 
     /** {@inheritDoc} */
@@ -382,6 +382,11 @@ else if (!node2part.nodeId().equals(loc.id())) {
                 fullMapString() + ']');
     }
 
+    /** {@inheritDoc} */
+    @Override public void afterStateRestored(AffinityTopologyVersion topVer) {
+        // no-op
+    }
+
     /** {@inheritDoc} */
     @Override public boolean afterExchange(GridDhtPartitionsExchangeFuture exchFut) throws IgniteCheckedException {
         AffinityTopologyVersion topVer = exchFut.topologyVersion();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index e1f1d6f43ce1e..e63aab6c0dc78 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -213,9 +213,6 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
             // TODO ignite-db
             throw new IgniteException(e);
         }
-
-        // Todo log moving state
-        casState(state.get(), MOVING);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 4ae68ef739f8c..13564c2af2666 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -117,11 +117,19 @@ public void beforeExchange(GridDhtPartitionsExchangeFuture exchFut,
     /**
      * @param affVer Affinity version.
      * @param exchFut Exchange future.
+     * @return {@code True} if partitions must be refreshed.
      * @throws IgniteInterruptedCheckedException If interrupted.
      */
-    public void initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut)
+    public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut)
         throws IgniteInterruptedCheckedException;
 
+    /**
+     * Initializes local data structures after partitions are restored from persistence.
+     *
+     * @param topVer Topology version.
+     */
+    public void afterStateRestored(AffinityTopologyVersion topVer);
+
     /**
      * Post-initializes this topology.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 0a2c1541bd70d..020c3e7ad34bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -299,10 +299,12 @@ private String mapString(GridDhtPartitionMap map) {
     }
 
     /** {@inheritDoc} */
-    @Override public void initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
+    @Override public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer,
         GridDhtPartitionsExchangeFuture exchFut)
         throws IgniteInterruptedCheckedException
     {
+        boolean needRefresh;
+
         ctx.database().checkpointReadLock();
 
         try {
@@ -310,11 +312,11 @@ private String mapString(GridDhtPartitionMap map) {
 
             try {
                 if (stopping)
-                    return;
+                    return false;
 
                 long updateSeq = this.updateSeq.incrementAndGet();
 
-                initPartitions0(affVer, exchFut, updateSeq);
+                needRefresh = initPartitions0(affVer, exchFut, updateSeq);
 
                 consistencyCheck();
             }
@@ -325,16 +327,21 @@ private String mapString(GridDhtPartitionMap map) {
         finally {
             ctx.database().checkpointReadUnlock();
         }
+
+        return needRefresh;
     }
 
     /**
      * @param affVer Affinity version to use.
      * @param exchFut Exchange future.
      * @param updateSeq Update sequence.
+     * @return {@code True} if partitions must be refreshed.
      */
-    private void initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
+    private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
         List> aff = grp.affinity().readyAssignments(affVer);
 
+        boolean needRefresh = false;
+
         if (grp.affinityNode()) {
             ClusterNode loc = ctx.localNode();
 
@@ -378,6 +385,8 @@ private void initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsEx
                                     ", part=" + locPart + ']');
                             }
 
+                            needRefresh = true;
+
                             updateSeq = updateLocal(p, locPart.state(), updateSeq, affVer);
                         }
                     }
@@ -423,6 +432,8 @@ else if (belongs) {
         }
 
         updateRebalanceVersion(aff);
+
+        return needRefresh;
     }
 
     /**
@@ -616,6 +627,30 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
         return grp.affinity().nodes(p, topVer).contains(ctx.localNode());
     }
 
+    /** {@inheritDoc} */
+    @Override public void afterStateRestored(AffinityTopologyVersion topVer) {
+        lock.writeLock().lock();
+
+        try {
+            if (node2part == null)
+                return;
+
+            long updateSeq = this.updateSeq.incrementAndGet();
+
+            for (int p = 0; p < grp.affinity().partitions(); p++) {
+                GridDhtLocalPartition locPart = locParts.get(p);
+
+                if (locPart == null)
+                    updateLocal(p, EVICTED, updateSeq, topVer);
+                else
+                    updateLocal(p, locPart.state(), updateSeq, topVer);
+            }
+        }
+        finally {
+            lock.writeLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public boolean afterExchange(GridDhtPartitionsExchangeFuture exchFut) {
         boolean changed = false;
@@ -996,9 +1031,11 @@ else if (loc != null && state == RENTING && !showRenting) {
                 map.put(i, part.state());
             }
 
+            GridDhtPartitionMap locPartMap = node2part != null ? node2part.get(ctx.localNodeId()) : null;
+
             return new GridDhtPartitionMap(ctx.localNodeId(),
                 updateSeq.get(),
-                readyTopVer,
+                locPartMap != null ? locPartMap.topologyVersion() : readyTopVer,
                 map,
                 true);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index a45c9b99cd85d..695c840de302a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -130,6 +130,9 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     private static final int RELEASE_FUTURE_DUMP_THRESHOLD =
         IgniteSystemProperties.getInteger(IGNITE_PARTITION_RELEASE_FUTURE_DUMP_THRESHOLD, 0);
 
+    /** */
+    private static final IgniteProductVersion FORCE_AFF_REASSIGNMENT_SINCE = IgniteProductVersion.fromString("2.4.3");
+
     /** */
     @GridToStringExclude
     private final Object mux = new Object();
@@ -231,6 +234,12 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
      */
     private boolean centralizedAff;
 
+    /**
+     * Enforce affinity reassignment based on actual partition distribution. This mode should be used when partitions
+     * might be distributed not according to affinity assignment.
+     */
+    private boolean forceAffReassignment;
+
     /** Change global state exception. */
     private Exception changeGlobalStateE;
 
@@ -615,6 +624,9 @@ public void init(boolean newCrd) throws IgniteInterruptedCheckedException {
 
                 DiscoveryCustomMessage msg = ((DiscoveryCustomEvent)firstDiscoEvt).customMessage();
 
+                forceAffReassignment = DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(msg)
+                    && firstEventCache().minimumNodeVersion().compareToIgnoreTimestamp(FORCE_AFF_REASSIGNMENT_SINCE) >= 0;
+
                 if (msg instanceof ChangeGlobalStateMessage) {
                     assert exchActions != null && !exchActions.empty();
 
@@ -636,6 +648,9 @@ else if (msg instanceof WalStateAbstractMessage)
                     exchange = onAffinityChangeRequest(crdNode);
                 }
 
+                if (forceAffReassignment)
+                    cctx.affinity().onCentralizedAffinityChange(this, crdNode);
+
                 initCoordinatorCaches(newCrd);
             }
             else {
@@ -781,7 +796,7 @@ private void initTopologies() throws IgniteCheckedException {
                     if (grp.isLocal())
                         continue;
 
-                    grp.topology().beforeExchange(this, !centralizedAff, false);
+                    grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
                 }
             }
         }
@@ -924,8 +939,6 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
         else if (req.activate()) {
             // TODO: BLT changes on inactive cluster can't be handled easily because persistent storage hasn't been initialized yet.
             try {
-                cctx.affinity().onBaselineTopologyChanged(this, crd);
-
                 if (CU.isPersistenceEnabled(cctx.kernalContext().config()) && !cctx.kernalContext().clientNode())
                     cctx.kernalContext().state().onBaselineTopologyChanged(req.baselineTopology(),
                         req.prevBaselineTopologyHistoryItem());
@@ -962,7 +975,8 @@ private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedExcep
      * @return Exchange type.
      */
     private ExchangeType onCustomMessageNoAffinityChange(boolean crd) {
-        cctx.affinity().onCustomMessageNoAffinityChange(this, crd, exchActions);
+        if (!forceAffReassignment)
+            cctx.affinity().onCustomMessageNoAffinityChange(this, crd, exchActions);
 
         return cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL;
     }
@@ -1017,7 +1031,7 @@ private ExchangeType onServerNodeEvent(boolean crd) throws IgniteCheckedExceptio
 
             exchCtx.events().warnNoAffinityNodes(cctx);
 
-            centralizedAff = cctx.affinity().onServerLeft(this, crd);
+            centralizedAff = cctx.affinity().onCentralizedAffinityChange(this, crd);
         }
         else
             cctx.affinity().onServerJoin(this, crd);
@@ -1088,8 +1102,6 @@ private void distributedExchange() throws IgniteCheckedException {
             }
         }
 
-        cctx.database().beforeExchange(this);
-
         if (!exchCtx.mergeExchanges()) {
             for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                 if (grp.isLocal() || cacheGroupStopping(grp.groupId()))
@@ -1097,10 +1109,14 @@ private void distributedExchange() throws IgniteCheckedException {
 
                 // It is possible affinity is not initialized yet if node joins to cluster.
                 if (grp.affinity().lastVersion().topologyVersion() > 0)
-                    grp.topology().beforeExchange(this, !centralizedAff, false);
+                    grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
             }
         }
 
+        // It is necessary to run database callback after all topology callbacks, so partition states could be
+        // correctly restored from the persistent store.
+        cctx.database().beforeExchange(this);
+
         changeWalModeIfNeeded();
 
         if (crd.isLocal()) {
@@ -1534,19 +1550,24 @@ public void finishMerged() {
         }
 
         if (err == null) {
-            if (centralizedAff) {
+            if (centralizedAff || forceAffReassignment) {
                 assert !exchCtx.mergeExchanges();
 
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                     if (grp.isLocal())
                         continue;
 
+                    boolean needRefresh = false;
+
                     try {
-                        grp.topology().initPartitionsWhenAffinityReady(res, this);
+                        needRefresh = grp.topology().initPartitionsWhenAffinityReady(res, this);
                     }
                     catch (IgniteInterruptedCheckedException e) {
                         U.error(log, "Failed to initialize partitions.", e);
                     }
+
+                    if (needRefresh)
+                        cctx.exchange().refreshPartitions();
                 }
             }
 
@@ -2342,7 +2363,7 @@ private void onAllReceived(@Nullable Collection sndResNodes) {
             if (!exchCtx.mergeExchanges() && !crd.equals(events().discoveryCache().serverNodes().get(0))) {
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                     if (!grp.isLocal())
-                        grp.topology().beforeExchange(this, !centralizedAff, false);
+                        grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
                 }
             }
 
@@ -2474,6 +2495,9 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
                     detectLostPartitions(resTopVer);
             }
 
+            if (!exchCtx.mergeExchanges() && forceAffReassignment)
+                idealAffDiff = cctx.affinity().onCustomEventWithEnforcedAffinityReassignment(this);
+
             for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
                 if (!grpCtx.isLocal())
                     grpCtx.topology().applyUpdateCounters();
@@ -2496,6 +2520,8 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
                 if (exchCtx.events().hasServerLeft())
                     msg.idealAffinityDiff(idealAffDiff);
             }
+            else if (forceAffReassignment)
+                msg.idealAffinityDiff(idealAffDiff);
 
             msg.prepareMarshal(cctx);
 
@@ -2549,65 +2575,69 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
                         nodes.addAll(sndResNodes);
                 }
 
-                IgniteCheckedException err = null;
+                if (!nodes.isEmpty())
+                    sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff);
 
-                if (stateChangeExchange()) {
-                    StateChangeRequest req = exchActions.stateChangeRequest();
+                if (!stateChangeExchange())
+                    onDone(exchCtx.events().topologyVersion(), null);
 
-                    assert req != null : exchActions;
+                for (Map.Entry e : pendingSingleMsgs.entrySet()) {
+                    if (log.isInfoEnabled()) {
+                        log.info("Process pending message on coordinator [node=" + e.getKey() +
+                            ", ver=" + initialVersion() +
+                            ", resVer=" + resTopVer + ']');
+                    }
 
-                    boolean stateChangeErr = false;
+                    processSingleMessage(e.getKey(), e.getValue());
+                }
+            }
 
-                    if (!F.isEmpty(changeGlobalStateExceptions)) {
-                        stateChangeErr = true;
+            if (stateChangeExchange()) {
+                IgniteCheckedException err = null;
 
-                        err = new IgniteCheckedException("Cluster state change failed.");
+                StateChangeRequest req = exchActions.stateChangeRequest();
 
-                        cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req);
-                    }
-                    else {
-                        boolean hasMoving = !partsToReload.isEmpty();
+                assert req != null : exchActions;
 
-                        Set waitGrps = cctx.affinity().waitGroups();
+                boolean stateChangeErr = false;
 
-                        if (!hasMoving) {
-                            for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
-                                if (waitGrps.contains(grpCtx.groupId()) && grpCtx.topology().hasMovingPartitions()) {
-                                    hasMoving = true;
+                if (!F.isEmpty(changeGlobalStateExceptions)) {
+                    stateChangeErr = true;
 
-                                    break;
-                                }
+                    err = new IgniteCheckedException("Cluster state change failed.");
 
-                            }
-                        }
+                    cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req);
+                }
+                else {
+                    boolean hasMoving = !partsToReload.isEmpty();
 
-                        cctx.kernalContext().state().onExchangeFinishedOnCoordinator(this, hasMoving);
-                    }
+                    Set waitGrps = cctx.affinity().waitGroups();
 
-                    boolean active = !stateChangeErr && req.activate();
+                    if (!hasMoving) {
+                        for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
+                            if (waitGrps.contains(grpCtx.groupId()) && grpCtx.topology().hasMovingPartitions()) {
+                                hasMoving = true;
 
-                    ChangeGlobalStateFinishMessage stateFinishMsg = new ChangeGlobalStateFinishMessage(
-                        req.requestId(),
-                        active,
-                        !stateChangeErr);
+                                break;
+                            }
 
-                    cctx.discovery().sendCustomEvent(stateFinishMsg);
+                        }
+                    }
+
+                    cctx.kernalContext().state().onExchangeFinishedOnCoordinator(this, hasMoving);
                 }
 
-                if (!nodes.isEmpty())
-                    sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff);
+                boolean active = !stateChangeErr && req.activate();
 
-                onDone(exchCtx.events().topologyVersion(), err);
+                ChangeGlobalStateFinishMessage stateFinishMsg = new ChangeGlobalStateFinishMessage(
+                    req.requestId(),
+                    active,
+                    !stateChangeErr);
 
-                for (Map.Entry e : pendingSingleMsgs.entrySet()) {
-                    if (log.isInfoEnabled()) {
-                        log.info("Process pending message on coordinator [node=" + e.getKey() +
-                            ", ver=" + initialVersion() +
-                            ", resVer=" + resTopVer + ']');
-                    }
+                cctx.discovery().sendCustomEvent(stateFinishMsg);
 
-                    processSingleMessage(e.getKey(), e.getValue());
-                }
+                if (!centralizedAff)
+                    onDone(exchCtx.events().topologyVersion(), err);
             }
         }
         catch (IgniteCheckedException e) {
@@ -2950,7 +2980,7 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
                     cctx.affinity().onLocalJoin(this, msg, resTopVer);
                 else {
                     if (exchCtx.events().hasServerLeft())
-                        cctx.affinity().mergeExchangesOnServerLeft(this, msg);
+                        cctx.affinity().applyAffinityFromFullMessage(this, msg);
                     else
                         cctx.affinity().onServerJoinWithExchangeMergeProtocol(this, false);
 
@@ -2964,6 +2994,8 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
             }
             else if (localJoinExchange() && !exchCtx.fetchAffinityOnJoin())
                 cctx.affinity().onLocalJoin(this, msg, resTopVer);
+            else if (forceAffReassignment)
+                cctx.affinity().applyAffinityFromFullMessage(this, msg);
 
             updatePartitionFullMap(resTopVer, msg);
 
@@ -3073,6 +3105,9 @@ public void onAffinityChangeMessage(final ClusterNode node, final CacheAffinityC
                             crd.isLocal(),
                             msg);
 
+                        IgniteCheckedException err = !F.isEmpty(msg.partitionsMessage().getErrorsMap()) ?
+                            new IgniteCheckedException("Cluster state change failed.") : null;
+
                         if (!crd.isLocal()) {
                             GridDhtPartitionsFullMessage partsMsg = msg.partitionsMessage();
 
@@ -3080,9 +3115,12 @@ public void onAffinityChangeMessage(final ClusterNode node, final CacheAffinityC
                             assert partsMsg.lastVersion() != null : partsMsg;
 
                             updatePartitionFullMap(resTopVer, partsMsg);
+
+                            if (exchActions != null && exchActions.stateChangeRequest() != null && err != null)
+                                cctx.kernalContext().state().onStateChangeError(msg.partitionsMessage().getErrorsMap(), exchActions.stateChangeRequest());
                         }
 
-                        onDone(resTopVer);
+                        onDone(resTopVer, err);
                     }
                     else {
                         if (log.isDebugEnabled()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 5dc81c5e7ad48..f1cc4ce562574 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -112,6 +112,8 @@
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
@@ -2344,6 +2346,9 @@ else if (restore != null) {
                     updateState(part, restore.get1());
                 }
             }
+
+            // After partition states are restored, it is necessary to update internal data structures in topology.
+            grp.topology().afterStateRestored(grp.topology().lastTopologyChangeVersion());
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 7b40b03cae393..6ccb450319180 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -25,11 +25,16 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
+import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.AffinityFunction;
+import org.apache.ignite.cache.affinity.AffinityFunctionContext;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -40,8 +45,15 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.TestDelayingCommunicationSpi;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -62,6 +74,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
     /** */
     private static final int NODE_COUNT = 4;
 
+    /** */
+    private static boolean delayRebalance = false;
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
@@ -106,6 +121,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
         if (client)
             cfg.setClientMode(true);
 
+        if (delayRebalance)
+            cfg.setCommunicationSpi(new DelayRebalanceCommunicationSpi());
+
         return cfg;
     }
 
@@ -594,7 +612,10 @@ else if (grid(i).localNode().equals(affNodes.get(1))) {
         assertEquals(val2, primary.cache(CACHE_NAME).get(key));
         assertEquals(val2, backup.cache(CACHE_NAME).get(key));
 
-        primary.cache(CACHE_NAME).rebalance().get();
+        for (int i = 0; i < NODE_COUNT; i++)
+            grid(i).cache(CACHE_NAME).rebalance().get();
+
+        awaitPartitionMapExchange();
 
         affNodes = (List) ig.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
 
@@ -697,6 +718,83 @@ public void testNonPersistentCachesIgnoreBaselineTopology() throws Exception {
         assertTrue(ig.affinity(inMemoryCache.getName()).allPartitions(newNode.cluster().localNode()).length > 0);
     }
 
+    /**
+     * @throws Exception if failed.
+     */
+    public void testAffinityAssignmentChangedAfterRestart() throws Exception {
+        delayRebalance = false;
+
+        int parts = 32;
+
+        final List partMapping = new ArrayList<>();
+
+        for (int p = 0; p < parts; p++)
+            partMapping.add(p);
+
+        final AffinityFunction affFunc = new TestAffinityFunction(new RendezvousAffinityFunction(false, parts));
+
+        TestAffinityFunction.partsAffMapping = partMapping;
+
+        String cacheName = CACHE_NAME + 2;
+
+        startGrids(4);
+
+        IgniteEx ig = grid(0);
+
+        ig.cluster().active(true);
+
+        IgniteCache cache = ig.createCache(
+            new CacheConfiguration()
+                .setName(cacheName)
+                .setCacheMode(PARTITIONED)
+                .setBackups(1)
+                .setPartitionLossPolicy(READ_ONLY_SAFE)
+                .setReadFromBackup(true)
+                .setWriteSynchronizationMode(FULL_SYNC)
+                .setRebalanceDelay(-1)
+                .setAffinity(affFunc));
+
+        Map keyToConsId = new HashMap<>();
+
+        for (int k = 0; k < 1000; k++) {
+            cache.put(k, k);
+
+            keyToConsId.put(k, ig.affinity(cacheName).mapKeyToNode(k).consistentId().toString());
+        }
+
+        stopAllGrids();
+
+        Collections.shuffle(TestAffinityFunction.partsAffMapping, new Random(1));
+
+        delayRebalance = true;
+
+        startGrids(4);
+
+        ig = grid(0);
+
+        ig.active(true);
+
+        cache = ig.cache(cacheName);
+
+        GridDhtPartitionFullMap partMap = ig.cachex(cacheName).context().topology().partitionMap(false);
+
+        for (int i = 1; i < 4; i++) {
+            IgniteEx ig0 = grid(i);
+
+            for (int p = 0; p < 32; p++)
+                assertEqualsCollections(ig.affinity(cacheName).mapPartitionToPrimaryAndBackups(p), ig0.affinity(cacheName).mapPartitionToPrimaryAndBackups(p));
+        }
+
+        for (Map.Entry e : keyToConsId.entrySet()) {
+            int p = ig.affinity(cacheName).partition(e.getKey());
+
+            assertEquals("p=" + p, GridDhtPartitionState.OWNING, partMap.get(ig.affinity(cacheName).mapKeyToNode(e.getKey()).id()).get(p));
+        }
+
+        for (int k = 0; k < 1000; k++)
+            assertEquals("k=" + k, Integer.valueOf(k), cache.get(k));
+    }
+
     /** */
     private Collection baselineNodes(Collection clNodes) {
         Collection res = new ArrayList<>(clNodes.size());
@@ -758,4 +856,73 @@ private TestValue(int a) {
             return result;
         }
     }
+
+    /**
+     *
+     */
+    private static class TestAffinityFunction implements AffinityFunction {
+        /** */
+        private final AffinityFunction delegate;
+
+        /** */
+        private static List partsAffMapping;
+
+        /** */
+        public TestAffinityFunction(AffinityFunction delegate) {
+            this.delegate = delegate;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void reset() {
+            delegate.reset();;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int partitions() {
+            return delegate.partitions();
+        }
+
+        /** {@inheritDoc} */
+        @Override public int partition(Object key) {
+            return delegate.partition(key);
+        }
+
+        /** {@inheritDoc} */
+        @Override public List> assignPartitions(AffinityFunctionContext affCtx) {
+            List> res0 = delegate.assignPartitions(affCtx);
+
+            List> res = new ArrayList<>(res0.size());
+
+            for (int p = 0; p < res0.size(); p++)
+                res.add(p, null);
+
+            for (int p = 0; p < res0.size(); p++)
+                res.set(partsAffMapping.get(p), res0.get(p));
+
+            return res;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void removeNode(UUID nodeId) {
+            delegate.removeNode(nodeId);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class DelayRebalanceCommunicationSpi extends TestDelayingCommunicationSpi {
+        /** {@inheritDoc} */
+        @Override protected boolean delayMessage(Message msg, GridIoMessage ioMsg) {
+            if (msg != null && (msg instanceof GridDhtPartitionDemandMessage || msg instanceof GridDhtPartitionSupplyMessage))
+                return true;
+
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected int delayMillis() {
+            return 1_000_000;
+        }
+    }
 }

From d3745e9d0a3ff5a64fba494889b7e2605f3af6bb Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk 
Date: Wed, 7 Feb 2018 21:10:32 +0300
Subject: [PATCH 61/65] IGNITE-7639 Fixed NPE

---
 .../cluster/DiscoveryDataClusterState.java    |  41 +++-
 .../IgniteClusterActivateDeactivateTest.java  | 222 ++++++++----------
 2 files changed, 128 insertions(+), 135 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index dea2ce79c311e..b022754c6670a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -28,22 +28,36 @@
 import org.jetbrains.annotations.Nullable;
 
 /**
- * Discovery data related to cluster state.
+ * A pojo-object representing current cluster global state. The state includes cluster active flag and cluster
+ * baseline topology.
+ * 

+ * This object also captures a transitional cluster state, when one or more fields are changing. In this case,
+ * a {@code transitionReqId} field is set to a non-null value and {@code prevState} captures previous cluster state.
+ * A joining node catching the cluster in an intermediate state will observe {@code transitionReqId} field to be
+ * non-null, however the {@code prevState} will not be sent to the joining node.
+ *
+ * TODO https://issues.apache.org/jira/browse/IGNITE-7640 This class must be immutable, transitionRes must be set by calling finish().
  */
 public class DiscoveryDataClusterState implements Serializable {
     /** */
     private static final long serialVersionUID = 0L;
 
-    /** */
+    /** Flag indicating if the cluster in in active state. */
     private final boolean active;
 
-    /** */
+    /** Current cluster baseline topology. */
     @Nullable private final BaselineTopology baselineTopology;
 
-    /** */
+    /**
+     * Transition request ID. Set to a non-null value if the cluster is changing it's state.
+     * The ID is assigned on the initiating node.
+     */
     private final UUID transitionReqId;
 
-    /** Topology version for state change exchange. */
+    /**
+     * Topology version in the cluster when state change request was received by the coordinator.
+     * The exchange fired for the cluster state change will be on version {@code transitionTopVer.nextMinorVersion()}.
+     */
     @GridToStringInclude
     private final AffinityTopologyVersion transitionTopVer;
 
@@ -51,13 +65,18 @@ public class DiscoveryDataClusterState implements Serializable {
     @GridToStringExclude
     private final Set transitionNodes;
 
-    /** Local flag for state transition result (global state is updated asynchronously by custom message). */
+    /**
+     * Local flag for state transition active state result (global state is updated asynchronously by custom message),
+     * {@code null} means that state change is not completed yet.
+     */
     private transient volatile Boolean transitionRes;
 
-    /** */
+    /**
+     * Previous cluster state if this state is a transition state and it was not received by a joining node.
+     */
     private transient DiscoveryDataClusterState prevState;
 
-    /** */
+    /** Transition result error. */
     private transient volatile Exception transitionError;
 
     /**
@@ -86,6 +105,7 @@ static DiscoveryDataClusterState createTransitionState(
         assert transitionReqId != null;
         assert transitionTopVer != null;
         assert !F.isEmpty(transitionNodes) : transitionNodes;
+        assert prevState != null;
 
         return new DiscoveryDataClusterState(
             prevState,
@@ -156,7 +176,7 @@ public boolean transition() {
      * @return {@code True} if cluster active state change is in progress, {@code false} otherwise.
      */
     public boolean activeStateChanging() {
-        return transition() && active != prevState.active;
+        return transition() && (prevState == null || (prevState.active != active));
     }
 
     /**
@@ -202,6 +222,9 @@ public void transitionError(Exception ex) {
     }
 
     /**
+     * Creates a non-transitional cluster state. This method effectively cleans all fields identifying the
+     * state as transitional and creates a new state with the state transition result.
+     *
      * @param success Transition success status.
      * @return Cluster state that finished transition.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 71718c9d16cc4..2337329c5ce83 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -37,7 +37,6 @@
 import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
@@ -49,7 +48,6 @@
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -84,9 +82,6 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
     /** */
     private boolean testSpi;
 
-    /** */
-    private boolean testDiscoSpi;
-
     /** */
     private boolean testReconnectSpi;
 
@@ -104,8 +99,6 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
 
             spi.setJoinTimeout(2 * 60_000);
         }
-        else if (testDiscoSpi)
-            cfg.setDiscoverySpi(new TestTcpDiscoverySpi());
 
         ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
 
@@ -220,14 +213,14 @@ private void activateSimple(int srvs, int clients, int activateFrom) throws Exce
         }
 
         for (int i = 0; i < srvs + clients; i++)
-            assertFalse(ignite(i).active());
+            assertFalse(ignite(i).cluster().active());
 
-        ignite(activateFrom).active(false); // Should be no-op.
+        ignite(activateFrom).cluster().active(false); // Should be no-op.
 
-        ignite(activateFrom).active(true);
+        ignite(activateFrom).cluster().active(true);
 
         for (int i = 0; i < srvs + clients; i++)
-            assertTrue(ignite(i).active());
+            assertTrue(ignite(i).cluster().active());
 
         for (int i = 0; i < srvs + clients; i++) {
             for (int c = 0; c < 2; c++)
@@ -308,16 +301,14 @@ public void testJoinWhileActivate1_Client() throws Exception {
     private void joinWhileActivate1(final boolean startClient, final boolean withNewCache) throws Exception {
         IgniteInternalFuture activeFut = startNodesAndBlockStatusChange(2, 0, 0, false);
 
-        IgniteInternalFuture startFut = GridTestUtils.runAsync(new Callable() {
-            @Override public Void call() throws Exception {
-                client = startClient;
+        IgniteInternalFuture startFut = GridTestUtils.runAsync((Callable)() -> {
+            client = startClient;
 
-                ccfgs = withNewCache ? cacheConfigurations2() : cacheConfigurations1();
+            ccfgs = withNewCache ? cacheConfigurations2() : cacheConfigurations1();
 
-                startGrid(2);
+            startGrid(2);
 
-                return null;
-            }
+            return null;
         });
 
         TestRecordingCommunicationSpi spi1 = TestRecordingCommunicationSpi.spi(ignite(1));
@@ -376,7 +367,7 @@ private IgniteInternalFuture startNodesAndBlockStatusChange(int srvs,
         int minorVer = 1;
 
         if (initiallyActive && persistenceEnabled()) {
-            ignite(0).active(true);
+            ignite(0).cluster().active(true);
 
             minorVer++;
         }
@@ -396,11 +387,9 @@ private IgniteInternalFuture startNodesAndBlockStatusChange(int srvs,
             blockExchangeSingleMessage(spi, STATE_CHANGE_TOP_VER);
         }
 
-        IgniteInternalFuture stateChangeFut = GridTestUtils.runAsync(new Runnable() {
-            @Override public void run() {
-                ignite(stateChangeFrom).active(!initiallyActive);
-            }
-        });
+        IgniteInternalFuture stateChangeFut = GridTestUtils.runAsync(() ->
+            ignite(stateChangeFrom).cluster().active(!initiallyActive)
+        );
 
         for (TestRecordingCommunicationSpi spi : spis)
             spi.waitForBlocked();
@@ -417,17 +406,15 @@ private IgniteInternalFuture startNodesAndBlockStatusChange(int srvs,
      * @param topVer Exchange topology version.
      */
     private void blockExchangeSingleMessage(TestRecordingCommunicationSpi spi, final AffinityTopologyVersion topVer) {
-        spi.blockMessages(new IgniteBiPredicate() {
-            @Override public boolean apply(ClusterNode clusterNode, Message msg) {
-                if (msg instanceof GridDhtPartitionsSingleMessage) {
-                    GridDhtPartitionsSingleMessage pMsg = (GridDhtPartitionsSingleMessage)msg;
-
-                    if (pMsg.exchangeId() != null && pMsg.exchangeId().topologyVersion().equals(topVer))
-                        return true;
-                }
+        spi.blockMessages((IgniteBiPredicate)(clusterNode, msg) -> {
+            if (msg instanceof GridDhtPartitionsSingleMessage) {
+                GridDhtPartitionsSingleMessage pMsg = (GridDhtPartitionsSingleMessage)msg;
 
-                return false;
+                if (pMsg.exchangeId() != null && pMsg.exchangeId().topologyVersion().equals(topVer))
+                    return true;
             }
+
+            return false;
         });
     }
 
@@ -460,16 +447,14 @@ public void testJoinWhileDeactivate1_Client() throws Exception {
     private void joinWhileDeactivate1(final boolean startClient, final boolean withNewCache) throws Exception {
         IgniteInternalFuture activeFut = startNodesAndBlockStatusChange(2, 0, 0, true);
 
-        IgniteInternalFuture startFut = GridTestUtils.runAsync(new Callable() {
-            @Override public Void call() throws Exception {
-                client = startClient;
+        IgniteInternalFuture startFut = GridTestUtils.runAsync((Callable)() -> {
+            client = startClient;
 
-                ccfgs = withNewCache ? cacheConfigurations2() : cacheConfigurations1();
+            ccfgs = withNewCache ? cacheConfigurations2() : cacheConfigurations1();
 
-                startGrid(2);
+            startGrid(2);
 
-                return null;
-            }
+            return null;
         });
 
         TestRecordingCommunicationSpi spi1 = TestRecordingCommunicationSpi.spi(ignite(1));
@@ -481,7 +466,7 @@ private void joinWhileDeactivate1(final boolean startClient, final boolean withN
 
         checkNoCaches(3);
 
-        ignite(2).active(true);
+        ignite(2).cluster().active(true);
 
         for (int c = 0; c < 2; c++)
             checkCache(ignite(2), CACHE_NAME_PREFIX + c, true);
@@ -529,30 +514,26 @@ public void testConcurrentJoinAndActivate() throws Exception {
 
             final CyclicBarrier b = new CyclicBarrier(START_NODES + 1);
 
-            IgniteInternalFuture fut1 = GridTestUtils.runAsync(new Callable() {
-                @Override public Void call() throws Exception {
-                    b.await();
+            IgniteInternalFuture fut1 = GridTestUtils.runAsync(() -> {
+                b.await();
 
-                    Thread.sleep(ThreadLocalRandom.current().nextLong(100) + 1);
+                U.sleep(ThreadLocalRandom.current().nextLong(100) + 1);
 
-                    ignite(0).active(true);
+                ignite(0).cluster().active(true);
 
-                    return null;
-                }
+                return null;
             });
 
             final AtomicInteger nodeIdx = new AtomicInteger(3);
 
-            IgniteInternalFuture fut2 = GridTestUtils.runMultiThreadedAsync(new Callable() {
-                @Override public Void call() throws Exception {
-                    int idx = nodeIdx.getAndIncrement();
+            IgniteInternalFuture fut2 = GridTestUtils.runMultiThreadedAsync((Callable)() -> {
+                int idx = nodeIdx.getAndIncrement();
 
-                    b.await();
+                b.await();
 
-                    startGrid(idx);
+                startGrid(idx);
 
-                    return null;
-                }
+                return null;
             }, START_NODES, "start-node");
 
             fut1.get();
@@ -619,19 +600,19 @@ private void deactivateSimple(int srvs, int clients, int deactivateFrom) throws
         }
 
         if (persistenceEnabled())
-            ignite(deactivateFrom).active(true);
+            ignite(deactivateFrom).cluster().active(true);
 
-        ignite(deactivateFrom).active(true); // Should be no-op.
+        ignite(deactivateFrom).cluster().active(true); // Should be no-op.
 
         checkCaches(srvs + clients, CACHES);
 
         for (int i = 0; i < srvs + clients; i++)
-            assertTrue(ignite(i).active());
+            assertTrue(ignite(i).cluster().active());
 
-        ignite(deactivateFrom).active(false);
+        ignite(deactivateFrom).cluster().active(false);
 
         for (int i = 0; i < srvs + clients; i++)
-            assertFalse(ignite(i).active());
+            assertFalse(ignite(i).cluster().active());
 
         checkNoCaches(srvs + clients);
 
@@ -648,12 +629,12 @@ private void deactivateSimple(int srvs, int clients, int deactivateFrom) throws
         checkNoCaches(srvs + clients + 2);
 
         for (int i = 0; i < srvs + clients + 2; i++)
-            assertFalse(ignite(i).active());
+            assertFalse(ignite(i).cluster().active());
 
-        ignite(deactivateFrom).active(true);
+        ignite(deactivateFrom).cluster().active(true);
 
         for (int i = 0; i < srvs + clients + 2; i++) {
-            assertTrue(ignite(i).active());
+            assertTrue(ignite(i).cluster().active());
 
             checkCache(ignite(i), CU.UTILITY_CACHE_NAME, true);
         }
@@ -695,7 +676,7 @@ public void testClientReconnectClusterActive() throws Exception {
         startWithCaches1(SRVS, CLIENTS);
 
         if (persistenceEnabled())
-            ignite(0).active(true);
+            ignite(0).cluster().active(true);
 
         Ignite srv = ignite(0);
         Ignite client = ignite(SRVS);
@@ -741,7 +722,7 @@ public void testClientReconnectClusterInactive() throws Exception {
 
         checkNoCaches(SRVS + CLIENTS);
 
-        ignite(0).active(true);
+        ignite(0).cluster().active(true);
 
         checkCache(client, CU.UTILITY_CACHE_NAME, true);
 
@@ -789,39 +770,38 @@ private void clientReconnectClusterDeactivated(final boolean transition) throws
         IgniteEx client = grid(SRVS);
 
         if (persistenceEnabled())
-            ignite(0).active(true);
+            ignite(0).cluster().active(true);
 
         checkCache(client, CU.UTILITY_CACHE_NAME, true);
 
         checkCaches1(SRVS + CLIENTS);
 
+        // Wait for late affinity assignment to finish.
+        grid(0).context().cache().context().exchange().affinityReadyFuture(
+            new AffinityTopologyVersion(SRVS + CLIENTS, 1)).get();
+
         final AffinityTopologyVersion STATE_CHANGE_TOP_VER = new AffinityTopologyVersion(SRVS + CLIENTS + 1, 1);
 
         final TestRecordingCommunicationSpi spi1 = transition ? TestRecordingCommunicationSpi.spi(ignite(1)) : null;
 
         final AtomicReference stateFut = new AtomicReference<>();
 
-        IgniteClientReconnectAbstractTest.reconnectClientNode(log, client, srv, new Runnable() {
-            @Override public void run() {
-                if (transition) {
-                    blockExchangeSingleMessage(spi1, STATE_CHANGE_TOP_VER);
-
-                    stateFut.set(GridTestUtils.runAsync(new Runnable() {
-                        @Override public void run() {
-                            srv.active(false);
-                        }
-                    }, "deactivate"));
-
-                    try {
-                        U.sleep(500);
-                    }
-                    catch (Exception e) {
-                        e.printStackTrace();
-                    }
+        IgniteClientReconnectAbstractTest.reconnectClientNode(log, client, srv, () -> {
+            if (transition) {
+                blockExchangeSingleMessage(spi1, STATE_CHANGE_TOP_VER);
+
+                stateFut.set(GridTestUtils.runAsync(() -> srv.cluster().active(false),
+                    "deactivate"));
+
+                try {
+                    U.sleep(500);
+                }
+                catch (Exception e) {
+                    e.printStackTrace();
                 }
-                else
-                    srv.active(false);
             }
+            else
+                srv.cluster().active(false);
         });
 
         if (transition) {
@@ -839,11 +819,11 @@ private void clientReconnectClusterDeactivated(final boolean transition) throws
 
         checkNoCaches(SRVS + CLIENTS);
 
-        ignite(0).active(true);
+        ignite(0).cluster().active(true);
 
         checkCache(client, CU.UTILITY_CACHE_NAME, true);
 
-        assertTrue(client.active());
+        assertTrue(client.cluster().active());
 
         checkCaches1(SRVS + CLIENTS);
 
@@ -900,27 +880,22 @@ private void clientReconnectClusterActivated(final boolean transition) throws Ex
 
         final AtomicReference stateFut = new AtomicReference<>();
 
-        IgniteClientReconnectAbstractTest.reconnectClientNode(log, client, srv, new Runnable() {
-            @Override public void run() {
-                if (transition) {
-                    blockExchangeSingleMessage(spi1, STATE_CHANGE_TOP_VER);
-
-                    stateFut.set(GridTestUtils.runAsync(new Runnable() {
-                        @Override public void run() {
-                            srv.active(true);
-                        }
-                    }, "activate"));
-
-                    try {
-                        U.sleep(500);
-                    }
-                    catch (Exception e) {
-                        e.printStackTrace();
-                    }
+        IgniteClientReconnectAbstractTest.reconnectClientNode(log, client, srv, () -> {
+            if (transition) {
+                blockExchangeSingleMessage(spi1, STATE_CHANGE_TOP_VER);
+
+                stateFut.set(GridTestUtils.runAsync(() -> srv.cluster().active(true),
+                    "activate"));
+
+                try {
+                    U.sleep(500);
+                }
+                catch (Exception e) {
+                    e.printStackTrace();
                 }
-                else
-                    srv.active(true);
             }
+            else
+                srv.cluster().active(true);
         });
 
         if (transition) {
@@ -989,7 +964,7 @@ public void testInactiveTopologyChanges() throws Exception {
 
         checkRecordedMessages(false);
 
-        ignite(0).active(true);
+        ignite(0).cluster().active(true);
 
         checkCaches1(SRVS + CLIENTS);
 
@@ -1033,12 +1008,10 @@ private void stateChangeFailover1(boolean activate) throws Exception {
         client = false;
 
         // Start one more node while transition is in progress.
-        IgniteInternalFuture startFut = GridTestUtils.runAsync(new Callable() {
-            @Override public Object call() throws Exception {
-                startGrid(8);
+        IgniteInternalFuture startFut = GridTestUtils.runAsync(() -> {
+            startGrid(8);
 
-                return null;
-            }
+            return null;
         }, "start-node");
 
         U.sleep(500);
@@ -1061,7 +1034,7 @@ private void stateChangeFailover1(boolean activate) throws Exception {
         if (!activate) {
             checkNoCaches(9);
 
-            ignite(0).active(true);
+            ignite(0).cluster().active(true);
         }
 
         checkCaches1(9);
@@ -1092,19 +1065,16 @@ private void stateChangeFailover2(boolean activate) throws Exception {
         client = false;
 
         // Start more nodes while transition is in progress.
-        IgniteInternalFuture startFut1 = GridTestUtils.runAsync(new Callable() {
-            @Override public Object call() throws Exception {
-                startGrid(8);
+        IgniteInternalFuture startFut1 = GridTestUtils.runAsync(() -> {
+            startGrid(8);
 
-                return null;
-            }
+            return null;
         }, "start-node1");
-        IgniteInternalFuture startFut2 = GridTestUtils.runAsync(new Callable() {
-            @Override public Object call() throws Exception {
-                startGrid(9);
 
-                return null;
-            }
+        IgniteInternalFuture startFut2 = GridTestUtils.runAsync(() -> {
+            startGrid(9);
+
+            return null;
         }, "start-node2");
 
         U.sleep(500);
@@ -1132,7 +1102,7 @@ private void stateChangeFailover2(boolean activate) throws Exception {
         if (!activate) {
             checkNoCaches(10);
 
-            ignite(0).active(true);
+            ignite(0).cluster().active(true);
         }
 
         checkCaches1(10);
@@ -1214,7 +1184,7 @@ void checkCache(Ignite node, String cacheName, boolean exp) throws IgniteChecked
 
         ((IgniteEx)node).context().state().publicApiActiveState(true);
 
-        GridCacheAdapter cache = ((IgniteKernal)node).context().cache().internalCache(cacheName);
+        GridCacheAdapter cache = ((IgniteEx)node).context().cache().internalCache(cacheName);
 
         if (exp)
             assertNotNull("Cache not found [cache=" + cacheName + ", node=" + node.name() + ']', cache);
@@ -1229,7 +1199,7 @@ final void checkNoCaches(int nodes) {
         for (int i = 0; i < nodes; i++) {
             grid(i).context().state().publicApiActiveState(true);
 
-            GridCacheProcessor cache = ((IgniteKernal)ignite(i)).context().cache();
+            GridCacheProcessor cache = ((IgniteEx)ignite(i)).context().cache();
 
             assertTrue(cache.caches().isEmpty());
             assertTrue(cache.internalCaches().isEmpty());

From f7c16855ba802d9d47048521aec7e14285e4a281 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko 
Date: Fri, 9 Feb 2018 16:55:15 +0300
Subject: [PATCH 62/65] IGNITE-7540 Prevent page memory metadata corruption
 during checkpoint and group destroying. - Fixes #3490.

Signed-off-by: Alexey Goncharuk 
---
 .../processors/cache/GridCacheProcessor.java  |  19 +++
 ...tePdsCacheDestroyDuringCheckpointTest.java | 161 ++++++++++++++++++
 .../IgnitePdsWithIndexingCoreTestSuite.java   |   2 +
 3 files changed, 182 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 1561f25f8e0e4..ec456e17e795d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -89,6 +89,7 @@
 import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
 import org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
@@ -2232,6 +2233,24 @@ public void onExchangeDone(
                 }
             }
 
+            sharedCtx.database().checkpointReadLock();
+
+            try {
+                // Do not invoke checkpoint listeners for groups are going to be destroyed to prevent metadata corruption.
+                for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) {
+                    Integer groupId = action.descriptor().groupId();
+                    CacheGroupContext grp = cacheGrps.get(groupId);
+
+                    if (grp != null && grp.persistenceEnabled() && sharedCtx.database() instanceof GridCacheDatabaseSharedManager) {
+                        GridCacheDatabaseSharedManager mngr = (GridCacheDatabaseSharedManager) sharedCtx.database();
+                        mngr.removeCheckpointListener((DbCheckpointListener) grp.offheap());
+                    }
+                }
+            }
+            finally {
+                sharedCtx.database().checkpointReadUnlock();
+            }
+
             List> stoppedGroups = new ArrayList<>();
 
             for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java
new file mode 100644
index 0000000000000..72f73aaadb8d5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.file;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ * Test for cache creation/deletion with frequent checkpoints.
+ */
+public class IgnitePdsCacheDestroyDuringCheckpointTest extends GridCommonAbstractTest {
+    /** */
+    private static final String NAME_PREFIX = "CACHE-";
+
+    /** */
+    private static final int NUM_ITERATIONS = 10;
+
+    /** */
+    private static final int NUM_CACHES = 10;
+
+    /** */
+    private static final int NUM_ENTRIES_PER_CACHE = 200;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setDataStorageConfiguration(createDbConfig());
+
+        if (getTestIgniteInstanceIndex(gridName) == 1)
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    /**
+     * @return DB config.
+     */
+    private DataStorageConfiguration createDbConfig() {
+        DataStorageConfiguration storageCfg = new DataStorageConfiguration();
+        storageCfg.setCheckpointFrequency(300);
+
+        DataRegionConfiguration regionConfig = new DataRegionConfiguration();
+        regionConfig.setPersistenceEnabled(true);
+
+        storageCfg.setDefaultDataRegionConfiguration(regionConfig);
+
+        return storageCfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        deleteWorkFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+
+        deleteWorkFiles();
+    }
+
+    /**
+     * @throws Exception If fail.
+     */
+    public void testCacheCreatePutCheckpointDestroy() throws Exception {
+        IgniteEx ig = startGrid(0);
+        ig.active(true);
+
+        for (int j = 0; j < NUM_ITERATIONS; j++) {
+            Ignite client = startGrid(1);
+
+            for (int i = 0; i < NUM_CACHES; i++) {
+                IgniteCache cache = ig.cache(NAME_PREFIX + i);
+                if (cache != null)
+                    cache.destroy();
+            }
+
+            populateCache(client);
+            checkCacheSizes(client);
+
+            client.close();
+        }
+    }
+
+
+    /** */
+    private void populateCache(Ignite client) {
+        for (int i = 0; i < NUM_CACHES; i++) {
+            CacheConfiguration cfg = new CacheConfiguration();
+            cfg.setName(NAME_PREFIX + i).setAtomicityMode(CacheAtomicityMode.ATOMIC)
+                    .setBackups(1).setStatisticsEnabled(true).setManagementEnabled(true);
+            client.getOrCreateCache(cfg);
+
+            IgniteDataStreamer streamer = client.dataStreamer(NAME_PREFIX + i);
+
+            for (int j = 0; j < NUM_ENTRIES_PER_CACHE; j++) {
+                String bo = i + "|" + j + "|WHATEVER";
+                streamer.addData(j, bo);
+            }
+
+            streamer.close();
+            log.info("Streamer closed");
+        }
+    }
+
+    /** */
+    private void checkCacheSizes(Ignite client) {
+        for (int i = 0; i < NUM_CACHES; i++) {
+            IgniteCache cache = client.getOrCreateCache(NAME_PREFIX + i);
+
+            int size = cache.size(CachePeekMode.ALL);
+
+            if (NUM_ENTRIES_PER_CACHE != size) {
+                for (Object o : cache) {
+                    log.info("O " + o);
+                }
+                assertTrue(false);
+            }
+        }
+    }
+
+    /**
+     * @throws IgniteCheckedException If fail.
+     */
+    private void deleteWorkFiles() throws IgniteCheckedException {
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+    }
+}
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 9f86e0db08107..d3256c2a834da 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheDestroyDuringCheckpointTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheIntegrationTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsDiskErrorsRecoveringTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsNoActualWalHistoryTest;
@@ -59,6 +60,7 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteWalRecoveryPPCTest.class);
         suite.addTestSuite(IgnitePdsDiskErrorsRecoveringTest.class);
+        suite.addTestSuite(IgnitePdsCacheDestroyDuringCheckpointTest.class);
 
         suite.addTestSuite(IgnitePdsBinaryMetadataOnClusterRestartTest.class);
         suite.addTestSuite(IgnitePdsMarshallerMappingRestoreOnNodeStartTest.class);

From c92f167fc491078f02b9f94fe89edafc2902ebc2 Mon Sep 17 00:00:00 2001
From: ilantukh 
Date: Wed, 14 Feb 2018 15:40:13 +0300
Subject: [PATCH 63/65] Updated version in properties.

---
 modules/core/src/main/resources/ignite.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index 1f1f4d05acfc3..a4dffff1056aa 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.4.0-SNAPSHOT
+ignite.version=2.4.3-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970

From a89230b25f06cccfb14141321d46046d0243f79b Mon Sep 17 00:00:00 2001
From: ilantukh 
Date: Wed, 14 Feb 2018 17:07:58 +0300
Subject: [PATCH 64/65] gg-13475 : Fixed handling of state change message in
 compatibility mode. Decreased ignite.version to test on TC.

---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java   | 9 +++++++++
 modules/core/src/main/resources/ignite.properties        | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 695c840de302a..afb9055664951 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -939,6 +939,15 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
         else if (req.activate()) {
             // TODO: BLT changes on inactive cluster can't be handled easily because persistent storage hasn't been initialized yet.
             try {
+                if (!forceAffReassignment) {
+                    // possible only if cluster contains nodes without forceAffReassignment mode
+                    assert firstEventCache().minimumNodeVersion()
+                        .compareToIgnoreTimestamp(FORCE_AFF_REASSIGNMENT_SINCE) < 0
+                        : firstEventCache().minimumNodeVersion();
+
+                    cctx.affinity().onBaselineTopologyChanged(this, crd);
+                }
+
                 if (CU.isPersistenceEnabled(cctx.kernalContext().config()) && !cctx.kernalContext().clientNode())
                     cctx.kernalContext().state().onBaselineTopologyChanged(req.baselineTopology(),
                         req.prevBaselineTopologyHistoryItem());
diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index a4dffff1056aa..1f1f4d05acfc3 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.4.3-SNAPSHOT
+ignite.version=2.4.0-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970

From 1668e070fb16ff816771ce821e25da44702a6345 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh 
Date: Mon, 26 Feb 2018 18:08:04 +0300
Subject: [PATCH 65/65] gg-13475 : Reverted version change.

---
 modules/core/src/main/resources/ignite.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index 1f1f4d05acfc3..a4dffff1056aa 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.4.0-SNAPSHOT
+ignite.version=2.4.3-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970