From d6e7050280817b4705302346e170a1809a3d1a91 Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Wed, 3 May 2017 11:24:01 +0300 Subject: [PATCH 001/155] IGNITE-5134: Fixed ClassCastException in IgniteCacheDatabaseSharedManager. This closes #1895. --- .../IgniteCacheDatabaseSharedManager.java | 21 ++++++++++++------- .../impl/PageMemoryNoLoadSelfTest.java | 3 ++- .../IgniteDbDynamicCacheSelfTest.java | 1 + .../database/MetadataStorageSelfTest.java | 3 ++- .../processors/igfs/IgfsSizeSelfTest.java | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java index 5062d0f6a0ec0..7151b2fb7f6ac 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java @@ -715,7 +715,8 @@ public void ensureFreeSpace(MemoryPolicy memPlc) throws IgniteCheckedException { * @param memMetrics {@link MemoryMetrics} object to collect memory usage metrics. * @return Memory policy instance. */ - private MemoryPolicy initMemory(MemoryConfiguration memCfg, MemoryPolicyConfiguration plcCfg, MemoryMetricsImpl memMetrics) { + private MemoryPolicy initMemory(MemoryConfiguration memCfg, MemoryPolicyConfiguration plcCfg, + MemoryMetricsImpl memMetrics) { File allocPath = buildAllocPath(plcCfg); DirectMemoryProvider memProvider = allocPath == null ? @@ -726,23 +727,29 @@ private MemoryPolicy initMemory(MemoryConfiguration memCfg, MemoryPolicyConfigur PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics); - return new MemoryPolicy(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, - (PageMemoryNoStoreImpl)pageMem)); + return new MemoryPolicy(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem)); } /** * @param plc Memory Policy Configuration. * @param pageMem Page memory. */ - private PageEvictionTracker createPageEvictionTracker(MemoryPolicyConfiguration plc, PageMemoryNoStoreImpl pageMem) { + private PageEvictionTracker createPageEvictionTracker(MemoryPolicyConfiguration plc, PageMemory pageMem) { + if (plc.getPageEvictionMode() == DataPageEvictionMode.DISABLED) + return new NoOpPageEvictionTracker(); + + assert pageMem instanceof PageMemoryNoStoreImpl : pageMem.getClass(); + + PageMemoryNoStoreImpl pageMem0 = (PageMemoryNoStoreImpl)pageMem; + if (Boolean.getBoolean("override.fair.fifo.page.eviction.tracker")) - return new FairFifoPageEvictionTracker(pageMem, plc, cctx); + return new FairFifoPageEvictionTracker(pageMem0, plc, cctx); switch (plc.getPageEvictionMode()) { case RANDOM_LRU: - return new RandomLruPageEvictionTracker(pageMem, plc, cctx); + return new RandomLruPageEvictionTracker(pageMem0, plc, cctx); case RANDOM_2_LRU: - return new Random2LruPageEvictionTracker(pageMem, plc, cctx); + return new Random2LruPageEvictionTracker(pageMem0, plc, cctx); default: return new NoOpPageEvictionTracker(); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java index 84db5653cee79..0a283edde8603 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java @@ -280,7 +280,8 @@ public void testPageIdRotation() throws Exception { protected PageMemory memory() throws Exception { File memDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false); - MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration().setMaxSize(10 * 1024 * 1024); + MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration() + .setInitialSize(10 * 1024 * 1024).setMaxSize(10 * 1024 * 1024); DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), memDir); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbDynamicCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbDynamicCacheSelfTest.java index 3b3e1defa5f44..8655ba9aa8bf2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbDynamicCacheSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbDynamicCacheSelfTest.java @@ -44,6 +44,7 @@ public class IgniteDbDynamicCacheSelfTest extends GridCommonAbstractTest { MemoryPolicyConfiguration plc = new MemoryPolicyConfiguration(); plc.setName("dfltPlc"); + plc.setInitialSize(200 * 1024 * 1024); plc.setMaxSize(200 * 1024 * 1024); dbCfg.setDefaultMemoryPolicyName("dfltPlc"); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java index af0b849b625c7..b98f4294e810a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java @@ -157,7 +157,8 @@ private static String randomName() { protected PageMemory memory(boolean clean) throws Exception { DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath); - MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration().setMaxSize(30 * 1024 * 1024); + MemoryPolicyConfiguration plcCfg = new MemoryPolicyConfiguration() + .setMaxSize(30 * 1024 * 1024).setInitialSize(30 * 1024 * 1024); return new PageMemoryNoStoreImpl( log, diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java index 456971a64e6e5..597efe13cfe98 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java @@ -397,7 +397,7 @@ private void checkOversize() throws Exception { String memPlcName = "igfsDataMemPlc"; cfg.setMemoryConfiguration(new MemoryConfiguration().setMemoryPolicies( - new MemoryPolicyConfiguration().setMaxSize(maxSize).setName(memPlcName))); + new MemoryPolicyConfiguration().setMaxSize(maxSize).setInitialSize(maxSize).setName(memPlcName))); FileSystemConfiguration igfsCfg = cfg.getFileSystemConfiguration()[0]; From ff08c8e09033a1a703aae602c36a21565fbae606 Mon Sep 17 00:00:00 2001 From: Anton Vinogradov Date: Wed, 3 May 2017 12:49:18 +0300 Subject: [PATCH 002/155] Hibernate module deploy fix --- modules/hibernate-core/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/hibernate-core/pom.xml b/modules/hibernate-core/pom.xml index 9131ad3dd154d..91ec68bfaf207 100644 --- a/modules/hibernate-core/pom.xml +++ b/modules/hibernate-core/pom.xml @@ -71,14 +71,6 @@ org.apache.felix maven-bundle-plugin - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - - true - - From bc4735f7f5e3fe97aa5f30ef8bdf88e0749d6e2e Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Wed, 3 May 2017 18:12:49 +0300 Subject: [PATCH 003/155] IGNITE-5132: Fixed PutGet example. (cherry picked from commit ed72663ff8e6b6f46d77c91471400ec1c9ff0dfa) --- .../os/win/include/ignite/common/concurrent_os.h | 2 +- .../examples/putget-example/config/example-cache.xml | 10 +--------- .../putget-example/project/vs/putget-example.vcxproj | 3 +++ .../project/vs/putget-example.vcxproj.filters | 8 ++++++++ .../cpp/examples/putget-example/src/putget_example.cpp | 2 +- .../query-example/project/vs/query-example.vcxproj | 3 +++ .../project/vs/query-example.vcxproj.filters | 8 ++++++++ 7 files changed, 25 insertions(+), 11 deletions(-) diff --git a/modules/platforms/cpp/common/os/win/include/ignite/common/concurrent_os.h b/modules/platforms/cpp/common/os/win/include/ignite/common/concurrent_os.h index 77de4d8262144..57e732f93da37 100644 --- a/modules/platforms/cpp/common/os/win/include/ignite/common/concurrent_os.h +++ b/modules/platforms/cpp/common/os/win/include/ignite/common/concurrent_os.h @@ -76,7 +76,7 @@ namespace ignite * Special latch with count = 1. */ class IGNITE_IMPORT_EXPORT SingleLatch - { + { public: /** * Constructor. diff --git a/modules/platforms/cpp/examples/putget-example/config/example-cache.xml b/modules/platforms/cpp/examples/putget-example/config/example-cache.xml index 28b726c7b8300..af523cd335532 100644 --- a/modules/platforms/cpp/examples/putget-example/config/example-cache.xml +++ b/modules/platforms/cpp/examples/putget-example/config/example-cache.xml @@ -35,18 +35,10 @@ Partitioned cache example configuration with binary objects enabled. --> + - - - - - - - diff --git a/modules/platforms/cpp/examples/putget-example/project/vs/putget-example.vcxproj b/modules/platforms/cpp/examples/putget-example/project/vs/putget-example.vcxproj index 555e15ae42bb1..8842f3ab9ec3f 100644 --- a/modules/platforms/cpp/examples/putget-example/project/vs/putget-example.vcxproj +++ b/modules/platforms/cpp/examples/putget-example/project/vs/putget-example.vcxproj @@ -101,6 +101,9 @@ copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.co + + + diff --git a/modules/platforms/cpp/examples/putget-example/project/vs/putget-example.vcxproj.filters b/modules/platforms/cpp/examples/putget-example/project/vs/putget-example.vcxproj.filters index 1bcaff524594e..3bb8a8f310df8 100644 --- a/modules/platforms/cpp/examples/putget-example/project/vs/putget-example.vcxproj.filters +++ b/modules/platforms/cpp/examples/putget-example/project/vs/putget-example.vcxproj.filters @@ -13,6 +13,9 @@ {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + {487c5422-915c-4851-892d-c1599ea69e0c} + @@ -27,4 +30,9 @@ Header Files + + + Config + + \ No newline at end of file diff --git a/modules/platforms/cpp/examples/putget-example/src/putget_example.cpp b/modules/platforms/cpp/examples/putget-example/src/putget_example.cpp index 8bf9c8c1feffb..f7bd89414715c 100644 --- a/modules/platforms/cpp/examples/putget-example/src/putget_example.cpp +++ b/modules/platforms/cpp/examples/putget-example/src/putget_example.cpp @@ -100,7 +100,7 @@ int main() std::cout << std::endl; // Get cache instance. - Cache cache = grid.GetCache(NULL); + Cache cache = grid.GetCache("atomic"); // Clear cache. cache.Clear(); diff --git a/modules/platforms/cpp/examples/query-example/project/vs/query-example.vcxproj b/modules/platforms/cpp/examples/query-example/project/vs/query-example.vcxproj index ff8e5be47a9ce..22aa2b9bf4901 100644 --- a/modules/platforms/cpp/examples/query-example/project/vs/query-example.vcxproj +++ b/modules/platforms/cpp/examples/query-example/project/vs/query-example.vcxproj @@ -102,6 +102,9 @@ copy "$(ProjectDir)..\..\..\..\project\vs\$(Platform)\$(Configuration)\ignite.co + + + diff --git a/modules/platforms/cpp/examples/query-example/project/vs/query-example.vcxproj.filters b/modules/platforms/cpp/examples/query-example/project/vs/query-example.vcxproj.filters index 219d3f4dd87fd..506b25599e1c7 100644 --- a/modules/platforms/cpp/examples/query-example/project/vs/query-example.vcxproj.filters +++ b/modules/platforms/cpp/examples/query-example/project/vs/query-example.vcxproj.filters @@ -18,10 +18,18 @@ {b355095f-b4e2-4324-9516-854828c876ff} + + {89a5a9cc-a2c9-4d11-9044-869c3af6a2fd} + Source Files + + + Config + + \ No newline at end of file From a5ae1834621e3c09ed911777ba652ce4e53ea7e1 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 5 May 2017 12:16:36 +0300 Subject: [PATCH 004/155] Fixed JavaDocs in MemoryMetricsImpl. --- .../processors/cache/database/MemoryMetricsImpl.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java index 4a7e951d12238..ee356a1d0fdc0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java @@ -255,12 +255,16 @@ public void decrementLargeEntriesPages() { largeEntriesPages.decrement(); } - /** {@inheritDoc} */ + /** + * Enable metrics. + */ public void enableMetrics() { metricsEnabled = true; } - /** {@inheritDoc} */ + /** + * Disable metrics. + */ public void disableMetrics() { metricsEnabled = false; } From b051163be08b50fc1ead70e8640703376f4c1642 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 5 May 2017 18:06:00 +0300 Subject: [PATCH 005/155] Removed ML from GG PE build for now. --- assembly/release-fabric-base.xml | 1 + examples/pom-standalone-lgpl.xml | 19 ------------------- examples/pom-standalone.xml | 19 ------------------- examples/pom.xml | 30 ------------------------------ 4 files changed, 1 insertion(+), 68 deletions(-) diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml index 5007785bb2a58..7484dfa9505c2 100644 --- a/assembly/release-fabric-base.xml +++ b/assembly/release-fabric-base.xml @@ -239,6 +239,7 @@ **/package.html src/test/** + src/main/ml/** diff --git a/examples/pom-standalone-lgpl.xml b/examples/pom-standalone-lgpl.xml index 4798d0335a728..e7eca1ecf960c 100644 --- a/examples/pom-standalone-lgpl.xml +++ b/examples/pom-standalone-lgpl.xml @@ -33,7 +33,6 @@ src/main/java src/main/java src/main/java - src/main/java 1.7 @@ -105,23 +104,6 @@ - - ml - - - src/main/ml - 1.8 - - - - - org.apache.ignite - ignite-ml - to_be_replaced_by_ignite_version - - - - scala @@ -248,7 +230,6 @@ ${lgpl.folder} ${java8.folder} ${spark.folder} - ${ml.folder} diff --git a/examples/pom-standalone.xml b/examples/pom-standalone.xml index e74082c2f4a5c..65b5402491b0e 100644 --- a/examples/pom-standalone.xml +++ b/examples/pom-standalone.xml @@ -33,7 +33,6 @@ src/main/java src/main/java src/main/java - src/main/java 1.7 @@ -105,23 +104,6 @@ - - ml - - - src/main/ml - 1.8 - - - - - org.apache.ignite - ignite-ml - to_be_replaced_by_ignite_version - - - - scala @@ -249,7 +231,6 @@ ${lgpl.folder} ${java8.folder} ${spark.folder} - ${ml.folder} diff --git a/examples/pom.xml b/examples/pom.xml index 26f653d88e941..6e95fedd0f46a 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -101,7 +101,6 @@ src/main/java src/main/java src/main/java - src/main/java src/test/java src/test/java src/test/java @@ -235,34 +234,6 @@ - - ml - - - src/main/ml - - - - - - maven-compiler-plugin - - 1.8 - 1.8 - - - - - - - - org.apache.ignite - ignite-ml - ${project.version} - - - - lgpl @@ -322,7 +293,6 @@ ${lgpl.folder} ${java8.folder} ${spark.folder} - ${ml.folder} From 0c45750bdcaabf72980f775e0bc2f18edc603607 Mon Sep 17 00:00:00 2001 From: Sergi Vladykin Date: Thu, 11 May 2017 16:01:38 +0300 Subject: [PATCH 006/155] IGNITE-5190 - ArrayIndexOutOfBoundsException in GridMergeIndexSorted (cherry picked from commit 3a9dba5) --- .../h2/twostep/GridMergeIndexSorted.java | 3 + .../query/IgniteSqlSplitterSelfTest.java | 68 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java index f2d9de4afd64b..54c8dd40e1586 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java @@ -252,6 +252,9 @@ private void goFirst() { * */ private void goNext() { + if (off == streams.length) + return; // All streams are done. + if (streams[off].next()) bubbleUp(streams, off, streamCmp); else diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java index 8e56d36b2f0ec..6391286b7519f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java @@ -1579,6 +1579,37 @@ public void testImplicitJoinConditionGeneration() { } } + /** + * @throws Exception If failed. + */ + public void testJoinWithSubquery() throws Exception { + IgniteCache c1 = ignite(0).createCache( + cacheConfig("Contract", true, + Integer.class, Contract.class)); + + IgniteCache c2 = ignite(0).createCache( + cacheConfig("PromoContract", true, + Integer.class, PromoContract.class)); + + for (int i = 0; i < 100; i++) { + int coId = i % 10; + int cust = i / 10; + c1.put( i, new Contract(coId, cust)); + } + + for (int i = 0; i < 10; i++) + c2.put(i, new PromoContract((i % 5) + 1, i)); + + final List> res = c2.query(new SqlFieldsQuery("SELECT CO.CO_ID \n" + + "FROM PromoContract PMC \n" + + "INNER JOIN \"Contract\".Contract CO ON PMC.CO_ID = 5 \n" + + "AND PMC.CO_ID = CO.CO_ID \n" + + "INNER JOIN (SELECT CO_ID FROM PromoContract EBP WHERE EBP.CO_ID = 5 LIMIT 1) VPMC \n" + + "ON PMC.CO_ID = VPMC.CO_ID ")).getAll(); + + assertFalse(res.isEmpty()); + } + /** @throws Exception if failed. */ public void testDistributedAggregates() throws Exception { final String cacheName = "ints"; @@ -2098,4 +2129,41 @@ private static class OrderGood implements Serializable { @QuerySqlField private int goodId; } + + /** */ + private static class Contract implements Serializable { + /** */ + @QuerySqlField(index = true) + private final int CO_ID; + + /** */ + @QuerySqlField(index = true) + private final int CUSTOMER_ID; + + /** */ + public Contract(final int CO_ID, final int CUSTOMER_ID) { + this.CO_ID = CO_ID; + this.CUSTOMER_ID = CUSTOMER_ID; + } + + } + + /** */ + public class PromoContract implements Serializable { + /** */ + @QuerySqlField(index = true, orderedGroups = { + @QuerySqlField.Group(name = "myIdx", order = 1)}) + private final int CO_ID; + + /** */ + @QuerySqlField(index = true, orderedGroups = { + @QuerySqlField.Group(name = "myIdx", order = 0)}) + private final int OFFER_ID; + + /** */ + public PromoContract(final int co_Id, final int offer_Id) { + this.CO_ID = co_Id; + this.OFFER_ID = offer_Id; + } + } } From 1e1198555358eca9163753249fbef8bd1a3ef702 Mon Sep 17 00:00:00 2001 From: Sergi Vladykin Date: Tue, 9 May 2017 06:11:34 +0300 Subject: [PATCH 007/155] master - fixed SELECT (SELECT COUNT(1)) FROM (cherry picked from commit d10091d) --- .../query/h2/sql/GridSqlQuerySplitter.java | 7 ++++- .../query/IgniteSqlSplitterSelfTest.java | 30 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index b3d54e1bcca82..2bac505ca4c1a 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -1734,13 +1734,18 @@ private static void set(List list, int idx, Z item) { } /** - * @param el Expression. + * @param el Expression part in SELECT clause. * @return {@code true} If expression contains aggregates. */ private static boolean hasAggregates(GridSqlAst el) { if (el instanceof GridSqlAggregateFunction) return true; + // If in SELECT clause we have a subquery expression with aggregate, + // we should not split it. Run the whole subquery on MAP stage. + if (el instanceof GridSqlQuery) + return false; + for (int i = 0; i < el.size(); i++) { if (hasAggregates(el.child(i))) return true; diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java index 6391286b7519f..0fd937c2955ea 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java @@ -37,6 +37,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cache.affinity.AffinityKey; import org.apache.ignite.cache.affinity.AffinityKeyMapped; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.SqlFieldsQuery; @@ -290,6 +291,35 @@ private void doTestPartitionedTablesUsingReplicatedCache(int segments, boolean c } } + /** + */ + public void testSubQueryWithAggregate() { + CacheConfiguration ccfg1 = cacheConfig("pers", true, + AffinityKey.class, Person2.class); + + IgniteCache, Person2> c1 = ignite(0).getOrCreateCache(ccfg1); + + try { + int orgId = 100500; + + c1.put(new AffinityKey<>(1, orgId), new Person2(orgId, "Vasya")); + c1.put(new AffinityKey<>(2, orgId), new Person2(orgId, "Another Vasya")); + + List> rs = c1.query(new SqlFieldsQuery("select name, " + + "(select count(1) from Person2 q where q.orgId = p.orgId) " + + "from Person2 p order by name desc")).getAll(); + + assertEquals(2, rs.size()); + assertEquals("Vasya", rs.get(0).get(0)); + assertEquals(2L, rs.get(0).get(1)); + assertEquals("Another Vasya", rs.get(1).get(0)); + assertEquals(2L, rs.get(1).get(1)); + } + finally { + c1.destroy(); + } + } + /** * @throws InterruptedException If failed. */ From b4d8c341930f8873f03427b6e25d2c9f30514c0b Mon Sep 17 00:00:00 2001 From: Sergi Vladykin Date: Tue, 9 May 2017 18:17:40 +0300 Subject: [PATCH 008/155] master - minor fix for subqueries with aggregates (cherry picked from commit b039d05) --- .../internal/processors/query/h2/sql/GridSqlQuerySplitter.java | 2 +- .../internal/processors/query/IgniteSqlSplitterSelfTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index 2bac505ca4c1a..26c6b08261604 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -1743,7 +1743,7 @@ private static boolean hasAggregates(GridSqlAst el) { // If in SELECT clause we have a subquery expression with aggregate, // we should not split it. Run the whole subquery on MAP stage. - if (el instanceof GridSqlQuery) + if (el instanceof GridSqlSubquery) return false; for (int i = 0; i < el.size(); i++) { diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java index 0fd937c2955ea..f98f41bb350c4 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java @@ -306,7 +306,7 @@ public void testSubQueryWithAggregate() { c1.put(new AffinityKey<>(2, orgId), new Person2(orgId, "Another Vasya")); List> rs = c1.query(new SqlFieldsQuery("select name, " + - "(select count(1) from Person2 q where q.orgId = p.orgId) " + + "select count(1) from Person2 q where q.orgId = p.orgId " + "from Person2 p order by name desc")).getAll(); assertEquals(2, rs.size()); From b7d926e71e7e0af3a5f067891bd8115b21af9318 Mon Sep 17 00:00:00 2001 From: AMRepo Date: Fri, 26 May 2017 12:44:20 +0300 Subject: [PATCH 009/155] Fixed segmented indices snapshots. - Fixes #1936. Signed-off-by: Sergi Vladykin (cherry picked from commit 1554a16) --- .../processors/query/h2/opt/GridH2Table.java | 104 +++++++++++------- .../h2/twostep/GridMapQueryExecutor.java | 2 +- .../IgniteSqlSegmentedIndexSelfTest.java | 25 +++++ 3 files changed, 91 insertions(+), 40 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java index a00ea90b6bb6a..7c34c9af48e1e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java @@ -23,7 +23,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -74,6 +74,9 @@ public class GridH2Table extends TableBase { /** */ private volatile ArrayList idxs; + /** */ + private final int pkIndexPos; + /** */ private final Map tmpIdxs = new HashMap<>(); @@ -87,7 +90,7 @@ public class GridH2Table extends TableBase { private final ConcurrentMap sessions = new ConcurrentHashMap8<>(); /** */ - private final AtomicReference actualSnapshot = new AtomicReference<>(); + private final AtomicReferenceArray actualSnapshot; /** */ private IndexColumn affKeyCol; @@ -155,21 +158,31 @@ public GridH2Table(CreateTableData createTblData, @Nullable GridH2RowDescriptor assert idxs != null; List clones = new ArrayList<>(idxs.size()); - for (Index index: idxs) { + for (Index index : idxs) { Index clone = createDuplicateIndexIfNeeded(index); if (clone != null) - clones.add(clone); + clones.add(clone); } idxs.addAll(clones); + boolean hasHashIndex = idxs.size() >= 2 && index(0).getIndexType().isHash(); + // Add scan index at 0 which is required by H2. - if (idxs.size() >= 2 && index(0).getIndexType().isHash()) + if (hasHashIndex) idxs.add(0, new GridH2PrimaryScanIndex(this, index(1), index(0))); else idxs.add(0, new GridH2PrimaryScanIndex(this, index(0), null)); snapshotEnabled = desc == null || desc.snapshotableIndex(); + pkIndexPos = hasHashIndex ? 2 : 1; + + final int segments = desc != null ? desc.configuration().getQueryParallelism() : + // Get index segments count from PK index. Null desc can be passed from tests. + index(pkIndexPos).segmentsCount(); + + actualSnapshot = snapshotEnabled ? new AtomicReferenceArray(Math.max(segments, 1)) : null; + lock = new ReentrantReadWriteLock(); } @@ -224,8 +237,13 @@ public GridH2RowDescriptor rowDescriptor() { throw new IllegalStateException("Table " + identifier() + " already destroyed."); } - if (snapshotInLock()) - snapshotIndexes(null); + if (snapshotInLock()) { + final GridH2QueryContext qctx = GridH2QueryContext.get(); + + assert qctx != null; + + snapshotIndexes(null, qctx.segment()); + } return false; } @@ -246,21 +264,22 @@ private boolean snapshotInLock() { /** * @param qctx Query context. + * @param segment id of index segment to be snapshoted. */ - public void snapshotIndexes(GridH2QueryContext qctx) { + public void snapshotIndexes(GridH2QueryContext qctx, int segment) { if (!snapshotEnabled) return; - Object[] snapshots; + Object[] segmentSnapshot; // Try to reuse existing snapshots outside of the lock. - for (long waitTime = 200;; waitTime *= 2) { // Increase wait time to avoid starvation. - snapshots = actualSnapshot.get(); + for (long waitTime = 200; ; waitTime *= 2) { // Increase wait time to avoid starvation. + segmentSnapshot = actualSnapshot.get(segment); - if (snapshots != null) { // Reuse existing snapshot without locking. - snapshots = doSnapshotIndexes(snapshots, qctx); + if (segmentSnapshot != null) { // Reuse existing snapshot without locking. + segmentSnapshot = doSnapshotIndexes(segment, segmentSnapshot, qctx); - if (snapshots != null) + if (segmentSnapshot != null) return; // Reused successfully. } @@ -272,17 +291,17 @@ public void snapshotIndexes(GridH2QueryContext qctx) { ensureNotDestroyed(); // Try again inside of the lock. - snapshots = actualSnapshot.get(); + segmentSnapshot = actualSnapshot.get(segment); - if (snapshots != null) // Try reusing. - snapshots = doSnapshotIndexes(snapshots, qctx); + if (segmentSnapshot != null) // Try reusing. + segmentSnapshot = doSnapshotIndexes(segment, segmentSnapshot, qctx); - if (snapshots == null) { // Reuse failed, produce new snapshots. - snapshots = doSnapshotIndexes(null, qctx); + if (segmentSnapshot == null) { // Reuse failed, produce new snapshots. + segmentSnapshot = doSnapshotIndexes(segment,null, qctx); - assert snapshots != null; + assert segmentSnapshot != null; - actualSnapshot.set(snapshots); + actualSnapshot.set(segment, segmentSnapshot); } } finally { @@ -359,19 +378,22 @@ private void ensureNotDestroyed() { * Must be called inside of write lock because when using multiple indexes we have to ensure that all of them have * the same contents at snapshot taking time. * + * @param segment id of index segment snapshot. + * @param segmentSnapshot snapshot to be reused. * @param qctx Query context. * @return New indexes data snapshot. */ @SuppressWarnings("unchecked") - private Object[] doSnapshotIndexes(Object[] snapshots, GridH2QueryContext qctx) { + private Object[] doSnapshotIndexes(int segment, Object[] segmentSnapshot, GridH2QueryContext qctx) { assert snapshotEnabled; - if (snapshots == null) // Nothing to reuse, create new snapshots. - snapshots = new Object[idxs.size() - 2]; + //TODO: make HashIndex snapshotable or remove it at all? + if (segmentSnapshot == null) // Nothing to reuse, create new snapshots. + segmentSnapshot = new Object[idxs.size() - pkIndexPos]; - // Take snapshots on all except first which is scan and second which is hash. - for (int i = 2, len = idxs.size(); i < len; i++) { - Object s = snapshots[i - 2]; + // Take snapshots on all except first which is scan. + for (int i = pkIndexPos, len = idxs.size(); i < len; i++) { + Object s = segmentSnapshot[i - pkIndexPos]; boolean reuseExisting = s != null; @@ -385,20 +407,20 @@ private Object[] doSnapshotIndexes(Object[] snapshots, GridH2QueryContext qctx) if (qctx != null) qctx.clearSnapshots(); - for (int j = 2; j < i; j++) + for (int j = pkIndexPos; j < i; j++) if ((idxs.get(j) instanceof GridH2IndexBase)) index(j).releaseSnapshot(); // Drop invalidated snapshot. - actualSnapshot.compareAndSet(snapshots, null); + actualSnapshot.compareAndSet(segment, segmentSnapshot, null); return null; } - snapshots[i - 2] = s; + segmentSnapshot[i - pkIndexPos] = s; } - return snapshots; + return segmentSnapshot; } /** {@inheritDoc} */ @@ -571,7 +593,7 @@ boolean doUpdate(final GridH2Row row, boolean del) throws IgniteCheckedException int len = idxs.size(); - int i = 2; + int i = pkIndexPos; // Put row if absent to all indexes sequentially. // Start from 3 because 0 - Scan (don't need to update), 1 - PK hash (already updated), 2 - PK (already updated). @@ -593,7 +615,7 @@ boolean doUpdate(final GridH2Row row, boolean del) throws IgniteCheckedException if (old != null) { // Remove row from all indexes. // Start from 3 because 0 - Scan (don't need to update), 1 - PK hash (already updated), 2 - PK (already updated). - for (int i = 3, len = idxs.size(); i < len; i++) { + for (int i = pkIndexPos + 1, len = idxs.size(); i < len; i++) { if (!(idxs.get(i) instanceof GridH2IndexBase)) continue; Row res = index(i).remove(old); @@ -611,7 +633,8 @@ boolean doUpdate(final GridH2Row row, boolean del) throws IgniteCheckedException } // The snapshot is not actual after update. - actualSnapshot.set(null); + if (actualSnapshot != null) + actualSnapshot.set(pk.segmentForRow(row), null); return true; } @@ -668,9 +691,10 @@ private static boolean eq(Index pk, SearchRow r1, SearchRow r2) { ArrayList indexes() { ArrayList res = new ArrayList<>(idxs.size() - 2); - for (int i = 2, len = idxs.size(); i < len; i++) + for (int i = pkIndexPos, len = idxs.size(); i < len; i++) { if (idxs.get(i) instanceof GridH2IndexBase) res.add(index(i)); + } return res; } @@ -679,6 +703,8 @@ ArrayList indexes() { * */ public void markRebuildFromHashInProgress(boolean value) { + assert !value || (idxs.size() >= 2 && index(1).getIndexType().isHash()) : "Table has no hash index."; + rebuildFromHashInProgress = value; } @@ -804,14 +830,14 @@ public void removeIndex(Session session, Index h2Idx) { try { ArrayList idxs = new ArrayList<>(this.idxs); - Index targetIdx = (h2Idx instanceof GridH2ProxyIndex)? - ((GridH2ProxyIndex)h2Idx).underlyingIndex(): h2Idx; + Index targetIdx = (h2Idx instanceof GridH2ProxyIndex) ? + ((GridH2ProxyIndex)h2Idx).underlyingIndex() : h2Idx; - for (int i = 2; i < idxs.size(); ) { + for (int i = pkIndexPos; i < idxs.size(); ) { Index idx = idxs.get(i); if (idx == targetIdx || (idx instanceof GridH2ProxyIndex && - ((GridH2ProxyIndex)idx).underlyingIndex() == targetIdx)) { + ((GridH2ProxyIndex)idx).underlyingIndex() == targetIdx)) { idxs.remove(i); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 45d8f5033c7d2..8e7e9a181ede2 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -572,7 +572,7 @@ private void onQueryRequest0( Objects.requireNonNull(tbl, identifier); - tbl.snapshotIndexes(qctx); + tbl.snapshotIndexes(qctx, segmentId); snapshotedTbls.add(tbl); } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java index 586b81e38ccc0..03c3f1e8a68a7 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java @@ -134,6 +134,31 @@ public void testSegmentedIndex() throws Exception { checkLocalQueryWithSegmentedIndex(); } + /** + * Check correct index snapshots with segmented indices. + * @throws Exception If failed. + */ + public void testSegmentedIndexReproducableResults() throws Exception { + ignite(0).createCache(cacheConfig(ORG_CACHE_NAME, true, Integer.class, Organization.class)); + + IgniteCache cache = ignite(0).cache(ORG_CACHE_NAME); + + // Unequal entries distribution among partitions. + int expectedSize = nodesCount() * QRY_PARALLELISM_LVL * 3 / 2; + + for (int i = 0; i < expectedSize; i++) + cache.put(i, new Organization("org-" + i)); + + String select0 = "select * from \"org\".Organization o"; + + // Check for stable results. + for(int i = 0; i < 10; i++) { + List> result = cache.query(new SqlFieldsQuery(select0)).getAll(); + + assertEquals(expectedSize, result.size()); + } + } + /** * Run tests on single-node grid * From 5658c45549eef86d94ad806e76e48fb997e99a94 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Mon, 15 May 2017 20:24:10 +0300 Subject: [PATCH 010/155] IGNITE-5225: Fix NPE caused by changes in IGNITE-4577. (cherry picked from commit d463840) (cherry picked from commit 95850b4) --- .../ignite/internal/util/IgniteUtils.java | 4 ++-- .../communication/tcp/TcpCommunicationSpi.java | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index 6f8728c000421..f0bad68f727fa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -1858,11 +1858,11 @@ public static synchronized boolean isLocalHostChanged() throws IOException { * @return List of reachable addresses. */ public static List filterReachable(Collection addrs) { - final int reachTimeout = 2000; - if (addrs.isEmpty()) return Collections.emptyList(); + final int reachTimeout = 2000; + if (addrs.size() == 1) { InetAddress addr = F.first(addrs); diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java index be897d635493d..769a02ed0da8c 100755 --- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java @@ -2885,22 +2885,27 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) Set allInetAddrs = U.newHashSet(addrs.size()); - for (InetSocketAddress addr : addrs) - allInetAddrs.add(addr.getAddress()); + for (InetSocketAddress addr : addrs) { + // Skip unresolved as addr.getAddress() can return null. + if(!addr.isUnresolved()) + allInetAddrs.add(addr.getAddress()); + } List reachableInetAddrs = U.filterReachable(allInetAddrs); if (reachableInetAddrs.size() < allInetAddrs.size()) { LinkedHashSet addrs0 = U.newLinkedHashSet(addrs.size()); + List unreachableInetAddr = new ArrayList<>(allInetAddrs.size() - reachableInetAddrs.size()); + for (InetSocketAddress addr : addrs) { if (reachableInetAddrs.contains(addr.getAddress())) addrs0.add(addr); + else + unreachableInetAddr.add(addr); } - for (InetSocketAddress addr : addrs) { - if (!reachableInetAddrs.contains(addr.getAddress())) - addrs0.add(addr); - } + + addrs0.addAll(unreachableInetAddr); addrs = addrs0; } From 51868846b5792561ef345f4b90f41a3026528a1e Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Wed, 24 May 2017 13:19:16 +0300 Subject: [PATCH 011/155] IGNITE-5252: Expose getFieldName method to SqlFieldsQuery result. - Fixes #1982. Signed-off-by: Sergi Vladykin (cherry picked from commit 647fd195b310df10b230b67c92a8df04b5a064e2) (cherry picked from commit 4155d42) --- .../java/org/apache/ignite/IgniteCache.java | 13 +++++++ .../ignite/cache/query/FieldsQueryCursor.java | 39 +++++++++++++++++++ .../processors/cache/IgniteCacheProxy.java | 13 +++++-- .../processors/cache/QueryCursorImpl.java | 19 ++++++++- .../processors/query/GridQueryIndexing.java | 5 ++- .../processors/query/GridQueryProcessor.java | 22 ++++++----- .../multijvm/IgniteCacheProcessProxy.java | 8 ++++ .../processors/query/h2/IgniteH2Indexing.java | 9 +++-- .../query/h2/ddl/DdlStatementsProcessor.java | 4 +- .../cache/SqlFieldsQuerySelfTest.java | 12 ++++-- 10 files changed, 119 insertions(+), 25 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/cache/query/FieldsQueryCursor.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java index 9c8c0904a1289..aeabdb9e3b49b 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java @@ -21,6 +21,7 @@ import java.sql.Timestamp; import java.util.Collection; import java.util.Date; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -41,6 +42,7 @@ import org.apache.ignite.cache.CacheMetrics; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.cache.query.Query; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.QueryDetailMetrics; @@ -346,6 +348,7 @@ public IgniteFuture localLoadCacheAsync(@Nullable IgniteBiPredicate /** * Queries cache. Accepts any subclass of {@link Query} interface. + * See also {@link #query(SqlFieldsQuery)}. * * @param qry Query. * @return Cursor. @@ -354,9 +357,19 @@ public IgniteFuture localLoadCacheAsync(@Nullable IgniteBiPredicate * @see SqlFieldsQuery * @see TextQuery * @see SpiQuery + * */ public QueryCursor query(Query qry); + /** + * Queries cache. Accepts {@link SqlFieldsQuery} class. + * + * @param qry SqlFieldsQuery. + * @return Cursor. + * @see SqlFieldsQuery + */ + public FieldsQueryCursor> query(SqlFieldsQuery qry); + /** * Queries the cache transforming the entries on the server nodes. Can be used, for example, * to avoid network overhead in case only one field out of the large is required by client. diff --git a/modules/core/src/main/java/org/apache/ignite/cache/query/FieldsQueryCursor.java b/modules/core/src/main/java/org/apache/ignite/cache/query/FieldsQueryCursor.java new file mode 100644 index 0000000000000..4219bae961fe2 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/cache/query/FieldsQueryCursor.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.cache.query; + +/** + * SQL query result cursor. This extends {@link QueryCursor} + * to expose fields metadata to public API for SqlFieldsQueries. + */ +public interface FieldsQueryCursor extends QueryCursor { + /** + * Gets field name. + * + * @param idx field index. + * @return Field name. + */ + String getFieldName(int idx); + + /** + * Gets number of columns in a row. + * + * @return row size. + */ + int getColumnsCount(); +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index dfe817e178f7f..9bd90799eddcf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -49,6 +49,7 @@ import org.apache.ignite.cache.CacheMetrics; import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.cache.query.ContinuousQuery; +import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.cache.query.Query; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.QueryDetailMetrics; @@ -561,7 +562,7 @@ private QueryCursor query( if (needToConvert) { Map.Entry entry = (Map.Entry)next; - return (R) new CacheEntryImpl<>(entry.getKey(), entry.getValue()); + return (R)new CacheEntryImpl<>(entry.getKey(), entry.getValue()); } return (R)next; @@ -751,6 +752,12 @@ private QueryCursor> queryContinuous(ContinuousQuery qry, bool } } + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override public FieldsQueryCursor> query(SqlFieldsQuery qry) { + return (FieldsQueryCursor>)query((Query)qry); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public QueryCursor query(Query qry) { @@ -801,9 +808,9 @@ private QueryCursor> queryContinuous(ContinuousQuery qry, bool "Using both partitions and distributed JOINs is not supported for the same query"); if ((p.isReplicatedOnly() && isReplicatedDataNode()) || ctx.isLocal() || qry.isLocal()) - return (QueryCursor)ctx.kernalContext().query().queryLocalFields(ctx, p); + return (FieldsQueryCursor)ctx.kernalContext().query().queryLocalFields(ctx, p); - return (QueryCursor)ctx.kernalContext().query().queryTwoStep(ctx, p); + return (FieldsQueryCursor)ctx.kernalContext().query().queryTwoStep(ctx, p); } if (qry instanceof ScanQuery) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/QueryCursorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/QueryCursorImpl.java index 24789fc0cc16b..4fc1054cd69e0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/QueryCursorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/QueryCursorImpl.java @@ -25,6 +25,7 @@ import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; +import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.cache.query.QueryCancelledException; import org.apache.ignite.internal.processors.cache.query.QueryCursorEx; import org.apache.ignite.internal.processors.query.GridQueryCancel; @@ -38,7 +39,7 @@ /** * Query cursor implementation. */ -public class QueryCursorImpl implements QueryCursorEx { +public class QueryCursorImpl implements QueryCursorEx, FieldsQueryCursor { /** */ private final static AtomicReferenceFieldUpdater STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(QueryCursorImpl.class, State.class, "state"); @@ -188,6 +189,22 @@ public void fieldsMeta(List fieldsMeta) { return fieldsMeta; } + /** {@inheritDoc} */ + @Override public String getFieldName(int idx) { + assert this.fieldsMeta != null; + + GridQueryFieldMetadata metadata = fieldsMeta.get(idx); + + return metadata.fieldName(); + } + + /** {@inheritDoc} */ + @Override public int getColumnsCount() { + assert this.fieldsMeta != null; + + return fieldsMeta.size(); + } + /** Query cursor state */ protected enum State { /** Idle. */IDLE, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java index 0afba59b82441..9eff19976b01d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java @@ -24,6 +24,7 @@ import javax.cache.Cache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.SqlQuery; @@ -71,7 +72,7 @@ public interface GridQueryIndexing { * @return Cursor. * @throws IgniteCheckedException If failed. */ - public QueryCursor> queryTwoStep(GridCacheContext cctx, SqlFieldsQuery qry, GridQueryCancel cancel) + public FieldsQueryCursor> queryTwoStep(GridCacheContext cctx, SqlFieldsQuery qry, GridQueryCancel cancel) throws IgniteCheckedException; /** @@ -94,7 +95,7 @@ public QueryCursor> queryTwoStep(GridCacheContext cc * @param cancel Query cancel. * @return Cursor. */ - public QueryCursor> queryLocalSqlFields(GridCacheContext cctx, SqlFieldsQuery qry, + public FieldsQueryCursor> queryLocalSqlFields(GridCacheContext cctx, SqlFieldsQuery qry, IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException; /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index 448639b4d0483..0399fc4f25f76 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -24,6 +24,7 @@ import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.QueryIndex; +import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.SqlQuery; @@ -1647,18 +1648,19 @@ private void checkxEnabled() throws IgniteException { * @param qry Query. * @return Cursor. */ - public QueryCursor> queryTwoStep(final GridCacheContext cctx, final SqlFieldsQuery qry) { + public FieldsQueryCursor> queryTwoStep(final GridCacheContext cctx, final SqlFieldsQuery qry) { checkxEnabled(); if (!busyLock.enterBusy()) throw new IllegalStateException("Failed to execute query (grid is stopping)."); try { - return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), cctx, new IgniteOutClosureX>>() { - @Override public QueryCursor> applyx() throws IgniteCheckedException { - return idx.queryTwoStep(cctx, qry, null); - } - }, true); + return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), cctx, + new IgniteOutClosureX>>() { + @Override public FieldsQueryCursor> applyx() throws IgniteCheckedException { + return idx.queryTwoStep(cctx, qry, null); + } + }, true); } catch (IgniteCheckedException e) { throw new IgniteException(e); @@ -1928,16 +1930,16 @@ public String space(String schema) throws SQLException { * @return Iterator. */ @SuppressWarnings("unchecked") - public QueryCursor> queryLocalFields(final GridCacheContext cctx, final SqlFieldsQuery qry) { + public FieldsQueryCursor> queryLocalFields(final GridCacheContext cctx, final SqlFieldsQuery qry) { if (!busyLock.enterBusy()) throw new IllegalStateException("Failed to execute query (grid is stopping)."); try { - return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), cctx, new IgniteOutClosureX>>() { - @Override public QueryCursor> applyx() throws IgniteCheckedException { + return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), cctx, new IgniteOutClosureX>>() { + @Override public FieldsQueryCursor> applyx() throws IgniteCheckedException { GridQueryCancel cancel = new GridQueryCancel(); - final QueryCursor> cursor = idx.queryLocalSqlFields(cctx, qry, + final FieldsQueryCursor> cursor = idx.queryLocalSqlFields(cctx, qry, idx.backupFilter(requestTopVer.get(), qry.getPartitions()), cancel); return new QueryCursorImpl>(new Iterable>() { diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java index d2037941f16ad..630e44158042c 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.locks.Lock; @@ -38,10 +39,12 @@ import org.apache.ignite.cache.CacheEntryProcessor; import org.apache.ignite.cache.CacheMetrics; import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.cache.query.Query; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.QueryDetailMetrics; import org.apache.ignite.cache.query.QueryMetrics; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterGroup; import org.apache.ignite.internal.processors.cache.CacheEntryImpl; import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; @@ -190,6 +193,11 @@ private IgniteCacheProcessProxy(String name, boolean async, ExpiryPolicy plc, Ig throw new UnsupportedOperationException("Method should be supported."); } + /** {@inheritDoc} */ + @Override public FieldsQueryCursor> query(SqlFieldsQuery qry) { + throw new UnsupportedOperationException("Method should be supported."); + } + /** {@inheritDoc} */ @Override public QueryCursor query(Query qry, IgniteClosure transformer) { throw new UnsupportedOperationException("Method should be supported."); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index f7466a8390d13..29a91e5d3fb35 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -60,6 +60,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cache.QueryIndexType; +import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.cache.query.QueryCancelledException; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.SqlFieldsQuery; @@ -1335,7 +1336,7 @@ public static void setupConnection(Connection conn, boolean distributedJoins, bo } /** {@inheritDoc} */ - @Override public QueryCursor> queryLocalSqlFields(final GridCacheContext cctx, + @Override public FieldsQueryCursor> queryLocalSqlFields(final GridCacheContext cctx, final SqlFieldsQuery qry, final IndexingQueryFilter filter, final GridQueryCancel cancel) throws IgniteCheckedException { @@ -1527,7 +1528,7 @@ private Iterable> runQueryTwoStep( if (qry.getTimeout() > 0) fqry.setTimeout(qry.getTimeout(), TimeUnit.MILLISECONDS); - final QueryCursor> res = queryTwoStep(cctx, fqry, null); + final FieldsQueryCursor> res = queryTwoStep(cctx, fqry, null); final Iterable> converted = new Iterable>() { @Override public Iterator> iterator() { @@ -1568,7 +1569,7 @@ public static Session session(Connection c) { } /** {@inheritDoc} */ - @Override public QueryCursor> queryTwoStep(GridCacheContext cctx, SqlFieldsQuery qry, + @Override public FieldsQueryCursor> queryTwoStep(GridCacheContext cctx, SqlFieldsQuery qry, GridQueryCancel cancel) { final String space = cctx.name(); final String sqlQry = qry.getSql(); @@ -1737,7 +1738,7 @@ public static Session session(Connection c) { cancel = new GridQueryCancel(); QueryCursorImpl> cursor = new QueryCursorImpl<>( - runQueryTwoStep(cctx, twoStepQry, cctx.keepBinary(), enforceJoinOrder, qry.getTimeout(), cancel, + runQueryTwoStep(cctx, twoStepQry, cctx.keepBinary(), enforceJoinOrder, qry.getTimeout(), cancel, qry.getArgs(), qry.getPartitions()), cancel); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java index 949ea6afe1561..8d8f52730ac7c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java @@ -24,7 +24,7 @@ import java.util.Map; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.QueryIndex; -import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; @@ -75,7 +75,7 @@ public void start(final GridKernalContext ctx, IgniteH2Indexing idx) { * @param stmt H2 statement to parse and execute. */ @SuppressWarnings("unchecked") - public QueryCursor> runDdlStatement(String sql, PreparedStatement stmt) + public FieldsQueryCursor> runDdlStatement(String sql, PreparedStatement stmt) throws IgniteCheckedException { assert stmt instanceof JdbcPreparedStatement; diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlFieldsQuerySelfTest.java index a23f254c2c9af..8860b2ba495e9 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlFieldsQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/SqlFieldsQuerySelfTest.java @@ -21,7 +21,7 @@ import java.util.List; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.FieldsQueryCursor; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.configuration.CacheConfiguration; @@ -66,11 +66,17 @@ public void testSqlFieldsQueryWithTopologyChanges() throws Exception { private void executeQuery() { IgniteCache cache = grid(1).cache("person"); - SqlFieldsQuery qry = new SqlFieldsQuery("select name, age from person where age > 10"); + SqlFieldsQuery qry = new SqlFieldsQuery("select name as \"Full Name\", age from person where age > 10"); - QueryCursor> qryCursor = cache.query(qry); + FieldsQueryCursor> qryCursor = cache.query(qry); assertEquals(2, qryCursor.getAll().size()); + + assertEquals(2, qryCursor.getColumnsCount()); // Row contains "name" and "age" fields. + + assertEquals("Full Name", qryCursor.getFieldName(0)); + + assertEquals("AGE", qryCursor.getFieldName(1)); } From 318f2816f84c3f1b69001ee157aabe83f2c290fb Mon Sep 17 00:00:00 2001 From: AMRepo Date: Fri, 26 May 2017 20:08:43 +0300 Subject: [PATCH 012/155] IGNITE-11423: all changes. --- assembly/dependencies-fabric-lgpl.xml | 2 + assembly/dependencies-fabric.xml | 2 + assembly/release-fabric-lgpl.xml | 1 + assembly/release-fabric.xml | 1 + assembly/release-yardstick.xml | 98 +++++++ .../Apache.Ignite.Core.nuspec | 5 +- modules/platforms/dotnet/build.ps1 | 5 +- modules/web-console/web-agent/pom.xml | 8 + parent/pom.xml | 8 +- pom.xml | 277 +++++++----------- 10 files changed, 218 insertions(+), 189 deletions(-) create mode 100644 assembly/release-yardstick.xml diff --git a/assembly/dependencies-fabric-lgpl.xml b/assembly/dependencies-fabric-lgpl.xml index 4cff146459817..e2fac3ccce3b8 100644 --- a/assembly/dependencies-fabric-lgpl.xml +++ b/assembly/dependencies-fabric-lgpl.xml @@ -131,6 +131,8 @@ org.apache.ignite:ignite-websphere-test org.apache.ignite:ignite-cassandra org.apache.ignite:ignite-yardstick + org.apache.ignite:ignite-benchmarks + org.apache.ignite:ignite-web-agent true diff --git a/assembly/dependencies-fabric.xml b/assembly/dependencies-fabric.xml index be68d0aeb0994..6c4101ecdb3dd 100644 --- a/assembly/dependencies-fabric.xml +++ b/assembly/dependencies-fabric.xml @@ -136,6 +136,8 @@ org.apache.ignite:ignite-websphere-test org.apache.ignite:ignite-cassandra org.apache.ignite:ignite-yardstick + org.apache.ignite:ignite-benchmarks + org.apache.ignite:ignite-web-agent true diff --git a/assembly/release-fabric-lgpl.xml b/assembly/release-fabric-lgpl.xml index 176655874352c..ff4d8c46952aa 100644 --- a/assembly/release-fabric-lgpl.xml +++ b/assembly/release-fabric-lgpl.xml @@ -32,6 +32,7 @@ release-base.xml release-fabric-base.xml + release-yardstick.xml diff --git a/assembly/release-fabric.xml b/assembly/release-fabric.xml index ffde8ecb70baf..7536d4ec5118e 100644 --- a/assembly/release-fabric.xml +++ b/assembly/release-fabric.xml @@ -32,6 +32,7 @@ release-base.xml release-fabric-base.xml + release-yardstick.xml diff --git a/assembly/release-yardstick.xml b/assembly/release-yardstick.xml new file mode 100644 index 0000000000000..0bd6f412cb560 --- /dev/null +++ b/assembly/release-yardstick.xml @@ -0,0 +1,98 @@ + + + + + + + + modules/yardstick/pom-standalone.xml + benchmarks/sources + pom.xml + + + + modules/yardstick/DEVNOTES-standalone.txt + benchmarks/sources + DEVNOTES.txt + + + + modules/yardstick/README.txt + benchmarks + README.txt + + + + + + modules/yardstick/target/assembly/bin + benchmarks/bin + + + + modules/yardstick/target/assembly/config + benchmarks/config + + benchmark.properties + benchmark-remote.properties + benchmark-sample.properties + benchmark-remote-sample.properties + benchmark-multicast.properties + ignite-base-config.xml + ignite-localhost-config.xml + ignite-remote-config.xml + ignite-multicast-config.xml + + + + + modules/yardstick/target/assembly/libs + benchmarks/libs + + junit-*.jar + ignite-apache-license-gen-*.jar + hamcrest-core-*.jar + tools-*.jar + + + + + modules/yardstick/src + benchmarks/sources/src + + + + modules/yardstick/target/assembly/config + benchmarks/sources/config + + benchmark.properties + benchmark-remote.properties + benchmark-sample.properties + benchmark-remote-sample.properties + benchmark-multicast.properties + ignite-base-config.xml + ignite-localhost-config.xml + ignite-remote-config.xml + ignite-multicast-config.xml + + + + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec index 8f562f109ced6..09236ba5a08d2 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.nuspec @@ -61,10 +61,7 @@ More info: https://apacheignite-net.readme.io/ Library files (jars) should not be included in project, so that NuGet package restore works properly. We keep jars in NuGet dir and copy them over in PostBuild event. --> - - - - + diff --git a/modules/platforms/dotnet/build.ps1 b/modules/platforms/dotnet/build.ps1 index b8e6a375638fb..222076a7cca78 100644 --- a/modules/platforms/dotnet/build.ps1 +++ b/modules/platforms/dotnet/build.ps1 @@ -112,10 +112,7 @@ if (!$skipJava) { $libsDir = "$PSScriptRoot\bin\Libs" mkdir -Force $libsDir; del -Force $libsDir\*.* - copy -Force target\release-package\libs\*.jar $libsDir - copy -Force target\release-package\libs\ignite-spring\*.jar $libsDir - copy -Force target\release-package\libs\ignite-indexing\*.jar $libsDir - copy -Force target\release-package\libs\licenses\*.jar $libsDir + ls modules\indexing\target,modules\core\target,modules\spring\target*.jar -recurse -include "ignite-core*","ignite-indexing*","ignite-shmem*","ignite-spring*","lucene*","h2*","cache-api*","commons-*","spring*" -exclude "*-sources*","*-javadoc*","*-tests*" | % { copy -Force $_ $libsDir } # Restore directory cd $PSScriptRoot diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml index 49e0ef0e841eb..168d150570d5e 100644 --- a/modules/web-console/web-agent/pom.xml +++ b/modules/web-console/web-agent/pom.xml @@ -181,6 +181,14 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + + true + + + org.apache.maven.plugins maven-deploy-plugin diff --git a/parent/pom.xml b/parent/pom.xml index 3b578952c4f12..110411b68e1d1 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -997,12 +997,8 @@ - release - - - !skipDefault - - + licenses + org.apache.ignite diff --git a/pom.xml b/pom.xml index 8d388a547a13d..eeab6b28b1353 100644 --- a/pom.xml +++ b/pom.xml @@ -163,11 +163,76 @@ + + javadoc + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + core-javadoc + + aggregate + + validate + + ${basedir}/target/javadoc + core + ${basedir}/assembly/docfiles/javadoc.css + org.apache.ignite -exclude org.apache.ignite.client:org.apache.ignite.codegen:org.apache.ignite.examples:org.apache.ignite.internal:org.apache.ignite.schema:org.apache.ignite.tests:org.apache.ignite.tools:org.apache.ignite.util:org.apache.ignite.spi.discovery.tcp.messages:org.apache.ignite.spi.discovery.tcp.internal:org.apache.ignite.spi.deployment.uri.scanners:org.apache.ignite.spi.deployment.uri.tasks:org.apache.ignite.yardstick:org.apache.ignite.webtest + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + false + + + org.apache.ignite + ignite-tools + ${project.version} + + + + + javadoc-postprocessing-new + + run + + initialize + + + + + + + + + + + + + + + + + + + + + + + + + + dev-libs - - true - @@ -208,11 +273,6 @@ release - - - !skipDefault - - - + - + - + @@ -418,8 +436,8 @@ false - dependencies - prepare-package + ignite-dependencies + validate single @@ -427,15 +445,15 @@ ${basedir}/assembly/dependencies-${ignite.edition}.xml - ${basedir}/target/release-package + ${basedir}/target/release-package-${ignite.edition} libs false - release - prepare-package + ignite-release + validate single @@ -443,14 +461,14 @@ assembly/release-${ignite.edition}.xml - release-package + release-package-${ignite.edition} false dependencies-visor-console - prepare-package + validate single @@ -458,7 +476,7 @@ assembly/dependencies-visor-console.xml - target/release-package/bin + target/release-package-${ignite.edition}/bin include false @@ -466,7 +484,7 @@ scala-scripts - prepare-package + validate single @@ -474,7 +492,7 @@ assembly/release-scala.xml - target/release-package + target/release-package-${ignite.edition} bin false @@ -498,97 +516,6 @@ modules/yardstick - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - false - - - org.apache.ignite - ignite-tools - ${project.version} - - - - - release-yardstick - - run - - prepare-package - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -661,7 +588,7 @@ source-release-assembly-ignite - package + initialize single @@ -684,7 +611,7 @@ attach-artifact - package + initialize run @@ -747,7 +674,7 @@ files - install + initialize @@ -779,7 +706,7 @@ run - install + initialize false From f3757ce315c5b2e4a22aaaa2a0cefed3ce0704f6 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Wed, 14 Jun 2017 19:37:38 +0300 Subject: [PATCH 013/155] IGNITE-5482 MAP query results reuse. --- .../processors/query/h2/IgniteH2Indexing.java | 27 +-- .../h2/twostep/GridMapQueryExecutor.java | 187 +++++++++++++++--- ...gniteCacheAbstractFieldsQuerySelfTest.java | 41 ++++ 3 files changed, 215 insertions(+), 40 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 29a91e5d3fb35..bdb723d25a448 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -79,10 +79,10 @@ import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager; import org.apache.ignite.internal.processors.cache.GridCacheContext; -import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; -import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; import org.apache.ignite.internal.processors.cache.database.CacheDataRow; import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO; @@ -102,8 +102,6 @@ import org.apache.ignite.internal.processors.query.GridRunningQueryInfo; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryIndexDescriptorImpl; -import org.apache.ignite.internal.processors.query.h2.ddl.DdlStatementsProcessor; -import org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode; import org.apache.ignite.internal.processors.query.h2.database.H2PkHashIndex; import org.apache.ignite.internal.processors.query.h2.database.H2RowFactory; import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex; @@ -111,16 +109,17 @@ import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasLeafIO; import org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO; import org.apache.ignite.internal.processors.query.h2.database.io.H2LeafIO; +import org.apache.ignite.internal.processors.query.h2.ddl.DdlStatementsProcessor; +import org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode; import org.apache.ignite.internal.processors.query.h2.opt.GridH2DefaultTableEngine; import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase; -import org.apache.ignite.internal.processors.query.h2.opt.GridH2SystemIndexFactory; import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOffheap; import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap; -import org.apache.ignite.internal.processors.query.h2.opt.GridH2ProxyIndex; import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowFactory; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2SystemIndexFactory; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.opt.GridH2ValueCacheObject; import org.apache.ignite.internal.processors.query.h2.opt.GridLuceneIndex; @@ -167,7 +166,6 @@ import org.h2.engine.SysProperties; import org.h2.index.Cursor; import org.h2.index.Index; -import org.h2.index.SpatialIndex; import org.h2.jdbc.JdbcConnection; import org.h2.jdbc.JdbcPreparedStatement; import org.h2.jdbc.JdbcStatement; @@ -473,7 +471,7 @@ private PreparedStatement prepareStatement(Connection c, String sql, boolean use return stmt; } - stmt = c.prepareStatement(sql); + stmt = c.prepareStatement(sql, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); cache.put(sql, stmt); @@ -660,17 +658,8 @@ private void onSqlException() { if (tbl.luceneIdx != null) tbl.luceneIdx.store(k, v, ver, expirationTime); - } - - /** - * @param o Object. - * @return {@code true} If it is a binary object. - */ - private boolean isBinary(CacheObject o) { - if (ctx == null) - return false; - return ctx.cacheObjects().isBinaryObject(o); + mapQryExec.clearRunningQueriesState(); } /** @@ -713,6 +702,8 @@ private GridCacheContext cacheContext(String space) { if (tbl.tbl.update(key, partId, val, ver, 0, true, 0)) { if (tbl.luceneIdx != null) tbl.luceneIdx.remove(key); + + mapQryExec.clearRunningQueriesState(); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 8e7e9a181ede2..917de616919dd 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -20,8 +20,10 @@ import java.lang.reflect.Field; import java.sql.Connection; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.AbstractCollection; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -30,7 +32,9 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceArray; import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; @@ -57,6 +61,7 @@ import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; import org.apache.ignite.internal.processors.query.GridQueryCancel; +import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode; import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext; @@ -70,6 +75,7 @@ import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; import org.apache.ignite.internal.util.GridSpinBusyLock; +import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; @@ -79,6 +85,7 @@ import org.h2.jdbc.JdbcResultSet; import org.h2.result.ResultInterface; import org.h2.value.Value; +import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; @@ -134,6 +141,10 @@ public class GridMapQueryExecutor { private final ConcurrentMap, GridReservable> reservations = new ConcurrentHashMap8<>(); + /** Futures for query results reuse. */ + private final AtomicReference>> futs = + new AtomicReference<>(); + /** * @param busyLock Busy lock. */ @@ -219,6 +230,13 @@ else if (msg instanceof GridQueryCancelRequest) } } + /** + * + */ + public void clearRunningQueriesState() { + futs.set(null); + } + /** * @param node Node. * @param msg Message. @@ -607,28 +625,10 @@ private void onQueryRequest0( // If we are not the target node for this replicated query, just ignore it. if (qry.node() == null || (segmentId == 0 && qry.node().equals(ctx.localNodeId()))) { - rs = h2.executeSqlQueryWithTimer(mainCctx.name(), conn, qry.query(), - F.asList(qry.parameters(params)), true, - timeout, - qr.cancels[qryIdx]); - - if (evt) { - ctx.event().record(new CacheQueryExecutedEvent<>( - node, - "SQL query executed.", - EVT_CACHE_QUERY_EXECUTED, - CacheQueryType.SQL.name(), - mainCctx.namex(), - null, - qry.query(), - null, - null, - params, - node.id(), - null)); - } + QueryKey key = new QueryKey(qry.query(), qry.parameters(params), distributedJoinMode, + enforceJoinOrder, parts, pageSize); - assert rs instanceof JdbcResultSet : rs.getClass(); + rs = runQuery(key, node, evt ? mainCctx.name() : null, conn, timeout, qr.cancels[qryIdx], evt); } qr.addResult(qryIdx, qry, node.id(), rs, params); @@ -1102,6 +1102,8 @@ private QueryResult(ResultSet rs, GridCacheContext cctx, UUID qrySrcNodeId throw new IllegalStateException(e); // Must not happen. } + assert res != null; + rowCnt = res.getRowCount(); cols = res.getVisibleColumnCount(); } @@ -1126,6 +1128,13 @@ synchronized boolean fetchNextPage(List rows, int pageSize) { boolean readEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ); + try { + rs.absolute(pageSize * page); + } + catch (SQLException e) { + throw new IgniteSQLException(e); + } + page++; for (int i = 0 ; i < pageSize; i++) { @@ -1209,7 +1218,141 @@ private List row(Value[] row) { closed = true; - U.close(rs, log); + //U.close(rs, log); + } + } + + /** + * Execute query specified by {@code key}. + * @param key Query key. + * @param node Cluster node. + * @param mainCctxName Main cache context name, may be {@code null}. + * @param conn H2 connection. + * @param timeout Query timeout. + * @param cancel Query cancel state holder. + * @param evt {@code true} if this query event is recordable, {@code false} otherwise. + * @return Query result. + * @throws IgniteCheckedException if failed. + */ + private ResultSet runQuery(QueryKey key, ClusterNode node, @Nullable String mainCctxName, Connection conn, + int timeout, GridQueryCancel cancel, boolean evt) throws IgniteCheckedException { + + ConcurrentHashMap> futs = this.futs.get(); + + ConcurrentHashMap> newFuts = null; + + while (futs == null) { + if (newFuts == null) + newFuts = new ConcurrentHashMap<>(); + + if (this.futs.compareAndSet(null, newFuts)) + futs = newFuts; + else + futs = this.futs.get(); + } + + GridFutureAdapter fut = futs.get(key); + + if (fut == null) { + GridFutureAdapter newFut = new GridFutureAdapter<>(); + + fut = futs.putIfAbsent(key, newFut); + + if (fut == null) { + fut = newFut; + + ResultSet rs = h2.executeSqlQueryWithTimer(mainCctxName, conn, key.qry, F.asList(key.params), true, + timeout, cancel); + + if (evt) { + ctx.event().record(new CacheQueryExecutedEvent<>( + node, + "SQL query executed.", + EVT_CACHE_QUERY_EXECUTED, + CacheQueryType.SQL.name(), + mainCctxName, + null, + key.qry, + null, + null, + key.params, + node.id(), + null)); + } + + assert rs instanceof JdbcResultSet : rs.getClass(); + + fut.onDone(rs); + + if (this.futs.get() == futs) + futs.remove(key); + } + } + + return fut.get(); + } + + /** + * Query key. + */ + private final static class QueryKey { + /** Query string. */ + private final String qry; + + /** Query parameters. */ + private final Object[] params; + + /** Distributed join mode. */ + private final DistributedJoinMode joinMode; + + /** Enforce join order flag. */ + private final boolean enforceJoinOrder; + + /** Partitions to run query on. */ + private final int[] parts; + + /** Page size. */ + private final int pageSize; + + /** + * Constructor. + * @param qry Query string. + * @param params Query parameters. + * @param joinMode Distributed join mode. + * @param enforceJoinOrder Enforce join order flag. + * @param parts Partitions to run query on. + * @param pageSize Page size. + */ + private QueryKey(String qry, Object[] params, DistributedJoinMode joinMode, boolean enforceJoinOrder, + int[] parts, int pageSize) { + this.qry = qry; + this.params = params; + this.joinMode = joinMode; + this.enforceJoinOrder = enforceJoinOrder; + this.parts = parts; + this.pageSize = pageSize; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + QueryKey qryKey = (QueryKey) o; + + return enforceJoinOrder == qryKey.enforceJoinOrder && qry.equals(qryKey.qry) && + Arrays.equals(params, qryKey.params) && joinMode == qryKey.joinMode && + Arrays.equals(parts, qryKey.parts) && pageSize == qryKey.pageSize; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = qry.hashCode(); + res = 31 * res + Arrays.hashCode(params); + res = 31 * res + joinMode.hashCode(); + res = 31 * res + (enforceJoinOrder ? 1 : 0); + res = 31 * res + Arrays.hashCode(parts); + return 31 * res + pageSize; } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java index c27747c03a829..d0bf95349994e 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java @@ -29,8 +29,12 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import javax.cache.CacheException; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheRebalanceMode; @@ -41,6 +45,8 @@ import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlIndexMetadata; @@ -51,6 +57,7 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.lang.IgniteCallable; import org.apache.ignite.spi.discovery.DiscoverySpi; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; @@ -875,6 +882,40 @@ public void testPaginationGet() throws Exception { } } + /** + * Test behavior when many similar concurrent queries are run. + */ + public void testConcurrentSimilarQueries() throws Exception { + IgniteCache largeIntCache = jcache("large", Integer.class, Integer.class); + + try (IgniteDataStreamer stream = grid(0).dataStreamer("large")) { + for (int i = 0; i < 1_000; i++) + stream.addData(i, i); + } + + final AtomicBoolean stop = new AtomicBoolean(); + + final SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * from \"large\".Integer WHERE _key > ? and _val < ?") + .setArgs(3000, 950000).setTimeout(20, TimeUnit.SECONDS); + + IgniteInternalFuture fut = multithreadedAsync(new IgniteCallable() { + @Override public Object call() throws Exception { + while (!stop.get()) { + IgniteEx node = grid(gridCount() > 0 ? ThreadLocalRandom.current().nextInt(0, gridCount()) : 0); + + node.cache("large").query(qry).getAll(); + } + return null; + } + }, 8); + + Thread.sleep(500); + + stop.set(true); + + fut.get(); + } + /** @throws Exception If failed. */ public void testEmptyGrid() throws Exception { QueryCursor> qry = personCache From 551a44341c620a7c6573f17679e5b35232dc870d Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 15 Jun 2017 17:08:32 +0300 Subject: [PATCH 014/155] IGNITE-5482 Continued --- .../h2/twostep/GridMapQueryExecutor.java | 255 +++++++++++------- ...gniteCacheAbstractFieldsQuerySelfTest.java | 2 +- 2 files changed, 163 insertions(+), 94 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 917de616919dd..ab9450f792398 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -80,6 +80,7 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.plugin.extensions.communication.Message; import org.h2.jdbc.JdbcResultSet; @@ -142,7 +143,7 @@ public class GridMapQueryExecutor { new ConcurrentHashMap8<>(); /** Futures for query results reuse. */ - private final AtomicReference>> futs = + private final AtomicReference>> futs = new AtomicReference<>(); /** @@ -620,7 +621,7 @@ private void onQueryRequest0( boolean evt = ctx.event().isRecordable(EVT_CACHE_QUERY_EXECUTED); for (GridCacheSqlQuery qry : qrys) { - ResultSet rs = null; + ResultSetWrapper rs = null; // If we are not the target node for this replicated query, just ignore it. if (qry.node() == null || @@ -994,7 +995,7 @@ QueryResult result(int qry) { * @param qrySrcNodeId Query source node. * @param rs Result set. */ - void addResult(int qry, GridCacheSqlQuery q, UUID qrySrcNodeId, ResultSet rs, Object[] params) { + void addResult(int qry, GridCacheSqlQuery q, UUID qrySrcNodeId, ResultSetWrapper rs, Object[] params) { if (!results.compareAndSet(qry, null, new QueryResult(rs, cctx, qrySrcNodeId, q, params))) throw new IllegalStateException(); } @@ -1049,7 +1050,7 @@ private class QueryResult implements AutoCloseable { private final ResultInterface res; /** */ - private final ResultSet rs; + private final ResultSetWrapper rs; /** */ private final GridCacheContext cctx; @@ -1085,7 +1086,7 @@ private class QueryResult implements AutoCloseable { * @param qry Query. * @param params Query params. */ - private QueryResult(ResultSet rs, GridCacheContext cctx, UUID qrySrcNodeId, GridCacheSqlQuery qry, + private QueryResult(ResultSetWrapper rs, GridCacheContext cctx, UUID qrySrcNodeId, GridCacheSqlQuery qry, Object[] params) { this.cctx = cctx; this.qry = qry; @@ -1096,7 +1097,7 @@ private QueryResult(ResultSet rs, GridCacheContext cctx, UUID qrySrcNodeId if (rs != null) { this.rs = rs; try { - res = (ResultInterface)RESULT_FIELD.get(rs); + res = (ResultInterface)RESULT_FIELD.get(rs.resultSet()); } catch (IllegalAccessException e) { throw new IllegalStateException(e); // Must not happen. @@ -1128,74 +1129,76 @@ synchronized boolean fetchNextPage(List rows, int pageSize) { boolean readEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ); - try { - rs.absolute(pageSize * page); - } - catch (SQLException e) { - throw new IgniteSQLException(e); - } + synchronized (rs) { + try { + rs.resultSet().absolute(pageSize * page); + } + catch (SQLException e) { + throw new IgniteSQLException(e); + } - page++; + page++; - for (int i = 0 ; i < pageSize; i++) { - if (!res.next()) - return true; + for (int i = 0; i < pageSize; i++) { + if (!res.next()) + return true; - Value[] row = res.currentRow(); + Value[] row = res.currentRow(); - if (cpNeeded) { - boolean copied = false; + if (cpNeeded) { + boolean copied = false; - for (int j = 0; j < row.length; j++) { - Value val = row[j]; + for (int j = 0; j < row.length; j++) { + Value val = row[j]; - if (val instanceof GridH2ValueCacheObject) { - GridH2ValueCacheObject valCacheObj = (GridH2ValueCacheObject)val; + if (val instanceof GridH2ValueCacheObject) { + GridH2ValueCacheObject valCacheObj = (GridH2ValueCacheObject) val; - GridCacheContext cctx = valCacheObj.getCacheContext(); + GridCacheContext cctx = valCacheObj.getCacheContext(); - if (cctx != null && cctx.needValueCopy()) { - row[j] = new GridH2ValueCacheObject(valCacheObj.getCacheContext(), valCacheObj.getCacheObject()) { - @Override public Object getObject() { - return getObject(true); - } - }; + if (cctx != null && cctx.needValueCopy()) { + row[j] = new GridH2ValueCacheObject(valCacheObj.getCacheContext(), valCacheObj.getCacheObject()) { + @Override public Object getObject() { + return getObject(true); + } + }; - copied = true; + copied = true; + } } } + + if (i == 0 && !copied) + cpNeeded = false; // No copy on read caches, skip next checks. } - if (i == 0 && !copied) - cpNeeded = false; // No copy on read caches, skip next checks. - } + assert row != null; + + if (readEvt) { + cctx.gridEvents().record(new CacheQueryReadEvent<>( + cctx.localNode(), + "SQL fields query result set row read.", + EVT_CACHE_QUERY_OBJECT_READ, + CacheQueryType.SQL.name(), + cctx.namex(), + null, + qry.query(), + null, + null, + params, + qrySrcNodeId, + null, + null, + null, + null, + row(row))); + } - assert row != null; - - if (readEvt) { - cctx.gridEvents().record(new CacheQueryReadEvent<>( - cctx.localNode(), - "SQL fields query result set row read.", - EVT_CACHE_QUERY_OBJECT_READ, - CacheQueryType.SQL.name(), - cctx.namex(), - null, - qry.query(), - null, - null, - params, - qrySrcNodeId, - null, - null, - null, - null, - row(row))); + rows.add(res.currentRow()); } - rows.add(res.currentRow()); + return false; } - - return false; } /** @@ -1218,7 +1221,7 @@ private List row(Value[] row) { closed = true; - //U.close(rs, log); + rs.release(); } } @@ -1234,12 +1237,12 @@ private List row(Value[] row) { * @return Query result. * @throws IgniteCheckedException if failed. */ - private ResultSet runQuery(QueryKey key, ClusterNode node, @Nullable String mainCctxName, Connection conn, + private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, @Nullable String mainCctxName, Connection conn, int timeout, GridQueryCancel cancel, boolean evt) throws IgniteCheckedException { - ConcurrentHashMap> futs = this.futs.get(); + ConcurrentHashMap> futs = this.futs.get(); - ConcurrentHashMap> newFuts = null; + ConcurrentHashMap> newFuts = null; while (futs == null) { if (newFuts == null) @@ -1251,45 +1254,64 @@ private ResultSet runQuery(QueryKey key, ClusterNode node, @Nullable String main futs = this.futs.get(); } - GridFutureAdapter fut = futs.get(key); - - if (fut == null) { - GridFutureAdapter newFut = new GridFutureAdapter<>(); - - fut = futs.putIfAbsent(key, newFut); - - if (fut == null) { - fut = newFut; - - ResultSet rs = h2.executeSqlQueryWithTimer(mainCctxName, conn, key.qry, F.asList(key.params), true, - timeout, cancel); - - if (evt) { - ctx.event().record(new CacheQueryExecutedEvent<>( - node, - "SQL query executed.", - EVT_CACHE_QUERY_EXECUTED, - CacheQueryType.SQL.name(), - mainCctxName, - null, - key.qry, - null, - null, - key.params, - node.id(), - null)); - } + ResultSetWrapper res = null; - assert rs instanceof JdbcResultSet : rs.getClass(); + GridFutureAdapter newFut = null; - fut.onDone(rs); + while (res == null) { + GridFutureAdapter fut = futs.get(key); + + if (fut != null) + res = fut.get(); + + if (res == null || !res.tryTake()) { + res = null; + + if (newFut == null) + newFut = new GridFutureAdapter<>(); + + fut = futs.putIfAbsent(key, newFut); + + if (fut != null) + res = fut.get(); + + if (res == null || !res.tryTake()) { + fut = newFut; + + ResultSet rs = h2.executeSqlQueryWithTimer(mainCctxName, conn, key.qry, F.asList(key.params), true, + timeout, cancel); + + if (evt) { + ctx.event().record(new CacheQueryExecutedEvent<>( + node, + "SQL query executed.", + EVT_CACHE_QUERY_EXECUTED, + CacheQueryType.SQL.name(), + mainCctxName, + null, + key.qry, + null, + null, + key.params, + node.id(), + null)); + } + + assert rs instanceof JdbcResultSet : rs.getClass(); - if (this.futs.get() == futs) futs.remove(key); + + res = new ResultSetWrapper(rs); + + if (!res.tryTake()) + throw new IllegalStateException(); + + fut.onDone(res); + } } } - return fut.get(); + return res; } /** @@ -1356,6 +1378,53 @@ private QueryKey(String qry, Object[] params, DistributedJoinMode joinMode, bool } } + private final class ResultSetWrapper { + private int cnt; + + private final ResultSet rs; + + private volatile boolean closed; + + private ResultSetWrapper(ResultSet rs) { + A.notNull(rs, "rs"); + + this.rs = rs; + } + + public ResultSet resultSet() { + checkState(); + + return rs; + } + + public boolean isClosed() { + return closed; + } + + public synchronized boolean tryTake() { + return ++cnt > 0 && !closed; + } + + public synchronized void release() { + checkState(); + + if (--cnt == 0) { + closed = true; + + log.error("CLOOOOOOSIIIIIIIING"); + + U.close(rs, log); + } + else if (cnt < 0) + throw new IllegalStateException(); + } + + private void checkState() { + if (closed) + throw new IllegalStateException(); + } + } + /** * Fake reservation object for replicated caches. */ diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java index d0bf95349994e..2456ca7138430 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java @@ -909,7 +909,7 @@ public void testConcurrentSimilarQueries() throws Exception { } }, 8); - Thread.sleep(500); + Thread.sleep(30000); stop.set(true); From 0a4dec754604ce409b6ad2a2738cb21e88a2a223 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 15 Jun 2017 21:05:23 +0300 Subject: [PATCH 015/155] IGNITE-5482 Finished --- .../h2/twostep/GridMapQueryExecutor.java | 155 +++++++++++------- ...gniteCacheAbstractFieldsQuerySelfTest.java | 24 ++- 2 files changed, 115 insertions(+), 64 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index ab9450f792398..e613215c06eed 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -20,7 +20,6 @@ import java.lang.reflect.Field; import java.sql.Connection; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.AbstractCollection; import java.util.ArrayList; import java.util.Arrays; @@ -61,7 +60,6 @@ import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; import org.apache.ignite.internal.processors.query.GridQueryCancel; -import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode; import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext; @@ -1086,24 +1084,31 @@ private class QueryResult implements AutoCloseable { * @param qry Query. * @param params Query params. */ + @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") private QueryResult(ResultSetWrapper rs, GridCacheContext cctx, UUID qrySrcNodeId, GridCacheSqlQuery qry, - Object[] params) { + Object[] params) { this.cctx = cctx; this.qry = qry; this.params = params; this.qrySrcNodeId = qrySrcNodeId; this.cpNeeded = cctx.isLocalNode(qrySrcNodeId); + ResultInterface res = null; + if (rs != null) { - this.rs = rs; - try { - res = (ResultInterface)RESULT_FIELD.get(rs.resultSet()); - } - catch (IllegalAccessException e) { - throw new IllegalStateException(e); // Must not happen. + synchronized (rs) { + try { + res = (ResultInterface)RESULT_FIELD.get(rs.resultSet()); + } + catch (IllegalAccessException e) { + throw new IllegalStateException(e); // Must not happen. + } } + } - assert res != null; + if (res != null) { + this.rs = rs; + this.res = res; rowCnt = res.getRowCount(); cols = res.getVisibleColumnCount(); @@ -1130,12 +1135,7 @@ synchronized boolean fetchNextPage(List rows, int pageSize) { boolean readEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ); synchronized (rs) { - try { - rs.resultSet().absolute(pageSize * page); - } - catch (SQLException e) { - throw new IgniteSQLException(e); - } + absolute(res, page * pageSize); page++; @@ -1256,62 +1256,78 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, @Nullable Stri ResultSetWrapper res = null; - GridFutureAdapter newFut = null; + GridFutureAdapter fut = futs.get(key); + + if (fut != null) + res = fut.get(); - while (res == null) { - GridFutureAdapter fut = futs.get(key); + if (res == null || !res.tryTake()) { + res = null; + + GridFutureAdapter newFut = new GridFutureAdapter<>(); + + fut = futs.putIfAbsent(key, newFut); if (fut != null) res = fut.get(); if (res == null || !res.tryTake()) { - res = null; - - if (newFut == null) - newFut = new GridFutureAdapter<>(); - - fut = futs.putIfAbsent(key, newFut); + fut = newFut; + + ResultSet rs = h2.executeSqlQueryWithTimer(mainCctxName, conn, key.qry, F.asList(key.params), true, + timeout, cancel); + + if (evt) { + ctx.event().record(new CacheQueryExecutedEvent<>( + node, + "SQL query executed.", + EVT_CACHE_QUERY_EXECUTED, + CacheQueryType.SQL.name(), + mainCctxName, + null, + key.qry, + null, + null, + key.params, + node.id(), + null)); + } - if (fut != null) - res = fut.get(); + assert rs instanceof JdbcResultSet : rs.getClass(); - if (res == null || !res.tryTake()) { - fut = newFut; + futs.remove(key); - ResultSet rs = h2.executeSqlQueryWithTimer(mainCctxName, conn, key.qry, F.asList(key.params), true, - timeout, cancel); + res = new ResultSetWrapper(rs); - if (evt) { - ctx.event().record(new CacheQueryExecutedEvent<>( - node, - "SQL query executed.", - EVT_CACHE_QUERY_EXECUTED, - CacheQueryType.SQL.name(), - mainCctxName, - null, - key.qry, - null, - null, - key.params, - node.id(), - null)); - } + if (!res.tryTake()) + throw new IllegalStateException(); - assert rs instanceof JdbcResultSet : rs.getClass(); + fut.onDone(res); + } + } - futs.remove(key); + return res; + } - res = new ResultSetWrapper(rs); + /** + * x + * @param res a + * @param pos b + * @return c + */ + private static boolean absolute(ResultInterface res, int pos) { + if (pos < 0) + pos = res.getRowCount() + pos + 1; - if (!res.tryTake()) - throw new IllegalStateException(); + if (--pos < res.getRowId()) + res.reset(); - fut.onDone(res); - } - } + while (res.getRowId() < pos) { + if (!res.next()) + return false; } - return res; + return res.getRowId() >= 0 && !res.isAfterLast(); } /** @@ -1378,47 +1394,68 @@ private QueryKey(String qry, Object[] params, DistributedJoinMode joinMode, bool } } + /** */ private final class ResultSetWrapper { + /** Usages counter. */ private int cnt; + /** Wrapped result set. */ private final ResultSet rs; + /** State flag. */ private volatile boolean closed; + /** + * Constructor. + * @param rs Result set to wrap. + */ private ResultSetWrapper(ResultSet rs) { A.notNull(rs, "rs"); this.rs = rs; } - public ResultSet resultSet() { + /** + * @return Wrapped result set. + */ + ResultSet resultSet() { checkState(); return rs; } + /** + * @return State flag. + */ public boolean isClosed() { return closed; } - public synchronized boolean tryTake() { + /** + * @return {@code true} if {@link ResultSet} wrapped by this object may be used. + */ + synchronized boolean tryTake() { return ++cnt > 0 && !closed; } + /** + * Release the hold on this result set. + */ public synchronized void release() { checkState(); if (--cnt == 0) { closed = true; - log.error("CLOOOOOOSIIIIIIIING"); - U.close(rs, log); } else if (cnt < 0) throw new IllegalStateException(); } + /** + * Check if {@link #closed} is set, throw an exception if so. + */ private void checkState() { if (closed) throw new IllegalStateException(); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java index 2456ca7138430..bceccb94a7070 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java @@ -19,6 +19,7 @@ import java.io.Serializable; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -32,6 +33,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import javax.cache.CacheException; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; @@ -886,30 +888,42 @@ public void testPaginationGet() throws Exception { * Test behavior when many similar concurrent queries are run. */ public void testConcurrentSimilarQueries() throws Exception { - IgniteCache largeIntCache = jcache("large", Integer.class, Integer.class); + jcache("large", Integer.class, Integer.class); try (IgniteDataStreamer stream = grid(0).dataStreamer("large")) { - for (int i = 0; i < 1_000; i++) + for (int i = 0; i < 1_000_000; i++) stream.addData(i, i); } final AtomicBoolean stop = new AtomicBoolean(); - final SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * from \"large\".Integer WHERE _key > ? and _val < ?") + final SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * from \"large\".Integer WHERE _key > ? and _val < ? " + + "order by _key") .setArgs(3000, 950000).setTimeout(20, TimeUnit.SECONDS); + final AtomicInteger cnt = new AtomicInteger(); + IgniteInternalFuture fut = multithreadedAsync(new IgniteCallable() { @Override public Object call() throws Exception { while (!stop.get()) { IgniteEx node = grid(gridCount() > 0 ? ThreadLocalRandom.current().nextInt(0, gridCount()) : 0); - node.cache("large").query(qry).getAll(); + List> res = node.cache("large").query(qry).getAll(); + + assertEquals(946999, res.size()); + + for (int i = 1; i <= res.size(); i++) + assertEquals(Arrays.asList(3000 + i, 3000 + i), res.get(i - 1)); + + cnt.incrementAndGet(); } return null; } }, 8); - Thread.sleep(30000); + Thread.sleep(120000); + + System.out.println("Iterations done: " + cnt); stop.set(true); From 2dcf98cd7641f5f99da5e7dd0bf0e8c95d2e9014 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 15 Jun 2017 21:14:35 +0300 Subject: [PATCH 016/155] Added comments. --- .../query/h2/twostep/GridMapQueryExecutor.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index e613215c06eed..2a2f384862bb9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -1135,6 +1135,7 @@ synchronized boolean fetchNextPage(List rows, int pageSize) { boolean readEvt = cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ); synchronized (rs) { + // We can't use ResultSet#absolute here as it may expire inside H2, and those calls will all fail. absolute(res, page * pageSize); page++; @@ -1310,10 +1311,10 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, @Nullable Stri } /** - * x - * @param res a - * @param pos b - * @return c + * Rewind given {@link ResultInterface} to the desired position. + * @param res Result. + * @param pos Position (JDBC row numbering compatible - 0 means 'before first row'). + * @return Resulting status. */ private static boolean absolute(ResultInterface res, int pos) { if (pos < 0) From 34eed248bc4c48eee8dadc79b10e164721df3a28 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Fri, 16 Jun 2017 13:05:10 +0300 Subject: [PATCH 017/155] IGNITE-5482 Cosmetics and test enhancement. --- .../processors/query/h2/IgniteH2Indexing.java | 2 +- .../h2/twostep/GridMapQueryExecutor.java | 52 ++++++++++--------- ...gniteCacheAbstractFieldsQuerySelfTest.java | 27 ++++++++-- 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index bdb723d25a448..5e8ab9dc68c64 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -471,7 +471,7 @@ private PreparedStatement prepareStatement(Connection c, String sql, boolean use return stmt; } - stmt = c.prepareStatement(sql, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + stmt = c.prepareStatement(sql); cache.put(sql, stmt); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 2a2f384862bb9..1d19b36d752f8 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -625,7 +625,7 @@ private void onQueryRequest0( if (qry.node() == null || (segmentId == 0 && qry.node().equals(ctx.localNodeId()))) { QueryKey key = new QueryKey(qry.query(), qry.parameters(params), distributedJoinMode, - enforceJoinOrder, parts, pageSize); + enforceJoinOrder, parts); rs = runQuery(key, node, evt ? mainCctx.name() : null, conn, timeout, qr.cancels[qryIdx], evt); } @@ -1241,19 +1241,7 @@ private List row(Value[] row) { private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, @Nullable String mainCctxName, Connection conn, int timeout, GridQueryCancel cancel, boolean evt) throws IgniteCheckedException { - ConcurrentHashMap> futs = this.futs.get(); - - ConcurrentHashMap> newFuts = null; - - while (futs == null) { - if (newFuts == null) - newFuts = new ConcurrentHashMap<>(); - - if (this.futs.compareAndSet(null, newFuts)) - futs = newFuts; - else - futs = this.futs.get(); - } + ConcurrentHashMap> futs = runningFutures(); ResultSetWrapper res = null; @@ -1296,7 +1284,8 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, @Nullable Stri assert rs instanceof JdbcResultSet : rs.getClass(); - futs.remove(key); + if (this.futs.get() == futs) + futs.remove(key); res = new ResultSetWrapper(rs); @@ -1310,6 +1299,27 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, @Nullable Stri return res; } + /** + * @return Map containing futures for currently running queries. + */ + private ConcurrentHashMap> runningFutures() { + ConcurrentHashMap> futs = this.futs.get(); + + ConcurrentHashMap> newFuts = null; + + while (futs == null) { + if (newFuts == null) + newFuts = new ConcurrentHashMap<>(); + + if (this.futs.compareAndSet(null, newFuts)) + futs = newFuts; + else + futs = this.futs.get(); + } + + return futs; + } + /** * Rewind given {@link ResultInterface} to the desired position. * @param res Result. @@ -1350,9 +1360,6 @@ private final static class QueryKey { /** Partitions to run query on. */ private final int[] parts; - /** Page size. */ - private final int pageSize; - /** * Constructor. * @param qry Query string. @@ -1360,16 +1367,14 @@ private final static class QueryKey { * @param joinMode Distributed join mode. * @param enforceJoinOrder Enforce join order flag. * @param parts Partitions to run query on. - * @param pageSize Page size. */ private QueryKey(String qry, Object[] params, DistributedJoinMode joinMode, boolean enforceJoinOrder, - int[] parts, int pageSize) { + int[] parts) { this.qry = qry; this.params = params; this.joinMode = joinMode; this.enforceJoinOrder = enforceJoinOrder; this.parts = parts; - this.pageSize = pageSize; } /** {@inheritDoc} */ @@ -1381,7 +1386,7 @@ private QueryKey(String qry, Object[] params, DistributedJoinMode joinMode, bool return enforceJoinOrder == qryKey.enforceJoinOrder && qry.equals(qryKey.qry) && Arrays.equals(params, qryKey.params) && joinMode == qryKey.joinMode && - Arrays.equals(parts, qryKey.parts) && pageSize == qryKey.pageSize; + Arrays.equals(parts, qryKey.parts); } /** {@inheritDoc} */ @@ -1390,8 +1395,7 @@ private QueryKey(String qry, Object[] params, DistributedJoinMode joinMode, bool res = 31 * res + Arrays.hashCode(params); res = 31 * res + joinMode.hashCode(); res = 31 * res + (enforceJoinOrder ? 1 : 0); - res = 31 * res + Arrays.hashCode(parts); - return 31 * res + pageSize; + return 31 * res + Arrays.hashCode(parts); } } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java index bceccb94a7070..bb65b43cbd842 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java @@ -31,7 +31,6 @@ import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import javax.cache.CacheException; @@ -897,10 +896,6 @@ public void testConcurrentSimilarQueries() throws Exception { final AtomicBoolean stop = new AtomicBoolean(); - final SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * from \"large\".Integer WHERE _key > ? and _val < ? " + - "order by _key") - .setArgs(3000, 950000).setTimeout(20, TimeUnit.SECONDS); - final AtomicInteger cnt = new AtomicInteger(); IgniteInternalFuture fut = multithreadedAsync(new IgniteCallable() { @@ -908,6 +903,10 @@ public void testConcurrentSimilarQueries() throws Exception { while (!stop.get()) { IgniteEx node = grid(gridCount() > 0 ? ThreadLocalRandom.current().nextInt(0, gridCount()) : 0); + SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * from \"large\".Integer WHERE _key > ? and " + + "_val < ? order by _key").setArgs(3000, 950000) + .setPageSize(ThreadLocalRandom.current().nextInt(100, 5000)); + List> res = node.cache("large").query(qry).getAll(); assertEquals(946999, res.size()); @@ -921,12 +920,30 @@ public void testConcurrentSimilarQueries() throws Exception { } }, 8); + // Let's also make some threads do occasional puts. + IgniteInternalFuture putFut = multithreadedAsync(new IgniteCallable() { + @Override public Object call() throws Exception { + while (!stop.get()) { + IgniteEx node = grid(gridCount() > 0 ? ThreadLocalRandom.current().nextInt(0, gridCount()) : 0); + + int x = ThreadLocalRandom.current().nextInt(0, 1_000_000); + + node.cache("large").put(x, x); + + Thread.sleep(ThreadLocalRandom.current().nextInt(500, 1000)); + } + return null; + } + }, 4); + Thread.sleep(120000); System.out.println("Iterations done: " + cnt); stop.set(true); + putFut.get(); + fut.get(); } From 74ab7b73a1a6fb15d968746e6715693b1723a298 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Fri, 16 Jun 2017 13:13:34 +0300 Subject: [PATCH 018/155] Minor fixes --- .../query/h2/twostep/GridMapQueryExecutor.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 1d19b36d752f8..480c5940a04b9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -84,7 +84,6 @@ import org.h2.jdbc.JdbcResultSet; import org.h2.result.ResultInterface; import org.h2.value.Value; -import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; @@ -627,7 +626,7 @@ private void onQueryRequest0( QueryKey key = new QueryKey(qry.query(), qry.parameters(params), distributedJoinMode, enforceJoinOrder, parts); - rs = runQuery(key, node, evt ? mainCctx.name() : null, conn, timeout, qr.cancels[qryIdx], evt); + rs = runQuery(key, node, mainCctx, conn, timeout, qr.cancels[qryIdx], evt); } qr.addResult(qryIdx, qry, node.id(), rs, params); @@ -1230,7 +1229,7 @@ private List row(Value[] row) { * Execute query specified by {@code key}. * @param key Query key. * @param node Cluster node. - * @param mainCctxName Main cache context name, may be {@code null}. + * @param mainCctx Cache context. * @param conn H2 connection. * @param timeout Query timeout. * @param cancel Query cancel state holder. @@ -1238,7 +1237,7 @@ private List row(Value[] row) { * @return Query result. * @throws IgniteCheckedException if failed. */ - private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, @Nullable String mainCctxName, Connection conn, + private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheContext mainCctx, Connection conn, int timeout, GridQueryCancel cancel, boolean evt) throws IgniteCheckedException { ConcurrentHashMap> futs = runningFutures(); @@ -1263,7 +1262,7 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, @Nullable Stri if (res == null || !res.tryTake()) { fut = newFut; - ResultSet rs = h2.executeSqlQueryWithTimer(mainCctxName, conn, key.qry, F.asList(key.params), true, + ResultSet rs = h2.executeSqlQueryWithTimer(mainCctx.name(), conn, key.qry, F.asList(key.params), true, timeout, cancel); if (evt) { @@ -1272,7 +1271,7 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, @Nullable Stri "SQL query executed.", EVT_CACHE_QUERY_EXECUTED, CacheQueryType.SQL.name(), - mainCctxName, + mainCctx.namex(), null, key.qry, null, From 0e5da2cbdc499ac9427297b813a540d9712b6ab6 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Fri, 16 Jun 2017 13:30:15 +0300 Subject: [PATCH 019/155] Minor. --- .../h2/twostep/GridMapQueryExecutor.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 480c5940a04b9..d27343858f207 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -1095,13 +1095,11 @@ private QueryResult(ResultSetWrapper rs, GridCacheContext cctx, UUID qrySr ResultInterface res = null; if (rs != null) { - synchronized (rs) { - try { - res = (ResultInterface)RESULT_FIELD.get(rs.resultSet()); - } - catch (IllegalAccessException e) { - throw new IllegalStateException(e); // Must not happen. - } + try { + res = (ResultInterface)RESULT_FIELD.get(rs.resultSet()); + } + catch (IllegalAccessException e) { + throw new IllegalStateException(e); // Must not happen. } } @@ -1422,30 +1420,31 @@ private ResultSetWrapper(ResultSet rs) { /** * @return Wrapped result set. */ - ResultSet resultSet() { + synchronized ResultSet resultSet() { checkState(); return rs; } - /** - * @return State flag. - */ - public boolean isClosed() { - return closed; - } - /** * @return {@code true} if {@link ResultSet} wrapped by this object may be used. */ synchronized boolean tryTake() { - return ++cnt > 0 && !closed; + if (++cnt <= 0) { + assert closed; + + // Should never happen - counter can be negative iff release has somehow negated it which at a glance + // does not seem possible. + throw new IllegalStateException(); + } + + return !closed; } /** * Release the hold on this result set. */ - public synchronized void release() { + synchronized void release() { checkState(); if (--cnt == 0) { From 4e75662ca8af6cd307d55ec1b726e129369309a2 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Fri, 16 Jun 2017 14:57:18 +0300 Subject: [PATCH 020/155] Segment id fix. --- .../processors/query/h2/IgniteH2Indexing.java | 8 +++---- .../h2/twostep/GridMapQueryExecutor.java | 21 ++++++++++++------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 5e8ab9dc68c64..e951ae26a1baf 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -654,12 +654,12 @@ private void onSqlException() { if (expirationTime == 0) expirationTime = Long.MAX_VALUE; + mapQryExec.clearRunningQueriesState(); + tbl.tbl.update(k, partId, v, ver, expirationTime, false, link); if (tbl.luceneIdx != null) tbl.luceneIdx.store(k, v, ver, expirationTime); - - mapQryExec.clearRunningQueriesState(); } /** @@ -699,11 +699,11 @@ private GridCacheContext cacheContext(String space) { if (tbl == null) return; + mapQryExec.clearRunningQueriesState(); + if (tbl.tbl.update(key, partId, val, ver, 0, true, 0)) { if (tbl.luceneIdx != null) tbl.luceneIdx.remove(key); - - mapQryExec.clearRunningQueriesState(); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index d27343858f207..e77c6d8a8627c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -229,7 +229,7 @@ else if (msg instanceof GridQueryCancelRequest) } /** - * + * Clear map of currently running queries to avoid re-use of their results. */ public void clearRunningQueriesState() { futs.set(null); @@ -623,7 +623,7 @@ private void onQueryRequest0( // If we are not the target node for this replicated query, just ignore it. if (qry.node() == null || (segmentId == 0 && qry.node().equals(ctx.localNodeId()))) { - QueryKey key = new QueryKey(qry.query(), qry.parameters(params), distributedJoinMode, + QueryKey key = new QueryKey(qry.query(), segmentId, qry.parameters(params), distributedJoinMode, enforceJoinOrder, parts); rs = runQuery(key, node, mainCctx, conn, timeout, qr.cancels[qryIdx], evt); @@ -1345,6 +1345,9 @@ private final static class QueryKey { /** Query string. */ private final String qry; + /** Segment id. */ + private final int segment; + /** Query parameters. */ private final Object[] params; @@ -1360,14 +1363,16 @@ private final static class QueryKey { /** * Constructor. * @param qry Query string. + * @param segment Segment id. * @param params Query parameters. * @param joinMode Distributed join mode. * @param enforceJoinOrder Enforce join order flag. * @param parts Partitions to run query on. */ - private QueryKey(String qry, Object[] params, DistributedJoinMode joinMode, boolean enforceJoinOrder, - int[] parts) { + private QueryKey(String qry, int segment, Object[] params, DistributedJoinMode joinMode, boolean enforceJoinOrder, + int[] parts) { this.qry = qry; + this.segment = segment; this.params = params; this.joinMode = joinMode; this.enforceJoinOrder = enforceJoinOrder; @@ -1381,18 +1386,20 @@ private QueryKey(String qry, Object[] params, DistributedJoinMode joinMode, bool QueryKey qryKey = (QueryKey) o; - return enforceJoinOrder == qryKey.enforceJoinOrder && qry.equals(qryKey.qry) && - Arrays.equals(params, qryKey.params) && joinMode == qryKey.joinMode && + return (segment == qryKey.segment) && (enforceJoinOrder == qryKey.enforceJoinOrder) && + F.eq(qry, qryKey.qry) && Arrays.equals(params, qryKey.params) && joinMode == qryKey.joinMode && Arrays.equals(parts, qryKey.parts); + } /** {@inheritDoc} */ @Override public int hashCode() { int res = qry.hashCode(); + res = 31 * res + segment; res = 31 * res + Arrays.hashCode(params); res = 31 * res + joinMode.hashCode(); res = 31 * res + (enforceJoinOrder ? 1 : 0); - return 31 * res + Arrays.hashCode(parts); + return 31 * res + Arrays.hashCode(parts); } } From 813b5101cd8c5e07c803af1950494f00cc875066 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Fri, 16 Jun 2017 16:20:19 +0300 Subject: [PATCH 021/155] Decreased test duration. --- .../cache/IgniteCacheAbstractFieldsQuerySelfTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java index bb65b43cbd842..382889d205dea 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java @@ -936,7 +936,7 @@ public void testConcurrentSimilarQueries() throws Exception { } }, 4); - Thread.sleep(120000); + Thread.sleep(30000); System.out.println("Iterations done: " + cnt); From 63fad7d78596024d32a32337a23fd1a4656644d9 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Fri, 16 Jun 2017 22:06:30 +0300 Subject: [PATCH 022/155] log4test --- .../processors/query/GridQueryProcessor.java | 51 +++++++++++-------- .../internal/util/GridSpinReadWriteLock.java | 6 +++ .../h2/twostep/GridMapQueryExecutor.java | 11 ++++ ...tedPartitionQueryNodeRestartsSelfTest.java | 4 +- 4 files changed, 49 insertions(+), 23 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index 0399fc4f25f76..2392b17b118b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -17,6 +17,25 @@ package org.apache.ignite.internal.processors.query; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.cache.Cache; +import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; @@ -33,8 +52,8 @@ import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.managers.communication.GridMessageListener; import org.apache.ignite.internal.NodeStoppingException; +import org.apache.ignite.internal.managers.communication.GridMessageListener; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheObject; @@ -74,9 +93,9 @@ import org.apache.ignite.internal.util.lang.GridClosureException; import org.apache.ignite.internal.util.lang.IgniteOutClosureX; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.T3; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; import org.apache.ignite.internal.util.worker.GridWorkerFuture; @@ -90,26 +109,6 @@ import org.apache.ignite.thread.IgniteThread; import org.jetbrains.annotations.Nullable; -import javax.cache.Cache; -import javax.cache.CacheException; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicBoolean; - import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; import static org.apache.ignite.internal.GridTopic.TOPIC_SCHEMA; import static org.apache.ignite.internal.IgniteComponentType.INDEXING; @@ -1654,6 +1653,8 @@ public FieldsQueryCursor> queryTwoStep(final GridCacheContext cctx, if (!busyLock.enterBusy()) throw new IllegalStateException("Failed to execute query (grid is stopping)."); + log.error("ENTERED BUSY"); + try { return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), cctx, new IgniteOutClosureX>>() { @@ -1667,6 +1668,8 @@ public FieldsQueryCursor> queryTwoStep(final GridCacheContext cctx, } finally { busyLock.leaveBusy(); + + log.error("EXITED BUSY"); } } @@ -1711,6 +1714,8 @@ public QueryCursor> queryTwoStep(final GridCacheContext>>() { @@ -1724,6 +1729,8 @@ public QueryCursor> queryTwoStep(final GridCacheContext row(Value[] row) { } } + + public AtomicInteger roots = new AtomicInteger(); + /** * Execute query specified by {@code key}. * @param key Query key. @@ -1285,6 +1293,7 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte futs.remove(key); res = new ResultSetWrapper(rs); + roots.incrementAndGet(); if (!res.tryTake()) throw new IllegalStateException(); @@ -1455,6 +1464,8 @@ synchronized void release() { checkState(); if (--cnt == 0) { + roots.getAndDecrement(); + closed = true; U.close(rs, log); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java index 68f98420ade35..153372a9c4c2c 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java @@ -22,8 +22,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerArray; - import org.apache.ignite.Ignite; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; import org.apache.ignite.internal.IgniteInternalFuture; @@ -78,6 +78,8 @@ public Void call() throws Exception { restartStats.incrementAndGet(grid); try { + IgniteEx ggrid = grid(grid); + stopGrid(grid); Thread.sleep(rnd.nextInt(NODE_RESTART_TIME)); From c47fa29783b5217631159e1c00e335537d86da92 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Sat, 17 Jun 2017 13:08:23 +0300 Subject: [PATCH 023/155] IGNITE-5482 QueryKey fixed --- .../processors/query/GridQueryProcessor.java | 4 -- .../internal/util/GridSpinReadWriteLock.java | 6 -- .../h2/twostep/GridMapQueryExecutor.java | 57 ++++++++++++------- 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index 2392b17b118b0..5bd68b2660829 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -1653,8 +1653,6 @@ public FieldsQueryCursor> queryTwoStep(final GridCacheContext cctx, if (!busyLock.enterBusy()) throw new IllegalStateException("Failed to execute query (grid is stopping)."); - log.error("ENTERED BUSY"); - try { return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), cctx, new IgniteOutClosureX>>() { @@ -1668,8 +1666,6 @@ public FieldsQueryCursor> queryTwoStep(final GridCacheContext cctx, } finally { busyLock.leaveBusy(); - - log.error("EXITED BUSY"); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridSpinReadWriteLock.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridSpinReadWriteLock.java index 92934647972f1..4f23979e29c7a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridSpinReadWriteLock.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridSpinReadWriteLock.java @@ -201,15 +201,9 @@ public void writeLock() { boolean interrupted = false; - long time = 0; - - System.out.println("****** " + writeLockOwner); - while (!compareAndSet(STATE_OFFS, 0, -1)) { try { Thread.sleep(10); - - time += 10; } catch (InterruptedException ignored) { interrupted = true; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 184211d2c589f..505b51f85178c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -184,8 +184,6 @@ public void start(final GridKernalContext ctx, IgniteH2Indexing h2) throws Ignit if (!busyLock.enterBusy()) return; - log.error("ENTERED MAP BUSY"); - try { if (msg instanceof GridCacheQueryMarshallable) ((GridCacheQueryMarshallable)msg).unmarshall(ctx.config().getMarshaller(), ctx); @@ -194,8 +192,6 @@ public void start(final GridKernalContext ctx, IgniteH2Indexing h2) throws Ignit } finally { busyLock.leaveBusy(); - - log.error("EXITED MAP BUSY"); } } }); @@ -628,8 +624,8 @@ private void onQueryRequest0( // If we are not the target node for this replicated query, just ignore it. if (qry.node() == null || (segmentId == 0 && qry.node().equals(ctx.localNodeId()))) { - QueryKey key = new QueryKey(qry.query(), segmentId, qry.parameters(params), distributedJoinMode, - enforceJoinOrder, parts); + QueryKey key = new QueryKey(qry.query(), topVer, segmentId, qry.parameters(params), distributedJoinMode, + enforceJoinOrder, partsMap, parts); rs = runQuery(key, node, mainCctx, conn, timeout, qr.cancels[qryIdx], evt); } @@ -1354,6 +1350,8 @@ private final static class QueryKey { /** Query string. */ private final String qry; + private final AffinityTopologyVersion topVer; + /** Segment id. */ private final int segment; @@ -1366,49 +1364,64 @@ private final static class QueryKey { /** Enforce join order flag. */ private final boolean enforceJoinOrder; + /** Partitions map. */ + private final Map partsMap; + /** Partitions to run query on. */ private final int[] parts; /** * Constructor. * @param qry Query string. + * @param topVer Topology version. * @param segment Segment id. * @param params Query parameters. * @param joinMode Distributed join mode. * @param enforceJoinOrder Enforce join order flag. + * @param partsMap Partitions map. * @param parts Partitions to run query on. */ - private QueryKey(String qry, int segment, Object[] params, DistributedJoinMode joinMode, boolean enforceJoinOrder, - int[] parts) { + private QueryKey(String qry, AffinityTopologyVersion topVer, int segment, Object[] params, + DistributedJoinMode joinMode, boolean enforceJoinOrder, Map partsMap, int[] parts) { this.qry = qry; + this.topVer = topVer; this.segment = segment; this.params = params; this.joinMode = joinMode; this.enforceJoinOrder = enforceJoinOrder; + this.partsMap = partsMap; this.parts = parts; } - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { + @Override + public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - QueryKey qryKey = (QueryKey) o; + QueryKey queryKey = (QueryKey) o; - return (segment == qryKey.segment) && (enforceJoinOrder == qryKey.enforceJoinOrder) && - F.eq(qry, qryKey.qry) && Arrays.equals(params, qryKey.params) && joinMode == qryKey.joinMode && - Arrays.equals(parts, qryKey.parts); + if (segment != queryKey.segment) return false; + if (enforceJoinOrder != queryKey.enforceJoinOrder) return false; + if (!qry.equals(queryKey.qry)) return false; + if (!topVer.equals(queryKey.topVer)) return false; + // Probably incorrect - comparing Object[] arrays with Arrays.equals + if (!Arrays.equals(params, queryKey.params)) return false; + if (joinMode != queryKey.joinMode) return false; + return (F.eq(partsMap, queryKey.partsMap)) && Arrays.equals(parts, queryKey.parts); } - /** {@inheritDoc} */ - @Override public int hashCode() { - int res = qry.hashCode(); - res = 31 * res + segment; - res = 31 * res + Arrays.hashCode(params); - res = 31 * res + joinMode.hashCode(); - res = 31 * res + (enforceJoinOrder ? 1 : 0); - return 31 * res + Arrays.hashCode(parts); + @Override + public int hashCode() { + int result = qry.hashCode(); + result = 31 * result + topVer.hashCode(); + result = 31 * result + segment; + result = 31 * result + Arrays.hashCode(params); + result = 31 * result + joinMode.hashCode(); + result = 31 * result + (enforceJoinOrder ? 1 : 0); + result = 31 * result + (partsMap != null ? partsMap.hashCode() : 0); + result = 31 * result + Arrays.hashCode(parts); + return result; } } From 39ce9d0ff016012e5c42e1a28023038f8a9dc4f8 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Sat, 17 Jun 2017 14:09:53 +0300 Subject: [PATCH 024/155] IGNITE-5482 Errors handling fix. --- .../h2/twostep/GridMapQueryExecutor.java | 108 +++++++++--------- 1 file changed, 52 insertions(+), 56 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 505b51f85178c..14157e60759ba 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -33,7 +33,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceArray; import javax.cache.CacheException; @@ -624,8 +623,8 @@ private void onQueryRequest0( // If we are not the target node for this replicated query, just ignore it. if (qry.node() == null || (segmentId == 0 && qry.node().equals(ctx.localNodeId()))) { - QueryKey key = new QueryKey(qry.query(), topVer, segmentId, qry.parameters(params), distributedJoinMode, - enforceJoinOrder, partsMap, parts); + QueryKey key = new QueryKey(qry.query(), topVer, segmentId, qry.parameters(params), + distributedJoinMode, enforceJoinOrder, partsMap, parts); rs = runQuery(key, node, mainCctx, conn, timeout, qr.cancels[qryIdx], evt); } @@ -1224,9 +1223,6 @@ private List row(Value[] row) { } } - - public AtomicInteger roots = new AtomicInteger(); - /** * Execute query specified by {@code key}. * @param key Query key. @@ -1264,37 +1260,44 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte if (res == null || !res.tryTake()) { fut = newFut; - ResultSet rs = h2.executeSqlQueryWithTimer(mainCctx.name(), conn, key.qry, F.asList(key.params), true, - timeout, cancel); - - if (evt) { - ctx.event().record(new CacheQueryExecutedEvent<>( - node, - "SQL query executed.", - EVT_CACHE_QUERY_EXECUTED, - CacheQueryType.SQL.name(), - mainCctx.namex(), - null, - key.qry, - null, - null, - key.params, - node.id(), - null)); - } + try { + ResultSet rs = h2.executeSqlQueryWithTimer(mainCctx.name(), conn, key.qry, F.asList(key.params), + true, timeout, cancel); + + if (evt) { + ctx.event().record(new CacheQueryExecutedEvent<>( + node, + "SQL query executed.", + EVT_CACHE_QUERY_EXECUTED, + CacheQueryType.SQL.name(), + mainCctx.namex(), + null, + key.qry, + null, + null, + key.params, + node.id(), + null)); + } + + assert rs instanceof JdbcResultSet : rs.getClass(); - assert rs instanceof JdbcResultSet : rs.getClass(); + if (this.futs.get() == futs) + futs.remove(key); - if (this.futs.get() == futs) - futs.remove(key); + res = new ResultSetWrapper(rs); - res = new ResultSetWrapper(rs); - roots.incrementAndGet(); + if (!res.tryTake()) + throw new IllegalStateException(); - if (!res.tryTake()) - throw new IllegalStateException(); + fut.onDone(res); + } + catch (Exception e) { + // All other folks waiting for the same query result must see the same exception. + fut.onDone(e); - fut.onDone(res); + throw e; + } } } @@ -1350,6 +1353,7 @@ private final static class QueryKey { /** Query string. */ private final String qry; + /** Topology version. */ private final AffinityTopologyVersion topVer; /** Segment id. */ @@ -1393,35 +1397,29 @@ private QueryKey(String qry, AffinityTopologyVersion topVer, int segment, Object this.parts = parts; } - @Override - public boolean equals(Object o) { + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - QueryKey queryKey = (QueryKey) o; + QueryKey qryKey = (QueryKey) o; - if (segment != queryKey.segment) return false; - if (enforceJoinOrder != queryKey.enforceJoinOrder) return false; - if (!qry.equals(queryKey.qry)) return false; - if (!topVer.equals(queryKey.topVer)) return false; - // Probably incorrect - comparing Object[] arrays with Arrays.equals - if (!Arrays.equals(params, queryKey.params)) return false; - if (joinMode != queryKey.joinMode) return false; - return (F.eq(partsMap, queryKey.partsMap)) && Arrays.equals(parts, queryKey.parts); + return (segment == qryKey.segment) && (enforceJoinOrder == qryKey.enforceJoinOrder) && + F.eq(qry, qryKey.qry) && F.eq(topVer, qryKey.topVer) && Arrays.equals(params, qryKey.params) && + (joinMode == qryKey.joinMode) && (F.eq(partsMap, qryKey.partsMap)) && Arrays.equals(parts, qryKey.parts); } - @Override - public int hashCode() { - int result = qry.hashCode(); - result = 31 * result + topVer.hashCode(); - result = 31 * result + segment; - result = 31 * result + Arrays.hashCode(params); - result = 31 * result + joinMode.hashCode(); - result = 31 * result + (enforceJoinOrder ? 1 : 0); - result = 31 * result + (partsMap != null ? partsMap.hashCode() : 0); - result = 31 * result + Arrays.hashCode(parts); - return result; + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = qry.hashCode(); + res = 31 * res + topVer.hashCode(); + res = 31 * res + segment; + res = 31 * res + Arrays.hashCode(params); + res = 31 * res + joinMode.hashCode(); + res = 31 * res + (enforceJoinOrder ? 1 : 0); + res = 31 * res + (partsMap != null ? partsMap.hashCode() : 0); + return 31 * res + Arrays.hashCode(parts); } } @@ -1477,8 +1475,6 @@ synchronized void release() { checkState(); if (--cnt == 0) { - roots.getAndDecrement(); - closed = true; U.close(rs, log); From 1edc9813a73b43f612d3cdb1c5521d202a02059b Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Sat, 17 Jun 2017 14:15:02 +0300 Subject: [PATCH 025/155] IGNITE-5482 Errors handling fix 2. --- .../processors/query/h2/twostep/GridMapQueryExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 14157e60759ba..981b4f27f8328 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -1292,7 +1292,7 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte fut.onDone(res); } - catch (Exception e) { + catch (Throwable e) { // All other folks waiting for the same query result must see the same exception. fut.onDone(e); From d20594cdf23ccd69dee8f5bc904a8987dc0d01be Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Sat, 17 Jun 2017 14:15:32 +0300 Subject: [PATCH 026/155] Minor. --- .../processors/query/h2/twostep/GridMapQueryExecutor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 981b4f27f8328..54710887a0660 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -1237,7 +1237,6 @@ private List row(Value[] row) { */ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheContext mainCctx, Connection conn, int timeout, GridQueryCancel cancel, boolean evt) throws IgniteCheckedException { - ConcurrentHashMap> futs = runningFutures(); ResultSetWrapper res = null; From c991054259a15fe3bb53ffbbe364e5e5258a8e22 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Sat, 17 Jun 2017 14:45:15 +0300 Subject: [PATCH 027/155] Minor. --- .../processors/query/h2/twostep/GridMapQueryExecutor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 54710887a0660..7c8a8b1933b2f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -1281,7 +1281,9 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte assert rs instanceof JdbcResultSet : rs.getClass(); - if (this.futs.get() == futs) + // No need to worry about removing this key from the map if we're not the one who put it there + // or if our map is stale. + if (res == null && this.futs.get() == futs) futs.remove(key); res = new ResultSetWrapper(rs); From 83413577b46ac5a80f77372fffbe49155b31a99c Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Mon, 19 Jun 2017 14:08:26 +0300 Subject: [PATCH 028/155] Removed undesired logging. --- .../ignite/internal/processors/query/GridQueryProcessor.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index 5bd68b2660829..2b472328d7ec7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -1710,8 +1710,6 @@ public QueryCursor> queryTwoStep(final GridCacheContext>>() { @@ -1725,8 +1723,6 @@ public QueryCursor> queryTwoStep(final GridCacheContext Date: Tue, 20 Jun 2017 13:09:33 +0300 Subject: [PATCH 029/155] IGNITE-5482 Conditional throttling. --- .../h2/twostep/GridMapQueryExecutor.java | 84 ++++++++++++++----- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 7c8a8b1933b2f..4369b037bc533 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -624,7 +624,7 @@ private void onQueryRequest0( if (qry.node() == null || (segmentId == 0 && qry.node().equals(ctx.localNodeId()))) { QueryKey key = new QueryKey(qry.query(), topVer, segmentId, qry.parameters(params), - distributedJoinMode, enforceJoinOrder, partsMap, parts); + distributedJoinMode, enforceJoinOrder, partsMap, parts); rs = runQuery(key, node, mainCctx, conn, timeout, qr.cancels[qryIdx], evt); } @@ -1224,8 +1224,8 @@ private List row(Value[] row) { } /** - * Execute query specified by {@code key}. - * @param key Query key. + * Execute query specified by {@code key} or reuse result of the same concurrently running query. + * @param key Query key to throttle similar queries as needed. * @param node Cluster node. * @param mainCctx Cache context. * @param conn H2 connection. @@ -1237,6 +1237,17 @@ private List row(Value[] row) { */ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheContext mainCctx, Connection conn, int timeout, GridQueryCancel cancel, boolean evt) throws IgniteCheckedException { + if (!key.isThrottled()) { + ResultSet rs = executeQuery(node, mainCctx, conn, key.qry, key.params, timeout, cancel, evt); + + ResultSetWrapper res = new ResultSetWrapper(rs); + + if (!res.tryTake()) + throw new IllegalStateException(); + + return res; + } + ConcurrentHashMap> futs = runningFutures(); ResultSetWrapper res = null; @@ -1260,26 +1271,7 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte fut = newFut; try { - ResultSet rs = h2.executeSqlQueryWithTimer(mainCctx.name(), conn, key.qry, F.asList(key.params), - true, timeout, cancel); - - if (evt) { - ctx.event().record(new CacheQueryExecutedEvent<>( - node, - "SQL query executed.", - EVT_CACHE_QUERY_EXECUTED, - CacheQueryType.SQL.name(), - mainCctx.namex(), - null, - key.qry, - null, - null, - key.params, - node.id(), - null)); - } - - assert rs instanceof JdbcResultSet : rs.getClass(); + ResultSet rs = executeQuery(node, mainCctx, conn, key.qry, key.params, timeout, cancel, evt); // No need to worry about removing this key from the map if we're not the one who put it there // or if our map is stale. @@ -1305,6 +1297,45 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte return res; } + /** + * Execute query on internal H2 instance and optionally record query event. + * @param node Cluster node. + * @param mainCctx Cache context. + * @param conn H2 connection. + * @param qry SQL query string. + * @param params Parameters. + * @param timeout Query timeout. + * @param cancel Query cancel state holder. + * @param evt {@code true} if this query event is recordable, {@code false} otherwise. + * @return Query result. + * @throws IgniteCheckedException if failed. + */ + private ResultSet executeQuery(ClusterNode node, GridCacheContext mainCctx, Connection conn, String qry, + Object[] params, int timeout, GridQueryCancel cancel, boolean evt) throws IgniteCheckedException { + ResultSet rs = h2.executeSqlQueryWithTimer(mainCctx.name(), conn, qry, F.asList(params), + true, timeout, cancel); + + if (evt) { + ctx.event().record(new CacheQueryExecutedEvent<>( + node, + "SQL query executed.", + EVT_CACHE_QUERY_EXECUTED, + CacheQueryType.SQL.name(), + mainCctx.namex(), + null, + qry, + null, + null, + params, + node.id(), + null)); + } + + assert rs instanceof JdbcResultSet : rs.getClass(); + + return rs; + } + /** * @return Map containing futures for currently running queries. */ @@ -1422,6 +1453,13 @@ private QueryKey(String qry, AffinityTopologyVersion topVer, int segment, Object res = 31 * res + (partsMap != null ? partsMap.hashCode() : 0); return 31 * res + Arrays.hashCode(parts); } + + /** + * @return {@code true} if this key corresponds to a query whose result may be reused, {@code false} otherwise. + */ + public boolean isThrottled() { + return parts != null; + } } /** */ From f6dfa68af7909f962fd4e5afbc2cedda0a5a0732 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Tue, 20 Jun 2017 19:46:56 +0300 Subject: [PATCH 030/155] IGNITE-5482 Throttling logic reworked. --- .../h2/twostep/GridMapQueryExecutor.java | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 4369b037bc533..d90210d38e14b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -1085,27 +1085,16 @@ private class QueryResult implements AutoCloseable { */ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") private QueryResult(ResultSetWrapper rs, GridCacheContext cctx, UUID qrySrcNodeId, GridCacheSqlQuery qry, - Object[] params) { + Object[] params) { this.cctx = cctx; this.qry = qry; this.params = params; this.qrySrcNodeId = qrySrcNodeId; this.cpNeeded = cctx.isLocalNode(qrySrcNodeId); - ResultInterface res = null; - if (rs != null) { - try { - res = (ResultInterface)RESULT_FIELD.get(rs.resultSet()); - } - catch (IllegalAccessException e) { - throw new IllegalStateException(e); // Must not happen. - } - } - - if (res != null) { this.rs = rs; - this.res = res; + this.res = rs.rows(); rowCnt = res.getRowCount(); cols = res.getVisibleColumnCount(); @@ -1237,7 +1226,7 @@ private List row(Value[] row) { */ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheContext mainCctx, Connection conn, int timeout, GridQueryCancel cancel, boolean evt) throws IgniteCheckedException { - if (!key.isThrottled()) { + if (!isThrottled(key)) { ResultSet rs = executeQuery(node, mainCctx, conn, key.qry, key.params, timeout, cancel, evt); ResultSetWrapper res = new ResultSetWrapper(rs); @@ -1297,6 +1286,14 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte return res; } + /** + * @return {@code true} if this key corresponds to a query whose result may be reused, {@code false} otherwise. + */ + private boolean isThrottled(QueryKey key) { + // Throttle the query if we don't have partitions list set, OR if it's what we've got from partitions map. + return key.parts == null || (key.partsMap != null && key.partsMap.get(ctx.localNodeId()) == key.parts); + } + /** * Execute query on internal H2 instance and optionally record query event. * @param node Cluster node. @@ -1453,13 +1450,6 @@ private QueryKey(String qry, AffinityTopologyVersion topVer, int segment, Object res = 31 * res + (partsMap != null ? partsMap.hashCode() : 0); return 31 * res + Arrays.hashCode(parts); } - - /** - * @return {@code true} if this key corresponds to a query whose result may be reused, {@code false} otherwise. - */ - public boolean isThrottled() { - return parts != null; - } } /** */ @@ -1470,6 +1460,9 @@ private final class ResultSetWrapper { /** Wrapped result set. */ private final ResultSet rs; + /** Rows of {@link #rs}. */ + private final ResultInterface rows; + /** State flag. */ private volatile boolean closed; @@ -1481,15 +1474,24 @@ private ResultSetWrapper(ResultSet rs) { A.notNull(rs, "rs"); this.rs = rs; + + try { + this.rows = (ResultInterface)RESULT_FIELD.get(rs); + } + catch (IllegalAccessException e) { + throw new IllegalStateException(e); // Must not happen. + } + + assert rows != null; } /** * @return Wrapped result set. */ - synchronized ResultSet resultSet() { + synchronized ResultInterface rows() { checkState(); - return rs; + return rows; } /** From f6678134318441d098198883938998fb86a1e458 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 22 Jun 2017 17:21:44 +0300 Subject: [PATCH 031/155] IGNITE-5482 Result set release logic reworked. --- .../h2/twostep/GridMapQueryExecutor.java | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index d90210d38e14b..10fe3c7e232bc 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -84,6 +84,7 @@ import org.h2.jdbc.JdbcResultSet; import org.h2.result.ResultInterface; import org.h2.value.Value; +import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; @@ -1229,7 +1230,7 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte if (!isThrottled(key)) { ResultSet rs = executeQuery(node, mainCctx, conn, key.qry, key.params, timeout, cancel, evt); - ResultSetWrapper res = new ResultSetWrapper(rs); + ResultSetWrapper res = new ResultSetWrapper(rs, null, null); if (!res.tryTake()) throw new IllegalStateException(); @@ -1262,12 +1263,7 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte try { ResultSet rs = executeQuery(node, mainCctx, conn, key.qry, key.params, timeout, cancel, evt); - // No need to worry about removing this key from the map if we're not the one who put it there - // or if our map is stale. - if (res == null && this.futs.get() == futs) - futs.remove(key); - - res = new ResultSetWrapper(rs); + res = new ResultSetWrapper(rs, key, fut); if (!res.tryTake()) throw new IllegalStateException(); @@ -1454,6 +1450,12 @@ private QueryKey(String qry, AffinityTopologyVersion topVer, int segment, Object /** */ private final class ResultSetWrapper { + /** Key that produced this result. */ + private final QueryKey key; + + /** Future holding this wrapper (for cleanup). */ + private final GridFutureAdapter fut; + /** Usages counter. */ private int cnt; @@ -1469,10 +1471,19 @@ private final class ResultSetWrapper { /** * Constructor. * @param rs Result set to wrap. + * @param key Key that produced this result (for cleanup), or {@code null} if this query was not throttled. + * @param fut Future holding this wrapper (for cleanup), or {@code null} if this query was not throttled. */ - private ResultSetWrapper(ResultSet rs) { + private ResultSetWrapper(ResultSet rs, @Nullable QueryKey key, + @Nullable GridFutureAdapter fut) { A.notNull(rs, "rs"); + if (fut != null) + A.notNull(key, "key"); + + this.key = key; + this.fut = fut; + this.rs = rs; try { @@ -1519,6 +1530,13 @@ synchronized void release() { closed = true; U.close(rs, log); + + if (fut != null) { + ConcurrentHashMap> m = futs.get(); + + if (m != null) + m.remove(key, fut); + } } else if (cnt < 0) throw new IllegalStateException(); From 8864b553d49603a8d908c02313e71500dd7a97f7 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 22 Jun 2017 21:29:18 +0300 Subject: [PATCH 032/155] IGNITE-5482 Optional throttling flag + benchmark. --- .../configuration/IgniteConfiguration.java | 45 ++++++ .../h2/twostep/GridMapQueryExecutor.java | 14 +- .../IgniteSqlQueryThrottlingBenchmark.java | 137 ++++++++++++++++++ 3 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryThrottlingBenchmark.java diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java index 9f6839944000d..5f654d1ccaead 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java @@ -196,6 +196,9 @@ public class IgniteConfiguration { /** Default value for active on start flag. */ public static final boolean DFLT_ACTIVE_ON_START = true; + /** Default value for SQL MAP throttling flag. */ + public static final boolean DFLT_SQL_MAP_THROTTLE = true; + /** Default failure detection timeout in millis. */ @SuppressWarnings("UnnecessaryBoxing") public static final Long DFLT_FAILURE_DETECTION_TIMEOUT = new Long(10_000); @@ -273,6 +276,9 @@ public class IgniteConfiguration { /** Daemon flag. */ private boolean daemon; + /** SQL MAP throttling flag. */ + private boolean sqlMapThrottle = DFLT_SQL_MAP_THROTTLE; + /** Whether or not peer class loading is enabled. */ private boolean p2pEnabled = DFLT_P2P_ENABLED; @@ -547,6 +553,7 @@ public IgniteConfiguration(IgniteConfiguration cfg) { segResolvers = cfg.getSegmentationResolvers(); sndRetryCnt = cfg.getNetworkSendRetryCount(); sndRetryDelay = cfg.getNetworkSendRetryDelay(); + sqlMapThrottle = cfg.isSqlMapThrottle(); sslCtxFactory = cfg.getSslContextFactory(); storeSesLsnrs = cfg.getCacheStoreSessionListenerFactories(); stripedPoolSize = cfg.getStripedPoolSize(); @@ -629,6 +636,44 @@ public IgniteConfiguration setDaemon(boolean daemon) { return this; } + /** + * Whether or not SQL MAP queries should be throttled on this node. + *

+ * Distributed SQL queries in Ignite are internally executed in two stages: MAP + * and REDUCE. + *

+ * If many identical SQL queries are executed on the grid within short amount of time + * or simultaneously, their MAP stages occurring on the same node may be throttled - + * i.e. instead of fetching data on each query, it's possible to make all identical queries + * arriving simultaneously reuse the same result set - as long as data is not concurrently modified. + * + * @return {@code True} if SQL MAP queries should be throttled on this node, {@code false} otherwise. + */ + public boolean isSqlMapThrottle() { + return sqlMapThrottle; + } + + /** + * Whether or not SQL MAP queries should be throttled on this node. + *

+ * Distributed SQL queries in Ignite are internally executed in two stages: MAP + * and REDUCE. + *

+ * If many identical SQL queries are executed on the grid within short amount of time + * or simultaneously, their MAP stages occurring on the same node may be throttled - + * i.e. instead of fetching data on each query, it's possible to make all identical queries + * arriving simultaneously reuse the same result set - as long as data is not concurrently modified. + * + * @param sqlMapThrottle SQL MAP throttling flag. + * @return {@code this} for chaining. + * + */ + public IgniteConfiguration setSqlMapThrottle(boolean sqlMapThrottle) { + this.sqlMapThrottle = sqlMapThrottle; + + return this; + } + /** * Sets grid name. Note that {@code null} is a default grid name. * diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 10fe3c7e232bc..b837e4207acb8 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -41,6 +41,7 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.cache.query.QueryCancelledException; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.events.CacheQueryReadEvent; import org.apache.ignite.events.DiscoveryEvent; @@ -130,6 +131,12 @@ public class GridMapQueryExecutor { /** */ private IgniteH2Indexing h2; + /** + * Identical simultaneous queries throttling flag. + * @see IgniteConfiguration#sqlMapThrottle + */ + private boolean throttle; + /** */ private ConcurrentMap qryRess = new ConcurrentHashMap8<>(); @@ -162,6 +169,8 @@ public void start(final GridKernalContext ctx, IgniteH2Indexing h2) throws Ignit log = ctx.log(GridMapQueryExecutor.class); + throttle = ctx.grid().configuration().isSqlMapThrottle(); + final UUID locNodeId = ctx.localNodeId(); ctx.event().addLocalEventListener(new GridLocalEventListener() { @@ -1286,8 +1295,9 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte * @return {@code true} if this key corresponds to a query whose result may be reused, {@code false} otherwise. */ private boolean isThrottled(QueryKey key) { - // Throttle the query if we don't have partitions list set, OR if it's what we've got from partitions map. - return key.parts == null || (key.partsMap != null && key.partsMap.get(ctx.localNodeId()) == key.parts); + return throttle && + // Throttle the query if we don't have partitions list set, OR if it's what we've got from partitions map. + (key.parts == null || (key.partsMap != null && key.partsMap.get(ctx.localNodeId()) == key.parts)); } /** diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryThrottlingBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryThrottlingBenchmark.java new file mode 100644 index 0000000000000..89a27bfd79a95 --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryThrottlingBenchmark.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import javax.cache.Cache; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.query.SqlQuery; +import org.apache.ignite.yardstick.cache.model.Person; +import org.yardstickframework.BenchmarkConfiguration; + +import static org.yardstickframework.BenchmarkUtils.println; + +/** + * Ignite benchmark that performs identical query operations with occasional puts to measure effect of query throttling. + */ +public class IgniteSqlQueryThrottlingBenchmark extends IgniteCacheAbstractBenchmark { + /** */ + private final AtomicInteger putCnt = new AtomicInteger(); + + /** */ + private final AtomicInteger qryCnt = new AtomicInteger(); + + /** Min salary to query. */ + private int minSalary; + + /** Max salary to query. */ + private int maxSalary; + + /** {@inheritDoc} */ + @Override public void setUp(BenchmarkConfiguration cfg) throws Exception { + super.setUp(cfg); + + println(cfg, "Populating query data..."); + + long start = System.nanoTime(); + + try (IgniteDataStreamer dataLdr = ignite().dataStreamer(cache.getName())) { + for (int i = 0; i < args.range() && !Thread.currentThread().isInterrupted(); i++) { + dataLdr.addData(i, person(i)); + + if (i % 100000 == 0) + println(cfg, "Populated persons: " + i); + } + } + + println(cfg, "Finished populating query data in " + ((System.nanoTime() - start) / 1_000_000) + " ms."); + + maxSalary = args.range() * 1000; + + minSalary = (int)(maxSalary * 0.15); + + maxSalary = (int)(maxSalary * 0.85); + } + + /** {@inheritDoc} */ + @Override public boolean test(Map ctx) throws Exception { + // Let's make puts 20% of all operations. + boolean put = (ThreadLocalRandom.current().nextInt(5) == 0); + + if (put) { + cache().put(ThreadLocalRandom.current().nextInt(args.range), + person(ThreadLocalRandom.current().nextInt(args.range))); + + putCnt.incrementAndGet(); + + return true; + } + + Collection> entries = executeQuery(minSalary, maxSalary); + + for (Cache.Entry entry : entries) { + Person p = entry.getValue(); + + if (p.getSalary() < minSalary || p.getSalary() > maxSalary) + throw new Exception("Invalid person retrieved [min=" + minSalary + ", max=" + maxSalary + + ", person=" + p + ']'); + } + + qryCnt.incrementAndGet(); + + return true; + } + + /** {@inheritDoc} */ + @Override public void tearDown() throws Exception { + println(cfg, "SQL query throttling benchmark has finished [putCnt=" + putCnt.get() + ", qryCnt=" + qryCnt.get() + + ']'); + + super.tearDown(); + } + + /** + * @param minSalary Min salary. + * @param maxSalary Max salary. + * @return Query result. + * @throws Exception If failed. + */ + private Collection> executeQuery(double minSalary, double maxSalary) throws Exception { + SqlQuery qry = new SqlQuery<>(Person.class, "salary >= ? and salary <= ?"); + + qry.setArgs(minSalary, maxSalary); + + return cache.query(qry).getAll(); + } + + /** {@inheritDoc} */ + @Override protected IgniteCache cache() { + return ignite().cache("query"); + } + + /** + * A {@link Person} with field values derived from param value. + */ + private static Person person(int p) { + return new Person(p, "firstName" + p, "lastName" + p, p * 1000); + } +} From bce1bb02560e25f33e3c42657101815934a1c63a Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Fri, 23 Jun 2017 15:10:52 +0300 Subject: [PATCH 033/155] IGNITE-5482 Renames. --- .../configuration/IgniteConfiguration.java | 33 +++++++++---------- .../h2/twostep/GridMapQueryExecutor.java | 18 +++++----- ... IgniteSqlQueryResultsReuseBenchmark.java} | 5 +-- 3 files changed, 28 insertions(+), 28 deletions(-) rename modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/{IgniteSqlQueryThrottlingBenchmark.java => IgniteSqlQueryResultsReuseBenchmark.java} (96%) diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java index 5f654d1ccaead..6daad8e5bab9b 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java @@ -196,8 +196,8 @@ public class IgniteConfiguration { /** Default value for active on start flag. */ public static final boolean DFLT_ACTIVE_ON_START = true; - /** Default value for SQL MAP throttling flag. */ - public static final boolean DFLT_SQL_MAP_THROTTLE = true; + /** Default value for SQL results reuse flag. */ + public static final boolean DFLT_SQL_RESULTS_REUSE = true; /** Default failure detection timeout in millis. */ @SuppressWarnings("UnnecessaryBoxing") @@ -276,8 +276,8 @@ public class IgniteConfiguration { /** Daemon flag. */ private boolean daemon; - /** SQL MAP throttling flag. */ - private boolean sqlMapThrottle = DFLT_SQL_MAP_THROTTLE; + /** SQL results reuse flag. */ + private boolean sqlResultsReuse = DFLT_SQL_RESULTS_REUSE; /** Whether or not peer class loading is enabled. */ private boolean p2pEnabled = DFLT_P2P_ENABLED; @@ -553,7 +553,7 @@ public IgniteConfiguration(IgniteConfiguration cfg) { segResolvers = cfg.getSegmentationResolvers(); sndRetryCnt = cfg.getNetworkSendRetryCount(); sndRetryDelay = cfg.getNetworkSendRetryDelay(); - sqlMapThrottle = cfg.isSqlMapThrottle(); + sqlResultsReuse = cfg.isSqlResultsReuse(); sslCtxFactory = cfg.getSslContextFactory(); storeSesLsnrs = cfg.getCacheStoreSessionListenerFactories(); stripedPoolSize = cfg.getStripedPoolSize(); @@ -643,14 +643,14 @@ public IgniteConfiguration setDaemon(boolean daemon) { * and REDUCE. *

* If many identical SQL queries are executed on the grid within short amount of time - * or simultaneously, their MAP stages occurring on the same node may be throttled - - * i.e. instead of fetching data on each query, it's possible to make all identical queries - * arriving simultaneously reuse the same result set - as long as data is not concurrently modified. + * or simultaneously, instead of fetching data on each query, MAP stages + * of all those identical queries occurring on the same node may + * reuse the same result - as long as data is not concurrently modified. * * @return {@code True} if SQL MAP queries should be throttled on this node, {@code false} otherwise. */ - public boolean isSqlMapThrottle() { - return sqlMapThrottle; + public boolean isSqlResultsReuse() { + return sqlResultsReuse; } /** @@ -660,16 +660,15 @@ public boolean isSqlMapThrottle() { * and REDUCE. *

* If many identical SQL queries are executed on the grid within short amount of time - * or simultaneously, their MAP stages occurring on the same node may be throttled - - * i.e. instead of fetching data on each query, it's possible to make all identical queries - * arriving simultaneously reuse the same result set - as long as data is not concurrently modified. + * or simultaneously, instead of fetching data on each query, MAP stages + * of all those identical queries occurring on the same node may + * reuse the same result - as long as data is not concurrently modified. * - * @param sqlMapThrottle SQL MAP throttling flag. + * @param sqlResultsReuse SQL results reuse flag. * @return {@code this} for chaining. - * */ - public IgniteConfiguration setSqlMapThrottle(boolean sqlMapThrottle) { - this.sqlMapThrottle = sqlMapThrottle; + public IgniteConfiguration setSqlResultsReuse(boolean sqlResultsReuse) { + this.sqlResultsReuse = sqlResultsReuse; return this; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index b837e4207acb8..03c3243787b25 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -132,10 +132,10 @@ public class GridMapQueryExecutor { private IgniteH2Indexing h2; /** - * Identical simultaneous queries throttling flag. - * @see IgniteConfiguration#sqlMapThrottle + * Identical simultaneous queries results reuse flag. + * @see IgniteConfiguration#sqlResultsReuse */ - private boolean throttle; + private boolean reuseResults; /** */ private ConcurrentMap qryRess = new ConcurrentHashMap8<>(); @@ -169,7 +169,7 @@ public void start(final GridKernalContext ctx, IgniteH2Indexing h2) throws Ignit log = ctx.log(GridMapQueryExecutor.class); - throttle = ctx.grid().configuration().isSqlMapThrottle(); + reuseResults = ctx.grid().configuration().isSqlResultsReuse(); final UUID locNodeId = ctx.localNodeId(); @@ -1224,7 +1224,7 @@ private List row(Value[] row) { /** * Execute query specified by {@code key} or reuse result of the same concurrently running query. - * @param key Query key to throttle similar queries as needed. + * @param key Query key to make similar queries reuse the same result as needed. * @param node Cluster node. * @param mainCctx Cache context. * @param conn H2 connection. @@ -1236,7 +1236,7 @@ private List row(Value[] row) { */ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheContext mainCctx, Connection conn, int timeout, GridQueryCancel cancel, boolean evt) throws IgniteCheckedException { - if (!isThrottled(key)) { + if (!isResultReused(key)) { ResultSet rs = executeQuery(node, mainCctx, conn, key.qry, key.params, timeout, cancel, evt); ResultSetWrapper res = new ResultSetWrapper(rs, null, null); @@ -1294,9 +1294,9 @@ private ResultSetWrapper runQuery(QueryKey key, ClusterNode node, GridCacheConte /** * @return {@code true} if this key corresponds to a query whose result may be reused, {@code false} otherwise. */ - private boolean isThrottled(QueryKey key) { - return throttle && - // Throttle the query if we don't have partitions list set, OR if it's what we've got from partitions map. + private boolean isResultReused(QueryKey key) { + return reuseResults && + // Reuse query result if we don't have partitions list set, OR if it's what we've got from partitions map. (key.parts == null || (key.partsMap != null && key.partsMap.get(ctx.localNodeId()) == key.parts)); } diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryThrottlingBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryResultsReuseBenchmark.java similarity index 96% rename from modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryThrottlingBenchmark.java rename to modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryResultsReuseBenchmark.java index 89a27bfd79a95..ed64cf8a0462e 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryThrottlingBenchmark.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryResultsReuseBenchmark.java @@ -31,9 +31,10 @@ import static org.yardstickframework.BenchmarkUtils.println; /** - * Ignite benchmark that performs identical query operations with occasional puts to measure effect of query throttling. + * Ignite benchmark that performs identical query operations with occasional puts to measure effect + * of query results reuse. */ -public class IgniteSqlQueryThrottlingBenchmark extends IgniteCacheAbstractBenchmark { +public class IgniteSqlQueryResultsReuseBenchmark extends IgniteCacheAbstractBenchmark { /** */ private final AtomicInteger putCnt = new AtomicInteger(); From 02908eb059a08a33d7ed50ba7878ba26e0653ef4 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Tue, 27 Jun 2017 18:55:48 +0300 Subject: [PATCH 034/155] IGNITE-5482 IgniteSqlQueryResultsReuseBenchmark - fixed for timed puts. --- .../IgniteSqlQueryResultsReuseBenchmark.java | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryResultsReuseBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryResultsReuseBenchmark.java index ed64cf8a0462e..98fd2e31574e9 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryResultsReuseBenchmark.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryResultsReuseBenchmark.java @@ -19,12 +19,14 @@ import java.util.Collection; import java.util.Map; -import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import javax.cache.Cache; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.query.SqlQuery; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.yardstick.cache.model.Person; import org.yardstickframework.BenchmarkConfiguration; @@ -35,6 +37,12 @@ * of query results reuse. */ public class IgniteSqlQueryResultsReuseBenchmark extends IgniteCacheAbstractBenchmark { + /** Timeout for this client to use to make cache puts. */ + private int putTimeoutMs = DFLT_PUT_TIMEOUT_MS; + + /** Time after which one of this client's threads is allowed to do a cache put. */ + private volatile long nextPutTime; + /** */ private final AtomicInteger putCnt = new AtomicInteger(); @@ -47,10 +55,29 @@ public class IgniteSqlQueryResultsReuseBenchmark extends IgniteCacheAbstractBenc /** Max salary to query. */ private int maxSalary; + /** Flag to synchronize puts (one thread gets to make an actual put within a timeout). */ + private final AtomicBoolean putFlag = new AtomicBoolean(); + + /** Default put timeout. */ + private final static int DFLT_PUT_TIMEOUT_MS = 500; + + /** Property name for put timeout. */ + private final static String PUT_TIMEOUT_PROP_NAME = "PUT_TIMEOUT"; + /** {@inheritDoc} */ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception { super.setUp(cfg); + String timeoutProp = cfg.customProperties().get(PUT_TIMEOUT_PROP_NAME); + + if (!F.isEmpty(timeoutProp)) { + putTimeoutMs = Integer.parseInt(timeoutProp); + + A.ensure(putTimeoutMs > 0, PUT_TIMEOUT_PROP_NAME); + + println(cfg, "Using custom PUT timeout: " + putTimeoutMs); + } + println(cfg, "Populating query data..."); long start = System.nanoTime(); @@ -75,16 +102,25 @@ public class IgniteSqlQueryResultsReuseBenchmark extends IgniteCacheAbstractBenc /** {@inheritDoc} */ @Override public boolean test(Map ctx) throws Exception { - // Let's make puts 20% of all operations. - boolean put = (ThreadLocalRandom.current().nextInt(5) == 0); + // Let's avoid locking by using CAS here. + boolean put = putFlag.compareAndSet(false, true); if (put) { - cache().put(ThreadLocalRandom.current().nextInt(args.range), - person(ThreadLocalRandom.current().nextInt(args.range))); + long curTime = System.currentTimeMillis(); + + if (curTime > nextPutTime) { + cache().put(nextRandom(args.range), person(nextRandom(args.range))); - putCnt.incrementAndGet(); + putCnt.incrementAndGet(); - return true; + nextPutTime = curTime + putTimeoutMs; + + putFlag.set(false); + + return true; + } + else // It's not the time to put yet, let's release the flag for others to try. + putFlag.set(false); } Collection> entries = executeQuery(minSalary, maxSalary); From 3240900902fd302d1d40bf43af5b15be4a0cbef7 Mon Sep 17 00:00:00 2001 From: EdShangGG Date: Thu, 29 Jun 2017 19:13:34 +0300 Subject: [PATCH 035/155] GG-12411 Backport [GG-12339] to 8.1.2 GG-12339 Improve logging of snasphot operation --- .../FinishSnapshotOperationAckDiscoveryMessage.java | 9 +++++++++ .../cache/persistence/GridCacheOffheapManager.java | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java index 1e5ed4256314e..83c512a31479c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java @@ -74,4 +74,13 @@ public IgniteUuid operationId() { public boolean success() { return success; } + + /** {@inheritDoc} */ + @Override public String toString() { + return "FinishSnapshotOperationAckDiscoveryMessage{" + + "id=" + id + + ", opId=" + opId + + ", success=" + success + + '}'; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 4e322b96a3dce..84b1efe76815b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -318,6 +318,10 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav io.setNextSnapshotTag(metaPageAddr, nextSnapshotTag + 1); + if (log != null && log.isDebugEnabled()) + log.debug("Save next snapshot before checkpoint start for grId = " + grpId + + ", nextSnapshotTag = " + nextSnapshotTag); + if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaPageId, metaPage, wal, null)) wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId, From 68c0281dc0f1784f5de2c8ca68c7c2a89f93c397 Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Fri, 30 Jun 2017 18:35:50 +0300 Subject: [PATCH 036/155] GG-12407: Backport [GG-12364] to 8.1.2 --- .../persistence/GridCacheOffheapManager.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 84b1efe76815b..a890fa126032b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -186,22 +186,22 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav IgniteWriteAheadLogManager wal = this.ctx.wal(); if (size > 0 || updCntr > 0) { - int state = -1; + GridDhtPartitionState state = null; if (!grp.isLocal()) { if (beforeDestroy) - state = GridDhtPartitionState.EVICTED.ordinal(); + state = GridDhtPartitionState.EVICTED; else { // localPartition will not acquire writeLock here because create=false. GridDhtLocalPartition part = grp.topology().localPartition(store.partId(), AffinityTopologyVersion.NONE, false); if (part != null && part.state() != GridDhtPartitionState.EVICTED) - state = part.state().ordinal(); + state = part.state(); } // Do not save meta for evicted partitions on next checkpoints. - if (state == -1) + if (state == null) return false; } @@ -229,8 +229,8 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav changed |= io.setGlobalRemoveId(pageAddr, rmvId); changed |= io.setSize(pageAddr, size); - if (state != -1) - changed |= io.setPartitionState(pageAddr, (byte)state); + if (state != null) + changed |= io.setPartitionState(pageAddr, (byte)state.ordinal()); else assert grp.isLocal() : grp.cacheOrGroupName(); @@ -327,8 +327,9 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId, nextSnapshotTag + 1)); - addPartition(ctx.partitionStatMap(), metaPageAddr, io, grpId, PageIdAllocator.INDEX_PARTITION, - this.ctx.kernalContext().cache().context().pageStore().pages(grpId, PageIdAllocator.INDEX_PARTITION)); + if (state == GridDhtPartitionState.OWNING) + addPartition(ctx.partitionStatMap(), metaPageAddr, io, grpId, PageIdAllocator.INDEX_PARTITION, + this.ctx.kernalContext().cache().context().pageStore().pages(grpId, PageIdAllocator.INDEX_PARTITION)); } finally { pageMem.writeUnlock(grpId, metaPageId, metaPage, null, true); @@ -361,7 +362,7 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav rmvId, size, cntrsPageId, - (byte)state, + (byte)state.ordinal(), pageCnt )); } From f4f8562e8b61e09e14aad61d0d5cf98a2c896101 Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Fri, 30 Jun 2017 20:47:45 +0300 Subject: [PATCH 037/155] GG-12407: Backport [GG-12364] to 8.1.2 - skipCrc flag added --- .../snapshot/SnapshotCheckParameters.java | 71 +++++++++++++++++++ .../pagemem/snapshot/SnapshotOperation.java | 26 +++++-- 2 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java new file mode 100644 index 0000000000000..e95e79d138148 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java @@ -0,0 +1,71 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.ignite.internal.pagemem.snapshot; + +import java.io.File; +import java.io.Serializable; +import java.util.Collection; +import org.jetbrains.annotations.Nullable; + +/** + * Tuple for passing optional parameters of {@link SnapshotOperationType#CHECK}. + */ +public class SnapshotCheckParameters implements Serializable { + /** Optional paths. */ + private final Collection optionalPaths; + + /** Flag for skipping CRC check. */ + private final boolean skipCrc; + + /** + * Factory method. + * + * @return Tuple with optional parameters or null if parameters are default. + * + * @param optionalPaths Optional paths. + * @param skipCrc Skip crc. + */ + @Nullable public static SnapshotCheckParameters valueOf(Collection optionalPaths, boolean skipCrc) { + if (optionalPaths == null && !skipCrc) + return null; + + return new SnapshotCheckParameters(optionalPaths, skipCrc); + } + + /** + * @param optionalPaths Optional paths. + * @param skipCrc Flag for skipping CRC check. + */ + private SnapshotCheckParameters(Collection optionalPaths, boolean skipCrc) { + this.optionalPaths = optionalPaths; + this.skipCrc = skipCrc; + } + + /** + * @return Optional paths. + */ + public Collection optionalPaths() { + return optionalPaths; + } + + /** + * @return Flag for skipping CRC check. + */ + public boolean skipCrc() { + return skipCrc; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java index 2fe9f45ddad36..863107a3ac618 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java @@ -134,14 +134,32 @@ public Set dependentSnapshotIds() { * @param op Op. */ public static Collection getOptionalPathsParameter(SnapshotOperation op) { - assert (op.type() == SnapshotOperationType.CHECK || - op.type() == SnapshotOperationType.RESTORE || - op.type() == SnapshotOperationType.RESTORE_2_PHASE) - && (op.extraParameter() == null || op.extraParameter() instanceof Collection); + assert (op.type() == SnapshotOperationType.RESTORE || + op.type() == SnapshotOperationType.RESTORE_2_PHASE) + && (op.extraParameter() == null || op.extraParameter() instanceof Collection) + || (op.type() == SnapshotOperationType.CHECK && + (op.extraParameter() == null || op.extraParameter() instanceof SnapshotCheckParameters)); + + if (op.type() == SnapshotOperationType.CHECK) { + if (op.extraParameter() == null) + return null; + else + return ((SnapshotCheckParameters)op.extraParameter()).optionalPaths(); + } return (Collection)op.extraParameter(); } + /** + * @param op Op. + */ + public static boolean getSkipCrcParameter(SnapshotOperation op) { + assert op.type() == SnapshotOperationType.CHECK && + (op.extraParameter() == null | op.extraParameter() instanceof SnapshotCheckParameters); + + return op.extraParameter() != null && ((SnapshotCheckParameters)op.extraParameter()).skipCrc(); + } + /** * @param op Op. */ From fbf9597570211209f968d802e91efd997acf49e6 Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Mon, 3 Jul 2017 18:01:14 +0300 Subject: [PATCH 038/155] GG-12408: Backport [GG-12385] and reproducing tests to 8.1.2 --- .../processors/cache/GridCacheProcessor.java | 20 ++++++++++++++++--- .../service/GridServiceProcessor.java | 9 ++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index c425bfbcd6fa9..cbbfca0141532 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -2114,9 +2114,12 @@ public void onExchangeDone( ) { initCacheProxies(topVer, err); - if (exchActions != null && exchActions.systemCachesStarting() && exchActions.newClusterState() == null) + if (exchActions != null && exchActions.systemCachesStarting() && exchActions.newClusterState() == null) { ctx.dataStructures().restoreStructuresState(ctx); + ctx.service().updateUtilityCache(); + } + if (exchActions != null && err == null) { Collection> stoppedGrps = null; @@ -2601,7 +2604,7 @@ private IgniteInternalFuture startClientCacheChange( */ public IgniteInternalFuture dynamicStartCaches(Collection ccfgList, boolean failIfExists, boolean checkThreadTx) { - return dynamicStartCaches(ccfgList, CacheType.USER, failIfExists, checkThreadTx); + return dynamicStartCaches(ccfgList, null, failIfExists, checkThreadTx); } /** @@ -2627,11 +2630,22 @@ private IgniteInternalFuture dynamicStartCaches( try { for (CacheConfiguration ccfg : ccfgList) { + CacheType ct = cacheType; + + if (ct == null) { + if (CU.isUtilityCache(ccfg.getName())) + ct = CacheType.UTILITY; + else if (internalCaches.contains(ccfg.getName())) + ct = CacheType.INTERNAL; + else + ct = CacheType.USER; + } + DynamicCacheChangeRequest req = prepareCacheChangeRequest( ccfg, ccfg.getName(), null, - cacheType, + ct, false, failIfExists, true diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java index 12be63b283bb0..3e77d2a2b338a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java @@ -224,7 +224,7 @@ CU.UTILITY_CACHE_NAME, new ServiceEntriesListener(), null, null * @throws IgniteCheckedException If failed. */ private void onKernalStart0() throws IgniteCheckedException { - cache = ctx.cache().utilityCache(); + updateUtilityCache(); if (!ctx.clientNode()) ctx.event().addDiscoveryEventListener(topLsnr, EVTS); @@ -285,6 +285,13 @@ private void onKernalStart0() throws IgniteCheckedException { log.debug("Started service processor."); } + /** + * + */ + public void updateUtilityCache() { + cache = ctx.cache().utilityCache(); + } + /** {@inheritDoc} */ @Override public void onKernalStop(boolean cancel) { if (ctx.isDaemon()) From a5abdd579cc7ace5d17ebcc6555e3ce66c777917 Mon Sep 17 00:00:00 2001 From: Vasiliy Sisko Date: Tue, 4 Jul 2017 18:44:42 +0700 Subject: [PATCH 039/155] ignite-2.1 Fixed minor issues and typos. --- .../visor/cache/VisorCacheConfiguration.java | 11 +--- .../visor/node/VisorBasicConfiguration.java | 2 +- .../node/VisorMemoryPolicyConfiguration.java | 2 +- .../states/configuration/caches/affinity.pug | 18 +++---- .../states/configuration/caches/general.pug | 50 +++++++++---------- 5 files changed, 38 insertions(+), 45 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java index 7dba8299aaa96..6fe056c59164b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java @@ -319,13 +319,6 @@ public boolean isEagerTtl() { return eagerTtl; } - /** - * @return Default lock acquisition timeout. - */ - public long getDfltLockTimeout() { - return dfltLockTimeout; - } - /** * @return {@code true} if cache statistics collection enabled. */ @@ -455,7 +448,7 @@ public String getEvictionFilter() { /** * @return Listener configurations. */ - public String getLsnrConfigurations() { + public String getListenerConfigurations() { return lsnrConfigurations; } @@ -476,7 +469,7 @@ public String getMemoryPolicyName() { /** * @return Maximum payload size for offheap indexes. */ - public int getSqlIdxMaxInlineSize() { + public int getSqlIndexMaxInlineSize() { return sqlIdxMaxInlineSize; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorBasicConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorBasicConfiguration.java index 56d000d1b5f5e..b32e814698b93 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorBasicConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorBasicConfiguration.java @@ -195,7 +195,7 @@ public VisorBasicConfiguration(IgniteEx ignite, IgniteConfiguration c) { addrRslvr = compactClass(c.getAddressResolver()); cacheSanityCheckEnabled = c.isCacheSanityCheckEnabled(); clsLdr = compactClass(c.getClassLoader()); - consistentId = String.valueOf(c.getConsistentId()); + consistentId = c.getConsistentId() != null ? String.valueOf(c.getConsistentId()) : null; failureDetectionTimeout = c.getFailureDetectionTimeout(); igniteWorkDir = c.getWorkDirectory(); lateAffAssignment = c.isLateAffinityAssignment(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryPolicyConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryPolicyConfiguration.java index ced312468503d..d3153a6c669b7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryPolicyConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryPolicyConfiguration.java @@ -98,7 +98,7 @@ public long getMaxSize() { /** * Initial memory region size defined by this memory policy. */ - public long getInitSize() { + public long getInitialSize() { return initSize; } diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug index cb333acb44c89..bc7674f0bca25 100644 --- a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug +++ b/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug @@ -40,20 +40,20 @@ include /app/helpers/jade/mixins .col-sm-6 .settings-row(ng-if='$ctrl.available(["1.0.0", "2.0.0"])') +dropdown('Function:', `${affModel}.kind`, '"AffinityKind"', 'true', 'Default', 'affinityFunction', - 'Key topology resolver to provide mapping from keys to nodes\ + 'Key topology resolver to provide mapping from keys to nodes
\

    \ -
  • Rendezvous - Based on Highest Random Weight algorithm
    • \ -
  • Fair - Tries to ensure that all nodes get equal number of partitions with minimum amount of reassignments between existing nodes
    • \ -
  • Custom - Custom implementation of key affinity fynction
    • \ -
  • Default - By default rendezvous affinity function with 1024 partitions is used
    • \ +
  • Rendezvous - Based on Highest Random Weight algorithm
    • \ +
  • Fair - Tries to ensure that all nodes get equal number of partitions with minimum amount of reassignments between existing nodes
    • \ +
  • Custom - Custom implementation of key affinity fynction
    • \ +
  • Default - By default rendezvous affinity function with 1024 partitions is used
    • \
') .settings-row(ng-if='$ctrl.available("2.0.0")') +dropdown('Function:', `${affModel}.kind`, '"AffinityKind"', 'true', 'Default', 'affinityFunction', - 'Key topology resolver to provide mapping from keys to nodes\ + 'Key topology resolver to provide mapping from keys to nodes
\
    \ -
  • Rendezvous - Based on Highest Random Weight algorithm
    • \ -
  • Custom - Custom implementation of key affinity fynction
    • \ -
  • Default - By default rendezvous affinity function with 1024 partitions is used
    • \ +
  • Rendezvous - Based on Highest Random Weight algorithm
    • \ +
  • Custom - Custom implementation of key affinity fynction
    • \ +
  • Default - By default rendezvous affinity function with 1024 partitions is used
    • \
') .panel-details(ng-if=rendezvousAff) .details-row diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/general.pug b/modules/web-console/frontend/app/modules/states/configuration/caches/general.pug index 33e62b87d3982..87c67f55060dd 100644 --- a/modules/web-console/frontend/app/modules/states/configuration/caches/general.pug +++ b/modules/web-console/frontend/app/modules/states/configuration/caches/general.pug @@ -82,31 +82,31 @@ include /app/helpers/jade/mixins 'Flag indicating whether data can be read from backup
\ If not set then always get data from primary node (never from backup)') //- Since ignite 2.0 - .settings-row(ng-if='$ctrl.available("2.0.0")') - +dropdown('Partition loss policy:', `${model}.partitionLossPolicy`, '"partitionLossPolicy"', 'true', 'IGNORE', - '[\ - {value: "READ_ONLY_SAFE", label: "READ_ONLY_SAFE"},\ - {value: "READ_ONLY_ALL", label: "READ_ONLY_ALL"},\ - {value: "READ_WRITE_SAFE", label: "READ_WRITE_SAFE"},\ - {value: "READ_WRITE_ALL", label: "READ_WRITE_ALL"},\ - {value: "IGNORE", label: "IGNORE"}\ - ]', - 'Partition loss policies:\ -
    \ -
  • READ_ONLY_SAFE - in this mode all writes to the cache will be failed with an exception,\ - reads will only be allowed for keys in non-lost partitions.\ - Reads from lost partitions will be failed with an exception.
    • \ -
  • READ_ONLY_ALL - in this mode Ñ„ll writes to the cache will be failed with an exception.\ - All reads will proceed as if all partitions were in a consistent state.\ - The result of reading from a lost partition is undefined and may be different on different nodes in the cluster.
    • \ -
  • READ_WRITE_SAFE - in this mode Aall reads and writes will be allowed for keys in valid partitions.\ - All reads and writes for keys in lost partitions will be failed with an exception.
    • \ -
  • READ_WRITE_ALL - in this mode all reads and writes will proceed as if all partitions were in a consistent state.\ - The result of reading from a lost partition is undefined and may be different on different nodes in the cluster.
    • \ -
  • IGNORE - in this mode if partition is lost, reset it state and do not clear intermediate data.\ - The result of reading from a previously lost and not cleared partition is undefined and may be different\ - on different nodes in the cluster.
    • \ -
') + .settings-row(ng-if='$ctrl.available("2.0.0")') + +dropdown('Partition loss policy:', `${model}.partitionLossPolicy`, '"partitionLossPolicy"', 'true', 'IGNORE', + '[\ + {value: "READ_ONLY_SAFE", label: "READ_ONLY_SAFE"},\ + {value: "READ_ONLY_ALL", label: "READ_ONLY_ALL"},\ + {value: "READ_WRITE_SAFE", label: "READ_WRITE_SAFE"},\ + {value: "READ_WRITE_ALL", label: "READ_WRITE_ALL"},\ + {value: "IGNORE", label: "IGNORE"}\ + ]', + 'Partition loss policies:\ +
    \ +
  • READ_ONLY_SAFE - in this mode all writes to the cache will be failed with an exception,\ + reads will only be allowed for keys in non-lost partitions.\ + Reads from lost partitions will be failed with an exception.
    • \ +
  • READ_ONLY_ALL - in this mode Ñ„ll writes to the cache will be failed with an exception.\ + All reads will proceed as if all partitions were in a consistent state.\ + The result of reading from a lost partition is undefined and may be different on different nodes in the cluster.
    • \ +
  • READ_WRITE_SAFE - in this mode Aall reads and writes will be allowed for keys in valid partitions.\ + All reads and writes for keys in lost partitions will be failed with an exception.
    • \ +
  • READ_WRITE_ALL - in this mode all reads and writes will proceed as if all partitions were in a consistent state.\ + The result of reading from a lost partition is undefined and may be different on different nodes in the cluster.
    • \ +
  • IGNORE - in this mode if partition is lost, reset it state and do not clear intermediate data.\ + The result of reading from a previously lost and not cleared partition is undefined and may be different\ + on different nodes in the cluster.
    • \ +
') .settings-row +checkbox('Copy on read', `${model}.copyOnRead`, '"copyOnRead"', 'Flag indicating whether copy of the value stored in cache should be created for cache operation implying return value
\ From c475f2765ca557191dcbff67a070d458143203b0 Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Tue, 4 Jul 2017 14:58:16 +0300 Subject: [PATCH 040/155] GG-12416: Backport [GG-12402] to 8.1.2 --- .../cache/persistence/DbCheckpointListener.java | 2 ++ .../persistence/GridCacheDatabaseSharedManager.java | 4 ++++ .../cache/persistence/GridCacheOffheapManager.java | 10 +++++----- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java index f4da637947031..daaccff5fc78a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java @@ -29,6 +29,8 @@ public interface Context { public boolean nextSnapshot(); public Map, T2> partitionStatMap(); + + public boolean needToSnapshot(String cacheOrGrpName); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 990f54c2a1a35..3b3932e6effa8 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -2155,6 +2155,10 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws @Override public Map, T2> partitionStatMap() { return map; } + + @Override public boolean needToSnapshot(String cacheOrGrpName) { + return curr.snapshotOperation.cacheGroupIds().contains(CU.cacheId(cacheOrGrpName)); + } }; // Listeners must be invoked before we write checkpoint record to WAL. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index a890fa126032b..ed008be423fa6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -47,6 +47,9 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl; import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator; import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; @@ -56,9 +59,6 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseListImpl; import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.lang.GridCursor; @@ -169,7 +169,7 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav boolean beforeDestroy) throws IgniteCheckedException { RowStore rowStore0 = store.rowStore(); - boolean beforeSnapshot = ctx != null && ctx.nextSnapshot(); + boolean needSnapshot = ctx != null && ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName()); boolean wasSaveToMeta = false; @@ -302,7 +302,7 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav int pageCnt; - if (beforeSnapshot) { + if (needSnapshot) { pageCnt = this.ctx.pageStore().pages(grpId, store.partId()); io.setCandidatePageCount(pageAddr, pageCnt); From 6c9d2228d2aba6ba5be64bd20ef244e634aeaa5a Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 4 Jul 2017 19:30:46 +0700 Subject: [PATCH 041/155] ignite-2.1 Added persistent store metrics. --- .../visor/node/VisorNodeDataCollectorJob.java | 32 ++- .../node/VisorNodeDataCollectorJobResult.java | 73 ++++-- .../node/VisorNodeDataCollectorTask.java | 14 +- .../VisorNodeDataCollectorTaskResult.java | 28 ++- .../visor/node/VisorPersistenceMetrics.java | 214 ++++++++++++++++++ .../visor/service/VisorServiceTask.java | 10 +- .../frontend/app/filters/duration.filter.js | 2 +- 7 files changed, 341 insertions(+), 32 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistenceMetrics.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java index f5ea5670484ba..3fd7b0d9eb1bd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java @@ -39,6 +39,7 @@ import org.apache.ignite.internal.visor.compute.VisorComputeMonitoringHolder; import org.apache.ignite.internal.visor.igfs.VisorIgfs; import org.apache.ignite.internal.visor.igfs.VisorIgfsEndpoint; +import org.apache.ignite.internal.visor.util.VisorExceptionWrapper; import org.apache.ignite.lang.IgniteProductVersion; import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isIgfsCache; @@ -119,7 +120,7 @@ protected void events(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto events0(res, arg.getEventsOrderKey(), arg.getEventsThrottleCounterKey(), arg.isTaskMonitoringEnabled()); } catch (Exception e) { - res.setEventsEx(e); + res.setEventsEx(new VisorExceptionWrapper(e)); } } @@ -160,9 +161,9 @@ protected void memoryMetrics(VisorNodeDataCollectorJobResult res) { memoryMetrics.add(new VisorMemoryMetrics(m)); } catch (Exception e) { - res.setMemoryMetricsEx(e); + res.setMemoryMetricsEx(new VisorExceptionWrapper(e)); } -} + } /** * Collect caches. @@ -205,7 +206,7 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto } } catch (Exception e) { - res.setCachesEx(e); + res.setCachesEx(new VisorExceptionWrapper(e)); } } @@ -245,7 +246,21 @@ protected void igfs(VisorNodeDataCollectorJobResult res) { } } catch (Exception e) { - res.setIgfssEx(e); + res.setIgfssEx(new VisorExceptionWrapper(e)); + } + } + + /** + * Collect persistence metrics. + * + * @param res Job result. + */ + protected void persistenceMetrics(VisorNodeDataCollectorJobResult res) { + try { + res.setPersistenceMetrics(new VisorPersistenceMetrics(ignite.persistentStoreMetrics())); + } + catch (Exception e) { + res.setPersistenceMetricsEx(new VisorExceptionWrapper(e)); } } @@ -292,7 +307,12 @@ protected VisorNodeDataCollectorJobResult run(VisorNodeDataCollectorJobResult re igfs(res); if (debug) - log(ignite.log(), "Collected igfs", getClass(), start0); + start0 = log(ignite.log(), "Collected igfs", getClass(), start0); + + persistenceMetrics(res); + + if (debug) + log(ignite.log(), "Collected persistence metrics", getClass(), start0); res.setErrorCount(ignite.context().exceptionRegistry().errorCount()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java index ce4f9fcbdf72d..90ecf6e68d97f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.visor.event.VisorGridEvent; import org.apache.ignite.internal.visor.igfs.VisorIgfs; import org.apache.ignite.internal.visor.igfs.VisorIgfsEndpoint; +import org.apache.ignite.internal.visor.util.VisorExceptionWrapper; /** * Data collector job result. @@ -51,19 +52,19 @@ public class VisorNodeDataCollectorJobResult extends VisorDataTransferObject { private List evts = new ArrayList<>(); /** Exception while collecting node events. */ - private Throwable evtsEx; + private VisorExceptionWrapper evtsEx; /** Node memory metrics. */ private List memoryMetrics = new ArrayList<>(); /** Exception while collecting memory metrics. */ - private Throwable memoryMetricsEx; + private VisorExceptionWrapper memoryMetricsEx; /** Node caches. */ private List caches = new ArrayList<>(); /** Exception while collecting node caches. */ - private Throwable cachesEx; + private VisorExceptionWrapper cachesEx; /** Node IGFSs. */ private List igfss = new ArrayList<>(); @@ -72,7 +73,7 @@ public class VisorNodeDataCollectorJobResult extends VisorDataTransferObject { private List igfsEndpoints = new ArrayList<>(); /** Exception while collecting node IGFSs. */ - private Throwable igfssEx; + private VisorExceptionWrapper igfssEx; /** Errors count. */ private long errCnt; @@ -83,6 +84,12 @@ public class VisorNodeDataCollectorJobResult extends VisorDataTransferObject { /** Whether pending exchange future exists. */ private boolean hasPendingExchange; + /** Persistence metrics. */ + private VisorPersistenceMetrics persistenceMetrics; + + /** Exception while collecting persistence metrics. */ + private VisorExceptionWrapper persistenceMetricsEx; + /** * Default constructor. */ @@ -142,14 +149,14 @@ public List getEvents() { /** * @return Exception caught during collecting events. */ - public Throwable getEventsEx() { + public VisorExceptionWrapper getEventsEx() { return evtsEx; } /** * @param evtsEx Exception caught during collecting events. */ - public void setEventsEx(Throwable evtsEx) { + public void setEventsEx(VisorExceptionWrapper evtsEx) { this.evtsEx = evtsEx; } @@ -163,14 +170,14 @@ public List getMemoryMetrics() { /** * @return Exception caught during collecting memory metrics. */ - public Throwable getMemoryMetricsEx() { + public VisorExceptionWrapper getMemoryMetricsEx() { return memoryMetricsEx; } /** * @param memoryMetricsEx Exception caught during collecting memory metrics. */ - public void setMemoryMetricsEx(Throwable memoryMetricsEx) { + public void setMemoryMetricsEx(VisorExceptionWrapper memoryMetricsEx) { this.memoryMetricsEx = memoryMetricsEx; } @@ -184,14 +191,14 @@ public List getCaches() { /** * @return Exception caught during collecting caches metrics. */ - public Throwable getCachesEx() { + public VisorExceptionWrapper getCachesEx() { return cachesEx; } /** * @param cachesEx Exception caught during collecting caches metrics. */ - public void setCachesEx(Throwable cachesEx) { + public void setCachesEx(VisorExceptionWrapper cachesEx) { this.cachesEx = cachesEx; } @@ -212,14 +219,14 @@ public List getIgfsEndpoints() { /** * @return Exception caught during collecting IGFSs metrics. */ - public Throwable getIgfssEx() { + public VisorExceptionWrapper getIgfssEx() { return igfssEx; } /** * @param igfssEx Exception caught during collecting IGFSs metrics. */ - public void setIgfssEx(Throwable igfssEx) { + public void setIgfssEx(VisorExceptionWrapper igfssEx) { this.igfssEx = igfssEx; } @@ -265,6 +272,36 @@ public void setHasPendingExchange(boolean hasPendingExchange) { this.hasPendingExchange = hasPendingExchange; } + /** + * Get persistence metrics. + */ + public VisorPersistenceMetrics getPersistenceMetrics() { + return persistenceMetrics; + } + + /** + * Set persistence metrics. + * + * @param persistenceMetrics Persistence metrics. + */ + public void setPersistenceMetrics(VisorPersistenceMetrics persistenceMetrics) { + this.persistenceMetrics = persistenceMetrics; + } + + /** + * @return Exception caught during collecting persistence metrics. + */ + public VisorExceptionWrapper getPersistenceMetricsEx() { + return persistenceMetricsEx; + } + + /** + * @param persistenceMetricsEx Exception caught during collecting persistence metrics. + */ + public void setPersistenceMetricsEx(VisorExceptionWrapper persistenceMetricsEx) { + this.persistenceMetricsEx = persistenceMetricsEx; + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeString(out, gridName); @@ -282,6 +319,8 @@ public void setHasPendingExchange(boolean hasPendingExchange) { out.writeLong(errCnt); out.writeObject(readyTopVer); out.writeBoolean(hasPendingExchange); + out.writeObject(persistenceMetrics); + out.writeObject(persistenceMetricsEx); } /** {@inheritDoc} */ @@ -290,17 +329,19 @@ public void setHasPendingExchange(boolean hasPendingExchange) { topVer = in.readLong(); taskMonitoringEnabled = in.readBoolean(); evts = U.readList(in); - evtsEx = (Throwable)in.readObject(); + evtsEx = (VisorExceptionWrapper)in.readObject(); memoryMetrics = U.readList(in); - memoryMetricsEx = (Throwable)in.readObject(); + memoryMetricsEx = (VisorExceptionWrapper)in.readObject(); caches = U.readList(in); - cachesEx = (Throwable)in.readObject(); + cachesEx = (VisorExceptionWrapper)in.readObject(); igfss = U.readList(in); igfsEndpoints = U.readList(in); - igfssEx = (Throwable)in.readObject(); + igfssEx = (VisorExceptionWrapper)in.readObject(); errCnt = in.readLong(); readyTopVer = (VisorAffinityTopologyVersion)in.readObject(); hasPendingExchange = in.readBoolean(); + persistenceMetrics = (VisorPersistenceMetrics)in.readObject(); + persistenceMetricsEx = (VisorExceptionWrapper)in.readObject(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java index 56b3718bb2f4b..80664a1d08ecb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java @@ -97,19 +97,19 @@ protected void reduceJobResult(VisorNodeDataCollectorTaskResult taskRes, taskRes.getEvents().addAll(jobRes.getEvents()); if (jobRes.getEventsEx() != null) - taskRes.getEventsEx().put(nid, new VisorExceptionWrapper(jobRes.getEventsEx())); + taskRes.getEventsEx().put(nid, jobRes.getEventsEx()); if (!jobRes.getMemoryMetrics().isEmpty()) taskRes.getMemoryMetrics().put(nid, jobRes.getMemoryMetrics()); if (jobRes.getMemoryMetricsEx() != null) - taskRes.getMemoryMetricsEx().put(nid, new VisorExceptionWrapper(jobRes.getMemoryMetricsEx())); + taskRes.getMemoryMetricsEx().put(nid, jobRes.getMemoryMetricsEx()); if (!jobRes.getCaches().isEmpty()) taskRes.getCaches().put(nid, jobRes.getCaches()); if (jobRes.getCachesEx() != null) - taskRes.getCachesEx().put(nid, new VisorExceptionWrapper(jobRes.getCachesEx())); + taskRes.getCachesEx().put(nid, jobRes.getCachesEx()); if (!jobRes.getIgfss().isEmpty()) taskRes.getIgfss().put(nid, jobRes.getIgfss()); @@ -118,7 +118,13 @@ protected void reduceJobResult(VisorNodeDataCollectorTaskResult taskRes, taskRes.getIgfsEndpoints().put(nid, jobRes.getIgfsEndpoints()); if (jobRes.getIgfssEx() != null) - taskRes.getIgfssEx().put(nid, new VisorExceptionWrapper(jobRes.getIgfssEx())); + taskRes.getIgfssEx().put(nid, jobRes.getIgfssEx()); + + if (jobRes.getPersistenceMetrics() != null) + taskRes.getPersistenceMetrics().put(nid, jobRes.getPersistenceMetrics()); + + if (jobRes.getPersistenceMetricsEx() != null) + taskRes.getPersistenceMetricsEx().put(nid, jobRes.getPersistenceMetricsEx()); taskRes.getReadyAffinityVersions().put(nid, jobRes.getReadyAffinityVersion()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java index cef3a29054ce9..093e86769e4f2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java @@ -94,6 +94,12 @@ public class VisorNodeDataCollectorTaskResult extends VisorDataTransferObject { /** Whether pending exchange future exists from nodes. */ private Map pendingExchanges = new HashMap<>(); + /** All persistence metrics collected from nodes. */ + private Map persistenceMetrics = new HashMap<>(); + + /** Exceptions caught during collecting persistence metrics from nodes. */ + private Map persistenceMetricsEx = new HashMap<>(); + /** * Default constructor. */ @@ -120,7 +126,9 @@ public boolean isEmpty() { igfsEndpoints.isEmpty() && igfssEx.isEmpty() && readyTopVers.isEmpty() && - pendingExchanges.isEmpty(); + pendingExchanges.isEmpty() && + persistenceMetrics.isEmpty() && + persistenceMetricsEx.isEmpty(); } /** @@ -249,6 +257,20 @@ public Map getPendingExchanges() { return pendingExchanges; } + /** + * All persistence metrics collected from nodes. + */ + public Map getPersistenceMetrics() { + return persistenceMetrics; + } + + /** + * @return Exceptions caught during collecting persistence metrics from nodes. + */ + public Map getPersistenceMetricsEx() { + return persistenceMetricsEx; + } + /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { out.writeBoolean(active); @@ -268,6 +290,8 @@ public Map getPendingExchanges() { U.writeMap(out, igfssEx); U.writeMap(out, readyTopVers); U.writeMap(out, pendingExchanges); + U.writeMap(out, persistenceMetrics); + U.writeMap(out, persistenceMetricsEx); } /** {@inheritDoc} */ @@ -289,6 +313,8 @@ public Map getPendingExchanges() { igfssEx = U.readMap(in); readyTopVers = U.readMap(in); pendingExchanges = U.readMap(in); + persistenceMetrics = U.readMap(in); + persistenceMetricsEx = U.readMap(in); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistenceMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistenceMetrics.java new file mode 100644 index 0000000000000..c83816151e300 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistenceMetrics.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.visor.node; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.PersistenceMetrics; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.visor.VisorDataTransferObject; + +/** + * DTO object for {@link PersistenceMetrics}. + */ +public class VisorPersistenceMetrics extends VisorDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private float walLoggingRate; + + /** */ + private float walWritingRate; + + /** */ + private int walArchiveSegments; + + /** */ + private float walFsyncTimeAvg; + + /** */ + private long lastCpDuration; + + /** */ + private long lastCpLockWaitDuration; + + /** */ + private long lastCpMmarkDuration; + + /** */ + private long lastCpPagesWriteDuration; + + /** */ + private long lastCpFsyncDuration; + + /** */ + private long lastCpTotalPages; + + /** */ + private long lastCpDataPages; + + /** */ + private long lastCpCowPages; + + /** + * Default constructor. + */ + public VisorPersistenceMetrics() { + // No-op. + } + + /** + * @param metrics Persistence metrics. + */ + public VisorPersistenceMetrics(PersistenceMetrics metrics) { + walLoggingRate = metrics.getWalLoggingRate(); + walWritingRate = metrics.getWalWritingRate(); + walArchiveSegments = metrics.getWalArchiveSegments(); + walFsyncTimeAvg = metrics.getWalFsyncTimeAverage(); + lastCpDuration = metrics.getLastCheckpointingDuration(); + lastCpLockWaitDuration = metrics.getLastCheckpointLockWaitDuration(); + lastCpMmarkDuration = metrics.getLastCheckpointMarkDuration(); + lastCpPagesWriteDuration = metrics.getLastCheckpointPagesWriteDuration(); + lastCpFsyncDuration = metrics.getLastCheckpointFsyncDuration(); + lastCpTotalPages = metrics.getLastCheckpointTotalPagesNumber(); + lastCpDataPages = metrics.getLastCheckpointDataPagesNumber(); + lastCpCowPages = metrics.getLastCheckpointCopiedOnWritePagesNumber(); + } + + /** + * @return Average number of WAL records per second written during the last time interval. + */ + public float getWalLoggingRate() { + return walLoggingRate; + } + + /** + * @return Average number of bytes per second written during the last time interval. + */ + public float getWalWritingRate(){ + return walWritingRate; + } + + /** + * @return Current number of WAL segments in the WAL archive. + */ + public int getWalArchiveSegments(){ + return walArchiveSegments; + } + + /** + * @return Average WAL fsync duration in microseconds over the last time interval. + */ + public float getWalFsyncTimeAverage(){ + return walFsyncTimeAvg; + } + + /** + * @return Total checkpoint duration in milliseconds. + */ + public long getLastCheckpointingDuration(){ + return lastCpDuration; + } + + /** + * @return Checkpoint lock wait time in milliseconds. + */ + public long getLastCheckpointLockWaitDuration(){ + return lastCpLockWaitDuration; + } + + /** + * @return Checkpoint mark duration in milliseconds. + */ + public long getLastCheckpointMarkDuration(){ + return lastCpMmarkDuration; + } + + /** + * @return Checkpoint pages write phase in milliseconds. + */ + public long getLastCheckpointPagesWriteDuration(){ + return lastCpPagesWriteDuration; + } + + /** + * @return Checkpoint fsync time in milliseconds. + */ + public long getLastCheckpointFsyncDuration(){ + return lastCpFsyncDuration; + } + + /** + * @return Total number of pages written during the last checkpoint. + */ + public long getLastCheckpointTotalPagesNumber(){ + return lastCpTotalPages; + } + + /** + * @return Total number of data pages written during the last checkpoint. + */ + public long getLastCheckpointDataPagesNumber(){ + return lastCpDataPages; + } + + /** + * @return Total number of pages copied to a temporary checkpoint buffer during the last checkpoint. + */ + public long getLastCheckpointCopiedOnWritePagesNumber(){ + return lastCpCowPages; + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + out.writeFloat(walLoggingRate); + out.writeFloat(walWritingRate); + out.writeInt(walArchiveSegments); + out.writeFloat(walFsyncTimeAvg); + out.writeLong(lastCpDuration); + out.writeLong(lastCpLockWaitDuration); + out.writeLong(lastCpMmarkDuration); + out.writeLong(lastCpPagesWriteDuration); + out.writeLong(lastCpFsyncDuration); + out.writeLong(lastCpTotalPages); + out.writeLong(lastCpDataPages); + out.writeLong(lastCpCowPages); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + walLoggingRate = in.readFloat(); + walWritingRate = in.readFloat(); + walArchiveSegments = in.readInt(); + walFsyncTimeAvg = in.readFloat(); + lastCpDuration = in.readLong(); + lastCpLockWaitDuration = in.readLong(); + lastCpMmarkDuration = in.readLong(); + lastCpPagesWriteDuration = in.readLong(); + lastCpFsyncDuration = in.readLong(); + lastCpTotalPages = in.readLong(); + lastCpDataPages = in.readLong(); + lastCpCowPages = in.readLong(); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorPersistenceMetrics.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java index 1b3495c5e917d..f2489bc62c1bd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java @@ -57,12 +57,14 @@ protected VisorServiceJob(Void arg, boolean debug) { /** {@inheritDoc} */ @Override protected Collection run(final Void arg) { - Collection services = ignite.services().serviceDescriptors(); + Collection res = new ArrayList<>(); - Collection res = new ArrayList<>(services.size()); + if (ignite.active()) { + Collection services = ignite.services().serviceDescriptors(); - for (ServiceDescriptor srvc: services) - res.add(new VisorServiceDescriptor(srvc)); + for (ServiceDescriptor srvc : services) + res.add(new VisorServiceDescriptor(srvc)); + } return res; } diff --git a/modules/web-console/frontend/app/filters/duration.filter.js b/modules/web-console/frontend/app/filters/duration.filter.js index 770bca3e7f371..46d50d861e536 100644 --- a/modules/web-console/frontend/app/filters/duration.filter.js +++ b/modules/web-console/frontend/app/filters/duration.filter.js @@ -34,7 +34,7 @@ export default [() => { const h = Math.floor((t - d * cd) / ch); const m = Math.floor((t - d * cd - h * ch) / cm); const s = Math.floor((t - d * cd - h * ch - m * cm) / cs); - const ms = t % 1000; + const ms = Math.round(t % 1000); return a(d, 'd') + a(h, 'h') + a(m, 'm') + a(s, 's') + (t < cm ? ms + 'ms' : ''); }; From f39129020999561e9def300b9cc136138ea3698e Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Fri, 16 Jun 2017 19:54:26 +0300 Subject: [PATCH 042/155] IGNITE-5527: Prevent starvation in stripe pool on unstable topology. (cherry picked from commit 97ea507) --- .../processors/cache/GridDeferredAckMessageSender.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java index 89aa725a6caef..a8e822660800b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridDeferredAckMessageSender.java @@ -173,7 +173,8 @@ private DeferredAckMessageBuffer(UUID nodeId) { * @return {@code True} if request was handled, {@code false} if this buffer is filled and cannot be used. */ public boolean add(T ver) { - readLock().lock(); + if(!readLock().tryLock()) + return false; // Here, writeLock is help by another thread and guard is already true. boolean snd = false; From b69f53e0ccf9d3da42a99d4423fb3d8bdd60a7bb Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 4 Jul 2017 17:42:31 +0300 Subject: [PATCH 043/155] IGNITE-5629 .NET: CacheConfiguration copy constructor --- .../utils/PlatformConfigurationUtils.java | 8 +- .../Cache/CacheConfigurationTest.cs | 21 ++++++ .../Cache/Configuration/CacheConfiguration.cs | 75 +++++++++++++++++-- .../Apache.Ignite.Core/IgniteConfiguration.cs | 7 +- 4 files changed, 99 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java index 23106ba1b69c3..92db41acd121a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java @@ -885,8 +885,10 @@ public static void writeCacheConfiguration(BinaryRawWriter writer, CacheConfigur writer.writeInt(cnt); for (CachePluginConfiguration cfg : plugins) { - if (cfg instanceof PlatformCachePluginConfiguration) - writer.writeObject(((PlatformCachePluginConfiguration)cfg).nativeCfg()); + if (cfg instanceof PlatformCachePluginConfiguration) { + writer.writeBoolean(false); // Pure platform plugin. + writer.writeObject(((PlatformCachePluginConfiguration) cfg).nativeCfg()); + } } } } @@ -1317,6 +1319,8 @@ private static PlatformPluginConfigurationClosure pluginConfiguration(final int private static void readCachePluginConfiguration(CacheConfiguration cfg, BinaryRawReader in) { int plugCfgFactoryId = in.readInt(); + in.readInt(); // skip size. + PlatformCachePluginConfigurationClosure plugCfg = cachePluginConfiguration(plugCfgFactoryId); plugCfg.apply(cfg, in); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs index 1f6dbcf627fac..435e65f88a7d4 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs @@ -189,6 +189,27 @@ public void TestCacheStore() Assert.AreEqual(factory.TestProperty, factory0.TestProperty); } + /// + /// Tests the copy constructor. + /// + [Test] + public void TestCopyConstructor() + { + foreach (var cfg in new[] + {new CacheConfiguration(), GetCustomCacheConfiguration(), GetCustomCacheConfiguration2()}) + { + // Check direct copy. + AssertConfigsAreEqual(cfg, cfg); + AssertConfigsAreEqual(cfg, new CacheConfiguration(cfg)); + + // Check copy via Ignite config. + var igniteCfg = new IgniteConfiguration {CacheConfiguration = new[] {cfg}}; + var igniteCfgCopy = new IgniteConfiguration(igniteCfg); + + AssertConfigsAreEqual(cfg, igniteCfgCopy.CacheConfiguration.Single()); + } + } + /// /// Asserts the configuration is default. /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs index ec0ac409c1e7b..f5a517949af96 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs @@ -25,6 +25,7 @@ namespace Apache.Ignite.Core.Cache.Configuration using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; + using System.IO; using System.Linq; using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Cache; @@ -34,11 +35,14 @@ namespace Apache.Ignite.Core.Cache.Configuration using Apache.Ignite.Core.Cache.Expiry; using Apache.Ignite.Core.Cache.Store; using Apache.Ignite.Core.Common; + using Apache.Ignite.Core.Impl; using Apache.Ignite.Core.Impl.Binary; using Apache.Ignite.Core.Impl.Cache.Affinity; using Apache.Ignite.Core.Impl.Cache.Expiry; using Apache.Ignite.Core.Log; using Apache.Ignite.Core.Plugin.Cache; + using BinaryReader = Apache.Ignite.Core.Impl.Binary.BinaryReader; + using BinaryWriter = Apache.Ignite.Core.Impl.Binary.BinaryWriter; /// /// Defines grid cache configuration. @@ -198,11 +202,44 @@ public CacheConfiguration(string name, params QueryEntity[] queryEntities) : thi QueryEntities = queryEntities; } + /// + /// Initializes a new instance of the class, + /// performing a deep copy of specified cache configuration. + /// + /// The other configuration to perfrom deep copy from. + public CacheConfiguration(CacheConfiguration other) + { + if (other != null) + { + using (var stream = IgniteManager.Memory.Allocate().GetStream()) + { + other.Write(BinaryUtils.Marshaller.StartMarshal(stream)); + + stream.SynchronizeOutput(); + stream.Seek(0, SeekOrigin.Begin); + + Read(BinaryUtils.Marshaller.StartUnmarshal(stream)); + } + + // Plugins should be copied directly. + PluginConfigurations = other.PluginConfigurations; + } + } + /// /// Initializes a new instance of the class. /// /// The reader. - internal CacheConfiguration(IBinaryRawReader reader) + internal CacheConfiguration(BinaryReader reader) + { + Read(reader); + } + + /// + /// Reads data into this instance from the specified reader. + /// + /// The reader. + private void Read(BinaryReader reader) { // Make sure system marshaller is used. Debug.Assert(((BinaryReader) reader).Marshaller == BinaryUtils.Marshaller); @@ -244,7 +281,9 @@ internal CacheConfiguration(IBinaryRawReader reader) CacheStoreFactory = reader.ReadObject>(); var count = reader.ReadInt(); - QueryEntities = count == 0 ? null : Enumerable.Range(0, count).Select(x => new QueryEntity(reader)).ToList(); + QueryEntities = count == 0 + ? null + : Enumerable.Range(0, count).Select(x => new QueryEntity(reader)).ToList(); NearConfiguration = reader.ReadBoolean() ? new NearCacheConfiguration(reader) : null; @@ -253,19 +292,35 @@ internal CacheConfiguration(IBinaryRawReader reader) ExpiryPolicyFactory = ExpiryPolicySerializer.ReadPolicyFactory(reader); count = reader.ReadInt(); - PluginConfigurations = count == 0 - ? null - : Enumerable.Range(0, count).Select(x => reader.ReadObject()).ToList(); + + if (count > 0) + { + PluginConfigurations = new List(count); + for (int i = 0; i < count; i++) + { + if (reader.ReadBoolean()) + { + // FactoryId-based plugin: skip. + var size = reader.ReadInt(); + reader.Stream.Seek(size, SeekOrigin.Current); + } + else + { + // Pure .NET plugin. + PluginConfigurations.Add(reader.ReadObject()); + } + } + } } /// /// Writes this instance to the specified writer. /// /// The writer. - internal void Write(IBinaryRawWriter writer) + internal void Write(BinaryWriter writer) { // Make sure system marshaller is used. - Debug.Assert(((BinaryWriter) writer).Marshaller == BinaryUtils.Marshaller); + Debug.Assert(writer.Marshaller == BinaryUtils.Marshaller); writer.WriteInt((int) AtomicityMode); writer.WriteInt(Backups); @@ -344,7 +399,13 @@ internal void Write(IBinaryRawWriter writer) { writer.WriteBoolean(true); writer.WriteInt(cachePlugin.CachePluginConfigurationClosureFactoryId.Value); + + int pos = writer.Stream.Position; + writer.WriteInt(0); // Reserve size. + cachePlugin.WriteBinary(writer); + + writer.Stream.WriteInt(pos, writer.Stream.Position - pos); // Write size. } else { diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs index 81736421edee5..4419e2ecf2205 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs @@ -48,6 +48,7 @@ namespace Apache.Ignite.Core using Apache.Ignite.Core.PersistentStore; using Apache.Ignite.Core.Plugin; using Apache.Ignite.Core.Transactions; + using BinaryReader = Apache.Ignite.Core.Impl.Binary.BinaryReader; using BinaryWriter = Apache.Ignite.Core.Impl.Binary.BinaryWriter; /// @@ -242,7 +243,7 @@ public IgniteConfiguration(IgniteConfiguration configuration) /// /// The binary reader. /// The base configuration. - internal IgniteConfiguration(IBinaryRawReader binaryReader, IgniteConfiguration baseConfig) + internal IgniteConfiguration(BinaryReader binaryReader, IgniteConfiguration baseConfig) { Debug.Assert(binaryReader != null); Debug.Assert(baseConfig != null); @@ -509,7 +510,7 @@ internal void Validate(ILogger log) /// Reads data from specified reader into current instance. /// /// The binary reader. - private void ReadCore(IBinaryRawReader r) + private void ReadCore(BinaryReader r) { // Simple properties _clientMode = r.ReadBooleanNullable(); @@ -630,7 +631,7 @@ private void ReadCore(IBinaryRawReader r) /// Reads data from specified reader into current instance. /// /// The binary reader. - private void Read(IBinaryRawReader binaryReader) + private void Read(BinaryReader binaryReader) { ReadCore(binaryReader); From ace802990822fa9a68ddb9824a159dacedc05901 Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Tue, 4 Jul 2017 17:45:37 +0300 Subject: [PATCH 044/155] Fixed updateCounter when node is removed (backport of 4fedd0fe) --- .../cache/distributed/dht/GridDhtPartitionTopologyImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index a6f1831a6f825..f24dd457d282f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -1934,12 +1934,11 @@ private void removeNode(UUID nodeId) { ClusterNode loc = ctx.localNode(); if (node2part != null) { - if (loc.equals(oldest) && !node2part.nodeId().equals(loc.id())) { - updateSeq.setIfGreater(node2part.updateSequence()); + updateSeq.setIfGreater(node2part.updateSequence()); + if (loc.equals(oldest) && !node2part.nodeId().equals(loc.id())) node2part = new GridDhtPartitionFullMap(loc.id(), loc.order(), updateSeq.incrementAndGet(), node2part, false); - } else node2part = new GridDhtPartitionFullMap(node2part, node2part.updateSequence()); From 2e7adbfbbab62f3b98e0409d2f4ceabad89b4120 Mon Sep 17 00:00:00 2001 From: agura Date: Tue, 4 Jul 2017 16:56:40 +0300 Subject: [PATCH 045/155] ignite-5685 JDBC prepared statement shouldn't clear parameters after execution --- .../jdbc2/JdbcPreparedStatementSelfTest.java | 35 +++++++++++++++++++ .../jdbc/JdbcPreparedStatementSelfTest.java | 35 +++++++++++++++++++ .../JdbcThinPreparedStatementSelfTest.java | 35 +++++++++++++++++++ .../internal/jdbc/JdbcPreparedStatement.java | 6 +--- .../jdbc/thin/JdbcThinPreparedStatement.java | 2 -- .../internal/jdbc2/JdbcPreparedStatement.java | 12 ++----- 6 files changed, 108 insertions(+), 17 deletions(-) diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java index e2939e6aea985..7df0e0298f3c8 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatementSelfTest.java @@ -156,6 +156,41 @@ public class JdbcPreparedStatementSelfTest extends GridCommonAbstractTest { } } + /** + * @throws Exception If failed. + */ + public void testRepeatableUsage() throws Exception { + stmt = conn.prepareStatement("select * from TestObject where id = ?"); + + stmt.setInt(1, 1); + + ResultSet rs = stmt.executeQuery(); + + int cnt = 0; + + while (rs.next()) { + if (cnt == 0) + assertEquals(1, rs.getInt(1)); + + cnt++; + } + + assertEquals(1, cnt); + + cnt = 0; + + rs = stmt.executeQuery(); + + while (rs.next()) { + if (cnt == 0) + assertEquals(1, rs.getInt(1)); + + cnt++; + } + + assertEquals(1, cnt); + } + /** * @throws Exception If failed. */ diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java index 0dfa0fde524ff..384036a2b7e66 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcPreparedStatementSelfTest.java @@ -153,6 +153,41 @@ public class JdbcPreparedStatementSelfTest extends GridCommonAbstractTest { } } + /** + * @throws Exception If failed. + */ + public void testRepeatableUsage() throws Exception { + stmt = conn.prepareStatement("select * from TestObject where id = ?"); + + stmt.setInt(1, 1); + + ResultSet rs = stmt.executeQuery(); + + int cnt = 0; + + while (rs.next()) { + if (cnt == 0) + assertEquals(1, rs.getInt(1)); + + cnt++; + } + + assertEquals(1, cnt); + + cnt = 0; + + rs = stmt.executeQuery(); + + while (rs.next()) { + if (cnt == 0) + assertEquals(1, rs.getInt(1)); + + cnt++; + } + + assertEquals(1, cnt); + } + /** * @throws Exception If failed. */ diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinPreparedStatementSelfTest.java index 6f18c75a7eb15..841a0af5b08ec 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinPreparedStatementSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinPreparedStatementSelfTest.java @@ -161,6 +161,41 @@ public class JdbcThinPreparedStatementSelfTest extends JdbcThinAbstractSelfTest } } + /** + * @throws Exception If failed. + */ + public void testRepeatableUsage() throws Exception { + stmt = conn.prepareStatement(SQL_PART + " where id = ?"); + + stmt.setInt(1, 1); + + ResultSet rs = stmt.executeQuery(); + + int cnt = 0; + + while (rs.next()) { + if (cnt == 0) + assertEquals(1, rs.getInt(1)); + + cnt++; + } + + assertEquals(1, cnt); + + cnt = 0; + + rs = stmt.executeQuery(); + + while (rs.next()) { + if (cnt == 0) + assertEquals(1, rs.getInt(1)); + + cnt++; + } + + assertEquals(1, cnt); + } + /** * @throws Exception If failed. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java index 7e5358becab93..93cda1e4768cd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/JdbcPreparedStatement.java @@ -69,11 +69,7 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat /** {@inheritDoc} */ @Override public ResultSet executeQuery() throws SQLException { - ResultSet rs = executeQuery(sql); - - args = null; - - return rs; + return executeQuery(sql); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java index 49a78b6ec743e..0c78a1365df11 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinPreparedStatement.java @@ -219,8 +219,6 @@ public class JdbcThinPreparedStatement extends JdbcThinStatement implements Prep */ private void executeWithArguments() throws SQLException { execute0(sql, args); - - args = null; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java index 1a66ced12f7e6..16030f7c5c984 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java @@ -73,22 +73,14 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat @Override public ResultSet executeQuery() throws SQLException { ensureNotClosed(); - ResultSet rs = executeQuery(sql); - - args = null; - - return rs; + return executeQuery(sql); } /** {@inheritDoc} */ @Override public int executeUpdate() throws SQLException { ensureNotClosed(); - int res = executeUpdate(sql); - - args = null; - - return res; + return executeUpdate(sql); } /** {@inheritDoc} */ From c02948ba4ce26c74e96b60225c75764ca3ba6ea5 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Tue, 4 Jul 2017 17:57:34 +0300 Subject: [PATCH 046/155] Reverted odd changes. --- ...iteCacheDistributedPartitionQueryNodeRestartsSelfTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java index e81b491cbaa85..852541023ffe4 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java @@ -22,8 +22,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerArray; + import org.apache.ignite.Ignite; -import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; import org.apache.ignite.internal.IgniteInternalFuture; @@ -76,8 +76,6 @@ public void testJoinQueryUnstableTopology() throws Exception { restartStats.incrementAndGet(grid); try { - IgniteEx ggrid = grid(grid); - stopGrid(grid); Thread.sleep(rnd.nextInt(NODE_RESTART_TIME)); From 54084f566dc642afac88f48a5efc3319d70e46fb Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Tue, 4 Jul 2017 18:01:09 +0300 Subject: [PATCH 047/155] Added page type check when saving cache metadata (backport of d38fd8d6) --- .../cache/persistence/GridCacheOffheapManager.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index ed008be423fa6..0bb22502d6ae5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -314,9 +314,11 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav long metaPageAddr = pageMem.writeLock(grpId, metaPageId, metaPage); try { - long nextSnapshotTag = io.getNextSnapshotTag(metaPageAddr); + PageMetaIO metaIo = PageMetaIO.getPageIO(metaPageAddr); - io.setNextSnapshotTag(metaPageAddr, nextSnapshotTag + 1); + long nextSnapshotTag = metaIo.getNextSnapshotTag(metaPageAddr); + + metaIo.setNextSnapshotTag(metaPageAddr, nextSnapshotTag + 1); if (log != null && log.isDebugEnabled()) log.debug("Save next snapshot before checkpoint start for grId = " + grpId @@ -328,7 +330,7 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav nextSnapshotTag + 1)); if (state == GridDhtPartitionState.OWNING) - addPartition(ctx.partitionStatMap(), metaPageAddr, io, grpId, PageIdAllocator.INDEX_PARTITION, + addPartition(ctx.partitionStatMap(), metaPageAddr, metaIo, grpId, PageIdAllocator.INDEX_PARTITION, this.ctx.kernalContext().cache().context().pageStore().pages(grpId, PageIdAllocator.INDEX_PARTITION)); } finally { @@ -407,7 +409,7 @@ private byte[] serializeCacheSizes(Map cacheSizes) { private static void addPartition( Map, T2> map, long pageAddr, - PagePartitionMetaIO io, + PageMetaIO io, int cacheId, int partition, int pages From 9743fa3d711d0f1bbf0fa7ceec65237879f76d0d Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Tue, 4 Jul 2017 18:10:25 +0300 Subject: [PATCH 048/155] Muted hanging tests. --- .../IgniteCommunicationBalanceMultipleConnectionsTest.java | 5 +++++ ...IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java | 5 +++++ .../apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java index e95b1ec0b514c..444f086522a8b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java @@ -25,4 +25,9 @@ public class IgniteCommunicationBalanceMultipleConnectionsTest extends IgniteCom @Override protected int connectionsPerNode() { return 5; } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-5689"); + } } diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java index 5154642b2a62b..edf02794316e8 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java @@ -30,4 +30,9 @@ public class IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest public IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest() { super(DUAL_ASYNC, true); } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-5691"); + } } \ No newline at end of file diff --git a/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java b/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java index 49bb1ac63ed61..5477d4384246d 100644 --- a/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java +++ b/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java @@ -99,6 +99,11 @@ public JavaEmbeddedIgniteRDDSelfTest() { super(false); } + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-5690"); + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); From 114c42e14013fe5b7bd6f186f7db59a533d61f68 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Tue, 4 Jul 2017 19:35:06 +0300 Subject: [PATCH 049/155] Cosmetic changes --- .../processors/cache/ClusterCachesInfo.java | 3 ++- .../processors/cache/GridCacheProcessor.java | 16 +++++++++++----- .../persistence/tree/io/TrackingPageIO.java | 12 ++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java index 4e9dcf1ddc5de..8f124b2b92062 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java @@ -43,6 +43,7 @@ import org.apache.ignite.internal.processors.query.QuerySchema; import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.processors.query.schema.SchemaOperationException; +import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.CU; @@ -74,7 +75,7 @@ class ClusterCachesInfo { private final ConcurrentMap registeredTemplates = new ConcurrentHashMap<>(); /** Caches currently being restarted. */ - private final Collection restartingCaches = new HashSet<>(); + private final Collection restartingCaches = new GridConcurrentHashSet<>(); /** */ private final IgniteLogger log; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 402d8743f2112..0f859eb828e2b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -831,8 +831,6 @@ public Collection cacheGroups() { /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public void onKernalStart() throws IgniteCheckedException { - ClusterNode locNode = ctx.discovery().localNode(); - boolean active = ctx.state().active(); try { @@ -881,7 +879,7 @@ public Collection cacheGroups() { final List syncFuts = new ArrayList<>(caches.size()); sharedCtx.forAllCaches(new CIX1() { - @Override public void applyx(GridCacheContext cctx) throws IgniteCheckedException { + @Override public void applyx(GridCacheContext cctx) { CacheConfiguration cfg = cctx.config(); if (cctx.affinityNode() && @@ -3014,6 +3012,13 @@ else if (rebalanceOrder < 0) return res; } + /** + * Reset restarting caches. + */ + public void resetRestartingCaches(){ + cachesInfo.restartingCaches().clear(); + } + /** * @param node Joining node to validate. * @return Node validation result if there was an issue with the joining node, {@code null} otherwise. @@ -3021,7 +3026,7 @@ else if (rebalanceOrder < 0) private IgniteNodeValidationResult validateRestartingCaches(ClusterNode node) { if (cachesInfo.hasRestartingCaches()) { String msg = "Joining node during caches restart is not allowed [joiningNodeId=" + node.id() + - ", restartingCaches=" + new HashSet(cachesInfo.restartingCaches()) + ']'; + ", restartingCaches=" + new HashSet<>(cachesInfo.restartingCaches()) + ']'; return new IgniteNodeValidationResult(node.id(), msg, msg); } @@ -3984,7 +3989,8 @@ private class RemovedItemsCleanupTask implements GridTimeoutObject { */ RemovedItemsCleanupTask(long timeout) { this.timeout = timeout; - this.endTime = U.currentTimeMillis() + timeout; + + endTime = U.currentTimeMillis() + timeout; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java index bbf452da51a25..22631305587ce 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java @@ -110,12 +110,12 @@ public boolean markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, lo /** * @param buf Buffer. * @param nextSnapshotTag Next snapshot id. - * @param lastSuccessfulSnapshotId Last successful snapshot id. + * @param lastSuccessfulSnapshotTag Last successful snapshot id. * @param pageSize Page size. */ - private void validateSnapshotId(ByteBuffer buf, long nextSnapshotTag, long lastSuccessfulSnapshotId, int pageSize) { - assert nextSnapshotTag != lastSuccessfulSnapshotId : "nextSnapshotTag = " + nextSnapshotTag + - ", lastSuccessfulSnapshotId = " + lastSuccessfulSnapshotId; + private void validateSnapshotId(ByteBuffer buf, long nextSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) { + assert nextSnapshotTag != lastSuccessfulSnapshotTag : "nextSnapshotTag = " + nextSnapshotTag + + ", lastSuccessfulSnapshotId = " + lastSuccessfulSnapshotTag; long last = getLastSnapshotTag(buf); @@ -126,7 +126,7 @@ private void validateSnapshotId(ByteBuffer buf, long nextSnapshotTag, long lastS int cntOfPage = countOfPageToTrack(pageSize); - if (last <= lastSuccessfulSnapshotId) { //we can drop our data + if (last <= lastSuccessfulSnapshotTag) { //we can drop our data buf.putLong(LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag); PageHandler.zeroMemory(buf, SIZE_FIELD_OFFSET, buf.capacity() - SIZE_FIELD_OFFSET); @@ -136,7 +136,7 @@ private void validateSnapshotId(ByteBuffer buf, long nextSnapshotTag, long lastS int sizeOff = useLeftHalf(nextSnapshotTag) ? SIZE_FIELD_OFFSET : BITMAP_OFFSET + len; int sizeOff2 = !useLeftHalf(nextSnapshotTag) ? SIZE_FIELD_OFFSET : BITMAP_OFFSET + len; - if (last - lastSuccessfulSnapshotId == 1) { //we should keep only data in last half + if (last - lastSuccessfulSnapshotTag == 1) { //we should keep only data in last half //new data will be written in the same half, we should move old data to another half if ((nextSnapshotTag - last) % 2 == 0) PageHandler.copyMemory(buf, sizeOff, buf, sizeOff2, len + SIZE_FIELD_SIZE); From 77c5dc7455b9bf020dee549cc91286a897dbfdbc Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Tue, 4 Jul 2017 19:40:21 +0300 Subject: [PATCH 050/155] Added onCacheGroupStopped callback --- .../cache/persistence/GridCacheDatabaseSharedManager.java | 6 +++++- .../cache/persistence/IgniteCacheSnapshotManager.java | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 990f54c2a1a35..b3ab1cd4b0b9e 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -821,7 +821,11 @@ private void shutdownCheckpointer(boolean cancel) { Map> destroyed = new HashMap<>(); for (IgniteBiTuple tup : stoppedGrps) { - PageMemoryEx pageMem = (PageMemoryEx)tup.get1().memoryPolicy().pageMemory(); + CacheGroupContext gctx = tup.get1(); + + snapshotMgr.onCacheGroupStop(gctx); + + PageMemoryEx pageMem = (PageMemoryEx)gctx.memoryPolicy().pageMemory(); Collection grpIds = destroyed.get(pageMem); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java index 95af487731a2a..ad804cb3b3cdb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java @@ -26,6 +26,7 @@ import org.apache.ignite.internal.pagemem.FullPageId; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation; +import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; @@ -108,6 +109,13 @@ public void onCacheStop(GridCacheContext cctx) { // No-op. } + /** + * @param gctx Cctx. + */ + public void onCacheGroupStop(CacheGroupContext gctx) { + // No-op. + } + /** * */ From ee7566b1063de69ca4f261ee2088d69d231b1c2a Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Tue, 4 Jul 2017 20:00:09 +0300 Subject: [PATCH 051/155] Minor code cleanup --- .../processors/cluster/GridClusterStateProcessor.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java index 031c596258168..d57c720aff92c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java @@ -725,13 +725,6 @@ private Exception onActivate(ChangeGlobalStateContext cgsCtx) { log.info("Start activation process [nodeId=" + ctx.localNodeId() + ", client=" + client + ", topVer=" + cgsCtx.topVer + "]"); - Collection cfgs = new ArrayList<>(); - - for (DynamicCacheChangeRequest req : cgsCtx.batch.requests()) { - if (req.startCacheConfiguration() != null) - cfgs.add(new StoredCacheData(req.startCacheConfiguration())); - } - try { if (!client) sharedCtx.database().lock(); From 44f3fac27bec89b5e70e87564c527e48565ddd2a Mon Sep 17 00:00:00 2001 From: dpavlov Date: Tue, 4 Jul 2017 20:23:40 +0300 Subject: [PATCH 052/155] IGNITE-5558 - Added ability to read WAL in standalone mode - Fixes #2174. Signed-off-by: Alexey Goncharuk --- .../PersistentStoreConfiguration.java | 39 +- .../org/apache/ignite/events/EventType.java | 12 + .../events/WalSegmentArchivedEvent.java | 62 ++ .../pagemem/wal/record/WALRecord.java | 11 +- .../IgniteCacheDatabaseSharedManager.java | 10 +- .../wal/AbstractWalRecordsIterator.java | 289 +++++++++ .../cache/persistence/wal/FileInput.java | 16 +- .../cache/persistence/wal/FileWALPointer.java | 4 +- .../wal/FileWriteAheadLogManager.java | 586 ++++++++---------- .../persistence/wal/RecordSerializer.java | 5 + .../persistence/wal/SegmentArchiveResult.java | 61 ++ .../persistence/wal/SegmentEofException.java | 3 +- .../wal/reader/IgniteWalIteratorFactory.java | 102 +++ .../reader/StandaloneGridKernalContext.java | 499 +++++++++++++++ ...aloneIgniteCacheDatabaseSharedManager.java | 30 + .../reader/StandaloneWalRecordsIterator.java | 258 ++++++++ .../wal/serializer/RecordV1Serializer.java | 45 +- ...nitePersistentStoreDataStructuresTest.java | 2 + .../wal/IgniteWalHistoryReservationsTest.java | 2 +- .../db/wal/reader/IgniteWalReaderTest.java | 385 ++++++++++++ .../db/wal/reader/MockWalIteratorFactory.java | 114 ++++ .../testsuites/IgnitePdsTestSuite2.java | 9 +- 22 files changed, 2194 insertions(+), 350 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/events/WalSegmentArchivedEvent.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentArchiveResult.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneIgniteCacheDatabaseSharedManager.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java index 1d41d41de8d5a..b531f9d1510ec 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java @@ -55,7 +55,7 @@ public class PersistentStoreConfiguration implements Serializable { /** */ public static final int DFLT_WAL_SEGMENTS = 10; - /** */ + /** Default WAL file segment size, 64MBytes */ public static final int DFLT_WAL_SEGMENT_SIZE = 64 * 1024 * 1024; /** Default wal mode. */ @@ -103,10 +103,10 @@ public class PersistentStoreConfiguration implements Serializable { /** Number of work WAL segments. */ private int walSegments = DFLT_WAL_SEGMENTS; - /** Number of WAL segments to keep. */ + /** Size of one WAL segment in bytes. 64 Mb is used by default. Maximum value is 2Gb */ private int walSegmentSize = DFLT_WAL_SEGMENT_SIZE; - /** WAL persistence path. */ + /** Directory where WAL is stored (work directory) */ private String walStorePath = DFLT_WAL_STORE_PATH; /** WAL archive path. */ @@ -121,7 +121,7 @@ public class PersistentStoreConfiguration implements Serializable { /** WAl thread local buffer size. */ private int tlbSize = DFLT_TLB_SIZE; - /** Wal flush frequency. */ + /** Wal flush frequency in milliseconds. */ private int walFlushFreq = DFLT_WAL_FLUSH_FREQ; /** Wal fsync delay. */ @@ -146,6 +146,11 @@ public class PersistentStoreConfiguration implements Serializable { /** Time interval (in milliseconds) for rate-based metrics. */ private long rateTimeInterval = DFLT_RATE_TIME_INTERVAL_MILLIS; + /** + * Time interval (in milliseconds) for running auto archiving for incompletely WAL segment + */ + private long walAutoArchiveAfterInactivity = -1; + /** * Returns a path the root directory where the Persistent Store will persist data and indexes. */ @@ -297,7 +302,7 @@ public PersistentStoreConfiguration setWalSegments(int walSegments) { } /** - * Gets size of a WAL segment. + * Gets size of a WAL segment in bytes. * * @return WAL segment size. */ @@ -308,7 +313,7 @@ public int getWalSegmentSize() { /** * Sets size of a WAL segment. * - * @param walSegmentSize WAL segment size. 64 MB is used by default. + * @param walSegmentSize WAL segment size. 64 MB is used by default. Maximum value is 2Gb * @return {@code this} for chaining. */ public PersistentStoreConfiguration setWalSegmentSize(int walSegmentSize) { @@ -533,6 +538,28 @@ public PersistentStoreConfiguration setAlwaysWriteFullPages(boolean alwaysWriteF return this; } + /** + * Note: setting this value with {@link WALMode#DEFAULT} may generate file size overhead for WAL segments in case + * grid is used rarely. + * + * @param walAutoArchiveAfterInactivity time in millis to run auto archiving segment (even if incomplete) after last + * record logging.
Positive value enables incomplete segment archiving after timeout (inactivity).
Zero or + * negative value disables auto archiving. + * @return current configuration instance for chaining + */ + public PersistentStoreConfiguration setWalAutoArchiveAfterInactivity(long walAutoArchiveAfterInactivity) { + this.walAutoArchiveAfterInactivity = walAutoArchiveAfterInactivity; + + return this; + } + + /** + * @return time in millis to run auto archiving WAL segment (even if incomplete) after last record log + */ + public long getWalAutoArchiveAfterInactivity() { + return walAutoArchiveAfterInactivity; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(PersistentStoreConfiguration.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/events/EventType.java b/modules/core/src/main/java/org/apache/ignite/events/EventType.java index 1960692e3b1f0..47b40893e1ce7 100644 --- a/modules/core/src/main/java/org/apache/ignite/events/EventType.java +++ b/modules/core/src/main/java/org/apache/ignite/events/EventType.java @@ -766,6 +766,18 @@ public interface EventType { */ public static final int EVT_IGFS_FILE_PURGED = 127; + /** + * Built-in event type: WAL segment movement to archive folder completed + *

+ * Fired for each completed WAL segment which was moved to archive + *

+ * NOTE: all types in range from 1 to 1000 are reserved for + * internal Ignite events and should not be used by user-defined events. + * + * @see WalSegmentArchivedEvent + */ + public static final int EVT_WAL_SEGMENT_ARCHIVED = 128; + /** * All checkpoint events. This array can be directly passed into * {@link IgniteEvents#localListen(IgnitePredicate, int...)} method to diff --git a/modules/core/src/main/java/org/apache/ignite/events/WalSegmentArchivedEvent.java b/modules/core/src/main/java/org/apache/ignite/events/WalSegmentArchivedEvent.java new file mode 100644 index 0000000000000..2fc1715991bb1 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/events/WalSegmentArchivedEvent.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.events; + +import java.io.File; +import org.apache.ignite.cluster.ClusterNode; +import org.jetbrains.annotations.NotNull; + +/** + * Event indicates there was movement of WAL segment file to archive has been completed + */ +public class WalSegmentArchivedEvent extends EventAdapter { + /** */ + private static final long serialVersionUID = 0L; + + /** Absolute WAL segment file index. */ + private long absWalSegmentIdx; + + /** Destination archive file. This file is completed and closed archive segment */ + private final File archiveFile; + + /** + * Creates WAL segment event + * + * @param node Node. + * @param absWalSegmentIdx Absolute wal segment index. + * @param archiveFile Archive file. + */ + public WalSegmentArchivedEvent( + @NotNull final ClusterNode node, + final long absWalSegmentIdx, + final File archiveFile) { + super(node, "", EventType.EVT_WAL_SEGMENT_ARCHIVED); + this.absWalSegmentIdx = absWalSegmentIdx; + this.archiveFile = archiveFile; + } + + /** @return {@link #archiveFile} */ + public File getArchiveFile() { + return archiveFile; + } + + /** @return {@link #absWalSegmentIdx} */ + public long getAbsWalSegmentIdx() { + return absWalSegmentIdx; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java index 678e1fad94df7..89f3c86672277 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.pagemem.wal.record; +import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -26,7 +27,8 @@ */ public abstract class WALRecord { /** - * Record type. + * Record type. Ordinal of this record will be written to file.
+ * Note: Do not change order of elements
*/ public enum RecordType { /** */ @@ -171,6 +173,13 @@ public enum RecordType { public static RecordType fromOrdinal(int ord) { return ord < 0 || ord >= VALS.length ? null : VALS[ord]; } + + /** + * Fake record type, causes stop iterating and indicates segment EOF + * Note: regular record type is incremented by 1 and minimal value written to file is also 1 + * For {@link WALMode#DEFAULT} this value is at least came from padding + */ + public static final int STOP_ITERATION_RECORD_TYPE = 0; } /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index ec0e8953ce0a7..f04c27898f4e0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -92,7 +92,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** */ private FreeListImpl dfltFreeList; - /** */ + /** Page size from memory configuration, may be set only for fake(standalone) IgniteCacheDataBaseSharedManager */ private int pageSize; /** {@inheritDoc} */ @@ -961,4 +961,12 @@ protected File buildPath(String path, String consId) { public String systemMemoryPolicyName() { return SYSTEM_MEMORY_POLICY_NAME; } + + /** + * Method for fake (standalone) context initialization. Not to be called in production code + * @param pageSize configured page size + */ + protected void setPageSize(int pageSize) { + this.pageSize = pageSize; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java new file mode 100644 index 0000000000000..7dc0a280e5f54 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.wal; + +import java.io.EOFException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.pagemem.wal.WALIterator; +import org.apache.ignite.internal.pagemem.wal.WALPointer; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; +import org.apache.ignite.internal.util.GridCloseableIteratorAdapter; +import org.apache.ignite.lang.IgniteBiTuple; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Iterator over WAL segments. This abstract class provides most functionality for reading records in log. + * Subclasses are to override segment switching functionality + */ +public abstract class AbstractWalRecordsIterator extends GridCloseableIteratorAdapter> + implements WALIterator { + /** */ + private static final long serialVersionUID = 0L; + + /** + * Current record preloaded, to be returned on next()
+ * Normally this should be not null because advance() method should already prepare some value
+ */ + protected IgniteBiTuple curRec; + + /** + * Current WAL segment absolute index.
+ * Determined as lowest number of file at start, is changed during advance segment + */ + protected long curWalSegmIdx = -1; + + /** + * Current WAL segment read file handle. To be filled by subclass advanceSegment + */ + private FileWriteAheadLogManager.ReadFileHandle currWalSegment; + + /** Logger */ + @NotNull protected final IgniteLogger log; + + /** Shared context for creating serializer of required version and grid name access */ + @NotNull private final GridCacheSharedContext sharedCtx; + + /** Serializer of current version to read headers. */ + @NotNull private final RecordSerializer serializer; + + /** Utility buffer for reading records */ + private final ByteBuffer buf; + + /** + * @param log Logger + * @param sharedCtx Shared context + * @param serializer Serializer of current version to read headers. + * @param bufSize buffer for reading records size + */ + protected AbstractWalRecordsIterator( + @NotNull final IgniteLogger log, + @NotNull final GridCacheSharedContext sharedCtx, + @NotNull final RecordSerializer serializer, + final int bufSize) { + this.log = log; + this.sharedCtx = sharedCtx; + this.serializer = serializer; + + // Do not allocate direct buffer for iterator. + buf = ByteBuffer.allocate(bufSize); + buf.order(ByteOrder.nativeOrder()); + + } + + /** + * Scans provided folder for a WAL segment files + * @param walFilesDir directory to scan + * @return found WAL file descriptors + */ + protected static FileWriteAheadLogManager.FileDescriptor[] loadFileDescriptors(@NotNull final File walFilesDir) throws IgniteCheckedException { + final File[] files = walFilesDir.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER); + + if (files == null) { + throw new IgniteCheckedException("WAL files directory does not not denote a " + + "directory, or if an I/O error occurs: [" + walFilesDir.getAbsolutePath() + "]"); + } + return FileWriteAheadLogManager.scan(files); + } + + /** {@inheritDoc} */ + @Override protected IgniteBiTuple onNext() throws IgniteCheckedException { + IgniteBiTuple ret = curRec; + + advance(); + + return ret; + } + + /** {@inheritDoc} */ + @Override protected boolean onHasNext() throws IgniteCheckedException { + return curRec != null; + } + + /** + * Switches records iterator to the next record. + *

    + *
  • {@link #curRec} will be updated.
    • + *
  • If end of segment reached, switch to new segment is called. {@link #currWalSegment} will be updated.
    • + *
+ * + * {@code advance()} runs a step ahead {@link #next()} + * + * @throws IgniteCheckedException If failed. + */ + protected void advance() throws IgniteCheckedException { + while (true) { + curRec = advanceRecord(currWalSegment); + + if (curRec != null) + return; + else { + currWalSegment = advanceSegment(currWalSegment); + + if (currWalSegment == null) + return; + } + } + } + + /** + * Closes and returns WAL segment (if any) + * @return closed handle + * @throws IgniteCheckedException if IO failed + */ + @Nullable protected FileWriteAheadLogManager.ReadFileHandle closeCurrentWalSegment() throws IgniteCheckedException { + final FileWriteAheadLogManager.ReadFileHandle walSegmentClosed = currWalSegment; + + if (walSegmentClosed != null) { + walSegmentClosed.close(); + currWalSegment = null; + } + return walSegmentClosed; + } + + /** + * Switches records iterator to the next WAL segment + * as result of this method, new reference to segment should be returned. + * Null for current handle means stop of iteration + * @throws IgniteCheckedException if reading failed + * @param curWalSegment current open WAL segment or null if there is no open segment yet + * @return new WAL segment to read or null for stop iteration + */ + protected abstract FileWriteAheadLogManager.ReadFileHandle advanceSegment( + @Nullable final FileWriteAheadLogManager.ReadFileHandle curWalSegment) throws IgniteCheckedException; + + /** + * Switches to new record + * @param hnd currently opened read handle + * @return next advanced record + */ + private IgniteBiTuple advanceRecord( + @Nullable final FileWriteAheadLogManager.ReadFileHandle hnd) { + if (hnd == null) + return null; + + final FileWALPointer ptr = new FileWALPointer( + hnd.idx, + (int)hnd.in.position(), + 0); + + try { + final WALRecord rec = hnd.ser.readRecord(hnd.in, ptr); + + ptr.length(rec.size()); + + // cast using diamond operator here can break compile for 7 + return new IgniteBiTuple<>((WALPointer)ptr, rec); + } + catch (IOException | IgniteCheckedException e) { + if (!(e instanceof SegmentEofException)) + handleRecordException(e, ptr); + return null; + } + } + + /** + * Handler for record deserialization exception + * @param e problem from records reading + * @param ptr file pointer was accessed + */ + protected void handleRecordException( + @NotNull final Exception e, + @Nullable final FileWALPointer ptr) { + if (log.isInfoEnabled()) + log.info("Stopping WAL iteration due to an exception: " + e.getMessage()); + } + + /** + * @param desc File descriptor. + * @param start Optional start pointer. Null means read from the beginning + * @return Initialized file handle. + * @throws FileNotFoundException If segment file is missing. + * @throws IgniteCheckedException If initialized failed due to another unexpected error. + */ + protected FileWriteAheadLogManager.ReadFileHandle initReadHandle( + @NotNull final FileWriteAheadLogManager.FileDescriptor desc, + @Nullable final FileWALPointer start) + throws IgniteCheckedException, FileNotFoundException { + try { + RandomAccessFile rf = new RandomAccessFile(desc.file, "r"); + + try { + FileChannel ch = rf.getChannel(); + FileInput in = new FileInput(ch, buf); + + // Header record must be agnostic to the serializer version. + WALRecord rec = serializer.readRecord(in, + new FileWALPointer(desc.idx, (int)ch.position(), 0)); + + if (rec == null) + return null; + + if (rec.type() != WALRecord.RecordType.HEADER_RECORD) + throw new IOException("Missing file header record: " + desc.file.getAbsoluteFile()); + + int ver = ((HeaderRecord)rec).version(); + + RecordSerializer ser = FileWriteAheadLogManager.forVersion(sharedCtx, ver); + + if (start != null && desc.idx == start.index()) + in.seek(start.fileOffset()); + + return new FileWriteAheadLogManager.ReadFileHandle(rf, desc.idx, sharedCtx.igniteInstanceName(), ser, in); + } + catch (SegmentEofException | EOFException ignore) { + try { + rf.close(); + } + catch (IOException ce) { + throw new IgniteCheckedException(ce); + } + + return null; + } + catch (IOException | IgniteCheckedException e) { + try { + rf.close(); + } + catch (IOException ce) { + e.addSuppressed(ce); + } + + throw e; + } + } + catch (FileNotFoundException e) { + throw e; + } + catch (IOException e) { + throw new IgniteCheckedException( + "Failed to initialize WAL segment: " + desc.file.getAbsolutePath(), e); + } + } + +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java index be1e477f9599c..e2d7cbaab5568 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java @@ -26,21 +26,25 @@ import org.jetbrains.annotations.NotNull; /** - * File input. + * File input, backed by byte buffer file input. + * This class allows to read data by chunks from file and then read primitives */ public final class FileInput implements ByteBufferBackedDataInput { - /** */ + /** + * Buffer for reading blocks of data into. + * Note: biggest block requested from this input can't be longer than buffer capacity + */ private ByteBuffer buf; - /** */ + /** File channel to read chunks from */ private FileChannel ch; /** */ private long pos; /** - * @param ch Channel. - * @param buf Buffer. + * @param ch Channel to read from + * @param buf Buffer for reading blocks of data into */ public FileInput(FileChannel ch, ByteBuffer buf) throws IOException { assert ch != null; @@ -101,7 +105,7 @@ public void seek(long pos) throws IOException { int read = ch.read(buf); if (read == -1) - throw new EOFException(); + throw new EOFException("EOF at position [" + ch.position() + "] expected to read [" + requested + "] bytes"); available += read; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java index b6ddfb8f5f3a8..3716de2e83a45 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java @@ -46,7 +46,7 @@ public FileWALPointer(long idx, int fileOffset, int len) { } /** - * @param idx File timestamp index. + * @param idx Absolute WAL segment file index . * @param fileOffset Offset in file, from the beginning. * @param len Record length. * @param forceFlush Force flush flag. @@ -59,7 +59,7 @@ public FileWALPointer(long idx, int fileOffset, int len, boolean forceFlush) { } /** - * @return Timestamp index. + * @return Absolute WAL segment file index . */ public long index() { return idx; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 591814161c114..f877a14c5d087 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -27,6 +27,7 @@ import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.nio.file.Files; +import java.sql.Time; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -34,11 +35,11 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.LockSupport; import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; import org.apache.ignite.IgniteCheckedException; @@ -46,8 +47,10 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.PersistentStoreConfiguration; import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.events.EventType; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.pagemem.wal.StorageException; import org.apache.ignite.internal.pagemem.wal.WALIterator; @@ -58,9 +61,11 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.PersistenceMetricsImpl; +import org.apache.ignite.events.WalSegmentArchivedEvent; import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer; -import org.apache.ignite.internal.util.GridCloseableIteratorAdapter; +import org.apache.ignite.internal.processors.timeout.GridTimeoutObject; +import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; @@ -69,6 +74,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -85,14 +91,14 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl /** */ private static final byte[] FILL_BUF = new byte[1024 * 1024]; - /** */ + /** Pattern for segment file names */ private static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal"); /** */ private static final Pattern WAL_TEMP_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal\\.tmp"); - /** */ - private static final FileFilter WAL_SEGMENT_FILE_FILTER = new FileFilter() { + /** WAL segment file filter, see {@link #WAL_NAME_PATTERN} */ + public static final FileFilter WAL_SEGMENT_FILE_FILTER = new FileFilter() { @Override public boolean accept(File file) { return !file.isDirectory() && WAL_NAME_PATTERN.matcher(file.getName()).matches(); } @@ -118,7 +124,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl private final int tlbSize; /** WAL flush frequency. Makes sense only for {@link WALMode#BACKGROUND} log WALMode. */ - public final int flushFreq; + private final int flushFreq; /** Fsync delay. */ private final long fsyncDelay; @@ -126,6 +132,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl /** */ private final PersistentStoreConfiguration psCfg; + /** Events service */ + private final GridEventStorageManager evt; + /** */ private IgniteConfiguration igCfg; @@ -135,10 +144,10 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl /** */ private File walWorkDir; - /** */ + /** WAL archive directory (including consistent ID as subfolder) */ private File walArchiveDir; - /** */ + /** Serializer of current version, used to read header record and for write records */ private RecordSerializer serializer; /** */ @@ -166,19 +175,42 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl /** */ private volatile FileArchiver archiver; - /** */ - private QueueFlusher flusher; - /** */ private final ThreadLocal lastWALPtr = new ThreadLocal<>(); /** Current log segment handle */ private volatile FileWriteHandle currentHnd; + /** + * Positive (non-0) value indicates WAL can be archived even if not complete
+ * See {@link PersistentStoreConfiguration#setWalAutoArchiveAfterInactivity(long)}
+ */ + private final long walAutoArchiveAfterInactivity; + + /** + * Container with last WAL record logged timestamp.
+ * Zero value means there was no records logged to current segment, skip possible archiving for this case
+ * Value is filled only for case {@link #walAutoArchiveAfterInactivity} > 0
+ */ + private AtomicLong lastRecordLoggedMs = new AtomicLong(); + + /** + * Cancellable task for {@link WALMode#BACKGROUND}, should be cancelled at shutdown + * Null for non background modes + */ + @Nullable private volatile GridTimeoutProcessor.CancelableTask backgroundFlushSchedule; + + /** + * Reference to the last added next archive timeout check object. + * Null if mode is not enabled. + * Should be cancelled at shutdown + */ + @Nullable private volatile GridTimeoutObject nextAutoArchiveTimeoutObj; + /** * @param ctx Kernal context. */ - public FileWriteAheadLogManager(GridKernalContext ctx) { + public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) { igCfg = ctx.config(); PersistentStoreConfiguration psCfg = igCfg.getPersistentStoreConfiguration(); @@ -193,6 +225,8 @@ public FileWriteAheadLogManager(GridKernalContext ctx) { flushFreq = psCfg.getWalFlushFrequency(); fsyncDelay = psCfg.getWalFsyncDelay(); alwaysWriteFullPages = psCfg.isAlwaysWriteFullPages(); + walAutoArchiveAfterInactivity = psCfg.getWalAutoArchiveAfterInactivity(); + evt = ctx.event(); } /** {@inheritDoc} */ @@ -248,8 +282,8 @@ private void checkWalConfiguration() throws IgniteCheckedException { if (psCfg.getWalStorePath() == null ^ psCfg.getWalArchivePath() == null) { throw new IgniteCheckedException( "Properties should be either both specified or both null " + - "[walStorePath = " + psCfg.getWalStorePath() + - ", walArchivePath = " + psCfg.getWalArchivePath() + "]" + "[walStorePath = " + psCfg.getWalStorePath() + + ", walArchivePath = " + psCfg.getWalArchivePath() + "]" ); } } @@ -271,26 +305,32 @@ protected String consistentId() { /** {@inheritDoc} */ @Override protected void stop0(boolean cancel) { - FileWriteHandle currentHnd = currentHandle(); + final GridTimeoutProcessor.CancelableTask schedule = backgroundFlushSchedule; - try { - QueueFlusher flusher0 = flusher; + if (schedule != null) + schedule.close(); - if (flusher0 != null) { - flusher0.shutdown(); + final GridTimeoutObject timeoutObj = nextAutoArchiveTimeoutObj; - if (currentHnd != null) - currentHnd.flush((FileWALPointer)null); + if (timeoutObj != null) + cctx.time().removeTimeoutObject(timeoutObj); + + final FileWriteHandle currHnd = currentHandle(); + + try { + if (mode == WALMode.BACKGROUND) { + if (currHnd != null) + currHnd.flush((FileWALPointer)null); } - if (currentHnd != null) - currentHnd.close(false); + if (currHnd != null) + currHnd.close(false); if (archiver != null) archiver.shutdown(); } catch (Exception e) { - U.error(log, "Failed to gracefully close WAL segment: " + currentHnd.file, e); + U.error(log, "Failed to gracefully close WAL segment: " + currHnd.file, e); } } @@ -350,39 +390,114 @@ protected String consistentId() { } if (mode == WALMode.BACKGROUND) { - flusher = new QueueFlusher(cctx.igniteInstanceName()); - - flusher.start(); + backgroundFlushSchedule = cctx.time().schedule(new Runnable() { + @Override public void run() { + doFlush(); + } + }, flushFreq, flushFreq); } + + if (walAutoArchiveAfterInactivity > 0) + scheduleNextInactivityPeriodElapsedCheck(); } catch (StorageException e) { throw new IgniteCheckedException(e); } } + /** + * Schedules next check of inactivity period expired. Based on current record update timestamp. + * At timeout method does check of inactivity period and schedules new launch. + */ + private void scheduleNextInactivityPeriodElapsedCheck() { + final long lastRecMs = lastRecordLoggedMs.get(); + final long nextPossibleAutoArchive = (lastRecMs <= 0 ? U.currentTimeMillis() : lastRecMs) + walAutoArchiveAfterInactivity; + + if (log.isDebugEnabled()) + log.debug("Schedule WAL rollover check at " + new Time(nextPossibleAutoArchive).toString()); + + nextAutoArchiveTimeoutObj = new GridTimeoutObject() { + private final IgniteUuid id = IgniteUuid.randomUuid(); + + @Override public IgniteUuid timeoutId() { + return id; + } + + @Override public long endTime() { + return nextPossibleAutoArchive; + } + + @Override public void onTimeout() { + if (log.isDebugEnabled()) + log.debug("Checking if WAL rollover required (" + new Time(U.currentTimeMillis()).toString() + ")"); + + checkWalRolloverRequiredDuringInactivityPeriod(); + + scheduleNextInactivityPeriodElapsedCheck(); + } + }; + cctx.time().addTimeoutObject(nextAutoArchiveTimeoutObj); + } + + /** + * Checks if there was elapsed significant period of inactivity. + * If WAL auto-archive is enabled using {@link #walAutoArchiveAfterInactivity} > 0 this method will activate + * roll over by timeout
+ */ + private void checkWalRolloverRequiredDuringInactivityPeriod() { + if (walAutoArchiveAfterInactivity <= 0) + return; // feature not configured, nothing to do + + final long lastRecMs = lastRecordLoggedMs.get(); + + if (lastRecMs == 0) + return; //no records were logged to current segment, does not consider inactivity + + final long elapsedMs = U.currentTimeMillis() - lastRecMs; + + if (elapsedMs <= walAutoArchiveAfterInactivity) + return; // not enough time elapsed since last write + + if (!lastRecordLoggedMs.compareAndSet(lastRecMs, 0)) + return; // record write occurred concurrently + + final FileWriteHandle handle = currentHandle(); + + try { + rollOver(handle); + } + catch (IgniteCheckedException e) { + U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e); + handle.invalidateEnvironment(e); + } + } + /** {@inheritDoc} */ @SuppressWarnings("TooBroadScope") @Override public WALPointer log(WALRecord record) throws IgniteCheckedException, StorageException { if (serializer == null || mode == WALMode.NONE) return null; - FileWriteHandle current = currentHandle(); + FileWriteHandle currWrHandle = currentHandle(); // Logging was not resumed yet. - if (current == null) + if (currWrHandle == null) return null; // Need to calculate record size first. record.size(serializer.size(record)); - for (; ; current = rollOver(current)) { - WALPointer ptr = current.addRecord(record); + for (; ; currWrHandle = rollOver(currWrHandle)) { + WALPointer ptr = currWrHandle.addRecord(record); if (ptr != null) { metrics.onWalRecordLogged(); lastWALPtr.set(ptr); + if (walAutoArchiveAfterInactivity > 0) + lastRecordLoggedMs.set(U.currentTimeMillis()); + return ptr; } @@ -665,6 +780,9 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I assert swapped : "Concurrent updates on rollover are not allowed"; + if (walAutoArchiveAfterInactivity > 0) + lastRecordLoggedMs.set(0); + // Let other threads to proceed with new segment. hnd.signalNextAvailable(); } @@ -888,7 +1006,7 @@ private File pollNextFile(long curIdx) throws IgniteCheckedException { * @param ver Serializer version. * @return Entry serializer. */ - private static RecordSerializer forVersion(GridCacheSharedContext cctx, int ver) throws IgniteCheckedException { + static RecordSerializer forVersion(GridCacheSharedContext cctx, int ver) throws IgniteCheckedException { if (ver <= 0) throw new IgniteCheckedException("Failed to create a serializer (corrupted WAL file)."); @@ -905,7 +1023,7 @@ private static RecordSerializer forVersion(GridCacheSharedContext cctx, int ver) /** * @return Sorted WAL files descriptors. */ - private static FileDescriptor[] scan(File[] allFiles) { + public static FileDescriptor[] scan(File[] allFiles) { if (allFiles == null) return EMPTY_DESCRIPTORS; @@ -931,11 +1049,11 @@ private static FileDescriptor[] scan(File[] allFiles) { * * Monitor of current object is used for notify on: *
    - *
  • exception occurred ({@link FileArchiver#cleanException}!=null)
    • - *
  • stopping thread ({@link FileArchiver#stopped}==true)
    • - *
  • current file index changed ({@link FileArchiver#curAbsWalIdx})
    • - *
  • last archived file index was changed ({@link FileArchiver#lastAbsArchivedIdx})
    • - *
  • some WAL index was removed from {@link FileArchiver#locked} map
    • + *
  • exception occurred ({@link FileArchiver#cleanException}!=null)
    • + *
  • stopping thread ({@link FileArchiver#stopped}==true)
    • + *
  • current file index changed ({@link FileArchiver#curAbsWalIdx})
    • + *
  • last archived file index was changed ({@link FileArchiver#lastAbsArchivedIdx})
    • + *
  • some WAL index was removed from {@link FileArchiver#locked} map
    • *
*/ private class FileArchiver extends Thread { @@ -1017,6 +1135,7 @@ private synchronized void reserve(long absIdx) { /** * Check if WAL segment locked or reserved + * * @param absIdx Index for check reservation. * @return {@code True} if index is reserved. */ @@ -1080,7 +1199,7 @@ private synchronized void release(long absIdx) { break; try { - File workFile = archiveSegment(toArchive); + final SegmentArchiveResult res = archiveSegment(toArchive); synchronized (this) { while (locked.containsKey(toArchive) && !stopped) @@ -1088,13 +1207,16 @@ private synchronized void release(long absIdx) { // Firstly, format working file if (!stopped) - formatFile(workFile); + formatFile(res.getOrigWorkFile()); // Then increase counter to allow rollover on clean working file lastAbsArchivedIdx = toArchive; notifyAll(); } + if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED)) + evt.record(new WalSegmentArchivedEvent(cctx.discovery().localNode(), + res.getAbsIdx(), res.getDstArchiveFile())); } catch (IgniteCheckedException e) { synchronized (this) { @@ -1115,7 +1237,7 @@ private synchronized void release(long absIdx) { * Blocks till there are available file to write * * @param curIdx Current absolute index that we want to increment. - * @return Next index (curIdx+1) when it is ready to be written. + * @return Next index (curWalSegmIdx+1) when it is ready to be written. * @throws IgniteCheckedException If failed (if interrupted or if exception occurred in the archiver thread). */ private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException { @@ -1195,9 +1317,12 @@ private void releaseWorkSegment(long absIdx) { } /** + * Moves WAL segment from work folder to archive folder. + * Temp file is used to do movement + * * @param absIdx Absolute index to archive. */ - private File archiveSegment(long absIdx) throws IgniteCheckedException { + private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedException { long segIdx = absIdx % psCfg.getWalSegments(); File origFile = new File(walWorkDir, FileDescriptor.fileName(segIdx)); @@ -1235,7 +1360,7 @@ private File archiveSegment(long absIdx) throws IgniteCheckedException { log.debug("Copied file [src=" + origFile.getAbsolutePath() + ", dst=" + dstFile.getAbsolutePath() + ']'); - return origFile; + return new SegmentArchiveResult(absIdx, origFile, dstFile); } /** @@ -1316,7 +1441,7 @@ private int readSerializerVersion(RandomAccessFile rf, File file, long idx) /** * WAL file descriptor. */ - private static class FileDescriptor implements Comparable { + public static class FileDescriptor implements Comparable { /** */ protected final File file; @@ -1324,9 +1449,11 @@ private static class FileDescriptor implements Comparable { protected final long idx; /** - * @param file File. + * Creates file descriptor. Index is restored from file name + * + * @param file WAL segment file. */ - private FileDescriptor(File file) { + public FileDescriptor(@NotNull File file) { this(file, null); } @@ -1334,7 +1461,7 @@ private FileDescriptor(File file) { * @param file WAL segment file. * @param idx Absolute WAL segment file index. For null value index is restored from file name */ - private FileDescriptor(@NotNull File file, @Nullable Long idx) { + public FileDescriptor(@NotNull File file, @Nullable Long idx) { this.file = file; String fileName = file.getName(); @@ -1350,7 +1477,7 @@ private FileDescriptor(@NotNull File file, @Nullable Long idx) { * @param segment Segment index. * @return Segment file name. */ - private static String fileName(long segment) { + public static String fileName(long segment) { SB b = new SB(); String segmentStr = Long.toString(segment); @@ -1402,6 +1529,20 @@ private static String segmentNumber(long segment) { @Override public int hashCode() { return (int)(idx ^ (idx >>> 32)); } + + /** + * @return Absolute WAL segment file index + */ + public long getIdx() { + return idx; + } + + /** + * @return absolute pathname string of this file descriptor pathname. + */ + public String getAbsolutePath() { + return file.getAbsolutePath(); + } } /** @@ -1438,14 +1579,17 @@ private FileHandle(RandomAccessFile file, long idx, String gridName) { /** * */ - private static class ReadFileHandle extends FileHandle { + public static class ReadFileHandle extends FileHandle { /** Entry serializer. */ - private RecordSerializer ser; + RecordSerializer ser; /** */ - private FileInput in; + FileInput in; - /** */ + /** + * true if this file handle came from work directory. + * false if this file handle came from archive directory. + */ private boolean workDir; /** @@ -1454,7 +1598,7 @@ private static class ReadFileHandle extends FileHandle { * @param ser Entry serializer. * @param in File input. */ - private ReadFileHandle( + ReadFileHandle( RandomAccessFile file, long idx, String gridName, @@ -1499,7 +1643,10 @@ private class FileWriteHandle extends FileHandle { */ private final AtomicReference head = new AtomicReference<>(); - /** Position in current file after the end of last written record (incremented after file channel write operation) */ + /** + * Position in current file after the end of last written record (incremented after file channel write + * operation) + */ private volatile long written; /** */ @@ -1508,7 +1655,7 @@ private class FileWriteHandle extends FileHandle { /** Environment failure. */ private volatile Throwable envFailed; - /** Stop guard to provide warranty that only one thread will be successful in calling {@link #close(boolean)}*/ + /** Stop guard to provide warranty that only one thread will be successful in calling {@link #close(boolean)} */ private final AtomicBoolean stop = new AtomicBoolean(false); /** */ @@ -1754,6 +1901,7 @@ private boolean flush(WALRecord expHead) throws StorageException, IgniteCheckedE /** * Serializes WAL records chain to provided byte buffer + * * @param buf Buffer, will be filled with records chain from end to beginning * @param head Head of the chain to write to the buffer. * @return Position in file for this buffer. @@ -1886,11 +2034,15 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx flushOrWait(null); try { - if (rollOver && written < (maxSegmentSize - 1)) { - ByteBuffer allocate = ByteBuffer.allocate(1); - allocate.put((byte) WALRecord.RecordType.SWITCH_SEGMENT_RECORD.ordinal()); - - ch.write(allocate, written); + int switchSegmentRecSize = RecordV1Serializer.REC_TYPE_SIZE + RecordV1Serializer.FILE_WAL_POINTER_SIZE; + if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) { + //it is expected there is sufficient space for this record because rollover should run early + final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize); + buf.put((byte)(WALRecord.RecordType.SWITCH_SEGMENT_RECORD.ordinal() + 1)); + final FileWALPointer pointer = new FileWALPointer(idx, (int)ch.position(), -1); + RecordV1Serializer.putPosition(buf, pointer); + buf.rewind(); + ch.write(buf, written); if (mode == WALMode.DEFAULT) ch.force(false); @@ -1951,8 +2103,8 @@ private void awaitNext() throws IgniteCheckedException { } /** - * @param pos Position in file to start write from. - * May be checked against actual position to wait previous writes to complete + * @param pos Position in file to start write from. May be checked against actual position to wait previous + * writes to complete * @param buf Buffer to write to file * @throws StorageException If failed. * @throws IgniteCheckedException If failed. @@ -2133,8 +2285,7 @@ private static final class FakeRecord extends WALRecord { /** * Iterator over WAL-log. */ - private static class RecordsIterator extends GridCloseableIteratorAdapter> - implements WALIterator { + private static class RecordsIterator extends AbstractWalRecordsIterator { /** */ private static final long serialVersionUID = 0L; /** */ @@ -2149,33 +2300,14 @@ private static class RecordsIterator extends GridCloseableIteratorAdapter curRec; - - /** */ - private long curIdx = -1; - - /** */ - private ReadFileHandle curHandle; - - /** */ - private ByteBuffer buf; - - /** */ - private IgniteLogger log; - /** * @param cctx Shared context. * @param walWorkDir WAL work dir. @@ -2183,78 +2315,55 @@ private static class RecordsIterator extends GridCloseableIteratorAdapter onNext() throws IgniteCheckedException { - IgniteBiTuple ret = curRec; - - advance(); - - return ret; - } - - /** {@inheritDoc} */ - @Override protected boolean onHasNext() throws IgniteCheckedException { - return curRec != null; - } - /** {@inheritDoc} */ @Override protected void onClose() throws IgniteCheckedException { curRec = null; - if (curHandle != null) { - curHandle.close(); + final ReadFileHandle handle = closeCurrentWalSegment(); + if (handle != null && handle.workDir) + releaseWorkSegment(curWalSegmIdx); - if (curHandle.workDir) - releaseWorkSegment(curIdx); - - curHandle = null; - } - - curIdx = Integer.MAX_VALUE; + curWalSegmIdx = Integer.MAX_VALUE; } /** * @throws IgniteCheckedException If failed to initialize first file handle. */ private void init() throws IgniteCheckedException { - FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER)); + FileDescriptor[] descs = loadFileDescriptors(walArchiveDir); if (start != null) { if (!F.isEmpty(descs)) { @@ -2264,13 +2373,13 @@ private void init() throws IgniteCheckedException { for (FileDescriptor desc : descs) { if (desc.idx == start.index()) { - curIdx = start.index(); + curWalSegmIdx = start.index(); break; } } - if (curIdx == -1) { + if (curWalSegmIdx == -1) { long lastArchived = descs[descs.length - 1].idx; if (lastArchived > start.index()) @@ -2278,203 +2387,86 @@ private void init() throws IgniteCheckedException { // This pointer may be in work files because archiver did not // copy the file yet, check that it is not too far forward. - curIdx = start.index(); + curWalSegmIdx = start.index(); } } else { // This means that whole checkpoint history fits in one segment in WAL work directory. // Will start from this index right away. - curIdx = start.index(); + curWalSegmIdx = start.index(); } } else - curIdx = !F.isEmpty(descs) ? descs[0].idx : 0; + curWalSegmIdx = !F.isEmpty(descs) ? descs[0].idx : 0; - curIdx--; + curWalSegmIdx--; if (log.isDebugEnabled()) - log.debug("Initialized WAL cursor [start=" + start + ", end=" + end + ", curIdx=" + curIdx + ']'); - } - - /** - * @throws IgniteCheckedException If failed. - */ - private void advance() throws IgniteCheckedException { - while (true) { - advanceRecord(); - - if (curRec != null) - return; - else { - advanceSegment(); - - if (curHandle == null) - return; - } - } - } - - /** - * - */ - private void advanceRecord() { - try { - ReadFileHandle hnd = curHandle; - - if (hnd != null) { - RecordSerializer ser = hnd.ser; - - int pos = (int)hnd.in.position(); - - FileWALPointer ptr = new FileWALPointer(hnd.idx, pos, 0); - - WALRecord rec = ser.readRecord(hnd.in, ptr); - - ptr.length(rec.size()); - - curRec = new IgniteBiTuple(ptr, rec); - } - } - catch (IOException | IgniteCheckedException e) { - if (!(e instanceof SegmentEofException)) { - if (log.isInfoEnabled()) - log.info("Stopping WAL iteration due to an exception: " + e.getMessage()); - } - - curRec = null; - } + log.debug("Initialized WAL cursor [start=" + start + ", end=" + end + ", curWalSegmIdx=" + curWalSegmIdx + ']'); } - /** - * @throws IgniteCheckedException If failed. - */ - private void advanceSegment() throws IgniteCheckedException { - ReadFileHandle cur0 = curHandle; - - if (cur0 != null) { - cur0.close(); + /** {@inheritDoc} */ + @Override protected ReadFileHandle advanceSegment( + @Nullable final ReadFileHandle curWalSegment) throws IgniteCheckedException { + if (curWalSegment != null) { + curWalSegment.close(); - if (cur0.workDir) - releaseWorkSegment(cur0.idx); + if (curWalSegment.workDir) + releaseWorkSegment(curWalSegment.idx); - curHandle = null; } // We are past the end marker. - if (end != null && curIdx + 1 > end.index()) - return; + if (end != null && curWalSegmIdx + 1 > end.index()) + return null; //stop iteration - curIdx++; + curWalSegmIdx++; FileDescriptor fd; - boolean readArchive = canReadArchiveOrReserveWork(curIdx); + boolean readArchive = canReadArchiveOrReserveWork(curWalSegmIdx); if (readArchive) { fd = new FileDescriptor(new File(walArchiveDir, - FileDescriptor.fileName(curIdx))); + FileDescriptor.fileName(curWalSegmIdx))); } else { - long workIdx = curIdx % psCfg.getWalSegments(); + long workIdx = curWalSegmIdx % psCfg.getWalSegments(); fd = new FileDescriptor( new File(walWorkDir, FileDescriptor.fileName(workIdx)), - curIdx); + curWalSegmIdx); } if (log.isDebugEnabled()) - log.debug("Reading next file [absIdx=" + curIdx + ", file=" + fd.file.getAbsolutePath() + ']'); + log.debug("Reading next file [absIdx=" + curWalSegmIdx + ", file=" + fd.file.getAbsolutePath() + ']'); assert fd != null; + ReadFileHandle nextHandle; try { - curHandle = initReadHandle(fd, start != null && curIdx == start.index() ? start : null); + nextHandle = initReadHandle(fd, start != null && curWalSegmIdx == start.index() ? start : null); } catch (FileNotFoundException e) { if (readArchive) throw new IgniteCheckedException("Missing WAL segment in the archive", e); else - curHandle = null; + nextHandle = null; } - if (curHandle != null) - curHandle.workDir = !readArchive; + if (nextHandle != null) + nextHandle.workDir = !readArchive; else - releaseWorkSegment(curIdx); + releaseWorkSegment(curWalSegmIdx); curRec = null; - } - - /** - * @param desc File descriptor. - * @param start Optional start pointer. - * @return Initialized file handle. - * @throws FileNotFoundException If segment file is missing. - * @throws IgniteCheckedException If initialized failed due to another unexpected error. - */ - private ReadFileHandle initReadHandle(FileDescriptor desc, FileWALPointer start) - throws IgniteCheckedException, FileNotFoundException { - try { - RandomAccessFile rf = new RandomAccessFile(desc.file, "r"); - - try { - FileChannel channel = rf.getChannel(); - FileInput in = new FileInput(channel, buf); - - // Header record must be agnostic to the serializer version. - WALRecord rec = serializer.readRecord(in, - new FileWALPointer(desc.idx, (int)channel.position(), 0)); - - if (rec == null) - return null; - - if (rec.type() != WALRecord.RecordType.HEADER_RECORD) - throw new IOException("Missing file header record: " + desc.file.getAbsoluteFile()); - - int ver = ((HeaderRecord)rec).version(); - - RecordSerializer ser = forVersion(cctx, ver); - - if (start != null && desc.idx == start.index()) - in.seek(start.fileOffset()); - - return new ReadFileHandle(rf, desc.idx, cctx.igniteInstanceName(), ser, in); - } - catch (SegmentEofException | EOFException ignore) { - try { - rf.close(); - } - catch (IOException ce) { - throw new IgniteCheckedException(ce); - } - - return null; - } - catch (IOException | IgniteCheckedException e) { - try { - rf.close(); - } - catch (IOException ce) { - e.addSuppressed(ce); - } - - throw e; - } - } - catch (FileNotFoundException e) { - throw e; - } - catch (IOException e) { - throw new IgniteCheckedException( - "Failed to initialize WAL segment: " + desc.file.getAbsolutePath(), e); - } + return nextHandle; } /** * @param absIdx Absolute index to check. - * @return {@code True} if we can safely read the archive, {@code false} if the segment has not been - * archived yet. In this case the corresponding work segment is reserved (will not be deleted until - * release). + * @return {@code True} if we can safely read the archive, {@code false} if the segment has not been archived + * yet. In this case the corresponding work segment is reserved (will not be deleted until release). */ private boolean canReadArchiveOrReserveWork(long absIdx) { return archiver != null && archiver.checkCanReadArchiveOrReserveWorkSegment(absIdx); @@ -2490,51 +2482,17 @@ private void releaseWorkSegment(long absIdx) { } /** - * Periodically flushes current file handle for {@link WALMode#BACKGROUND} WALMode. + * Flushes current file handle for {@link WALMode#BACKGROUND} WALMode. + * Called periodically from scheduler. */ - private class QueueFlusher extends Thread { - /** */ - private volatile boolean stopped; + private void doFlush() { + final FileWriteHandle hnd = currentHandle(); - /** - * @param gridName Grid name. - */ - private QueueFlusher(String gridName) { - super("wal-queue-flusher-#" + gridName); - } - - /** {@inheritDoc} */ - @Override public void run() { - while (!stopped) { - long wakeup = U.currentTimeMillis() + flushFreq; - - LockSupport.parkUntil(wakeup); - - FileWriteHandle hnd = currentHandle(); - - try { - hnd.flush(hnd.head.get()); - } - catch (IgniteCheckedException e) { - U.warn(log, "Failed to flush WAL record queue", e); - } - } + try { + hnd.flush(hnd.head.get()); } - - /** - * Signals stop, wakes up thread and waiting until completion. - */ - private void shutdown() { - stopped = true; - - LockSupport.unpark(this); - - try { - join(); - } - catch (InterruptedException ignore) { - // Got interrupted while waiting for flusher to shutdown. - } + catch (IgniteCheckedException e) { + U.warn(log, "Failed to flush WAL record queue", e); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordSerializer.java index 75a62a95a4bbd..1ea7fa62c61f4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordSerializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordSerializer.java @@ -33,6 +33,8 @@ public interface RecordSerializer { public int version(); /** + * Calculates record size in byte including expected wal pointer, CRC and type field + * * @param record Record. * @return Size in bytes. */ @@ -45,7 +47,10 @@ public interface RecordSerializer { public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException; /** + * Loads record from input + * * @param in Data input to read data from. + * @param expPtr expected WAL pointer for record. Used to validate actual position against expected from the file * @return Read entry. */ public WALRecord readRecord(FileInput in, WALPointer expPtr) throws IOException, IgniteCheckedException; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentArchiveResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentArchiveResult.java new file mode 100644 index 0000000000000..5b659703e721e --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentArchiveResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache.persistence.wal; + +import java.io.File; + +/** + * Result of archiving (movement) operation + * Replacement of generic T3-Tuple + */ +class SegmentArchiveResult { + /** Absolute WAL segment file index. */ + private final long absWalIdx; + + /** Original work file. May and most likely to be used for new WAL round */ + private final File origWorkFile; + + /** Destination archive file. This file is completed and closed archive segment */ + private final File dstArchiveFile; + + /** + * Creates result + * @param absWalIdx Absolute wal index. + * @param origWorkFile Orig work file. + * @param dstArchiveFile Dst archive file. + */ + SegmentArchiveResult(long absWalIdx, File origWorkFile, File dstArchiveFile) { + this.absWalIdx = absWalIdx; + this.origWorkFile = origWorkFile; + this.dstArchiveFile = dstArchiveFile; + } + + /** @return {@link #absWalIdx} */ + long getAbsIdx() { + return absWalIdx; + } + + /** @return {@link #origWorkFile} */ + File getOrigWorkFile() { + return origWorkFile; + } + + /** @return {@link #dstArchiveFile} */ + File getDstArchiveFile() { + return dstArchiveFile; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentEofException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentEofException.java index 80c375e2ad205..2f58e3de7b087 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentEofException.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentEofException.java @@ -21,7 +21,8 @@ /** * This exception is thrown either when we reach the end of file of WAL segment, or when we encounter - * a record with type equal to {@code 0}. + * a record with type equal to + * {@link org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType#STOP_ITERATION_RECORD_TYPE} */ public class SegmentEofException extends IgniteCheckedException { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java new file mode 100644 index 0000000000000..8ea058560391f --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.wal.reader; + +import java.io.File; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.MemoryConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.pagemem.wal.WALIterator; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.jetbrains.annotations.NotNull; + +/** + * Factory for creating iterator over WAL files + */ +public class IgniteWalIteratorFactory { + /** Logger. */ + private final IgniteLogger log; + /** Page size, in standalone iterator mode this value can't be taken from memory configuration */ + private final int pageSize; + + /** + * Creates WAL files iterator factory + * @param log Logger. + * @param pageSize Page size, size is validated + */ + public IgniteWalIteratorFactory(@NotNull final IgniteLogger log, final int pageSize) { + this.log = log; + this.pageSize = pageSize; + new MemoryConfiguration().setPageSize(pageSize); // just for validate + } + + /** + * Creates iterator for (archive) directory scan mode. + * Note in this mode total scanned files at end of iteration may be wider that initial files in directory. + * This mode does not support work directory scan because work directory contains unpredictable number in file name. + * Such file may broke iteration. + * + * @param walDirWithConsistentId directory with WAL files. Should already contain node consistent ID as subfolder + * @return closable WAL records iterator, should be closed when non needed + * @throws IgniteCheckedException if failed to read folder + */ + public WALIterator iteratorArchiveDirectory(@NotNull final File walDirWithConsistentId) throws IgniteCheckedException { + return new StandaloneWalRecordsIterator(walDirWithConsistentId, log, prepareSharedCtx()); + } + + /** + * Creates iterator for file by file scan mode. + * This method may be used only for archive folder (not for work). + * In this mode only provided WAL segments will be scanned. New WAL files created during iteration will be ignored + * @param files files to scan. Order it not important, but is significant to provide all segments without omissions + * @return closable WAL records iterator, should be closed when non needed + * @throws IgniteCheckedException if failed to read files + */ + public WALIterator iteratorArchiveFiles(@NotNull final File ...files) throws IgniteCheckedException { + return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), false, files); + } + + /** + * Creates iterator for file by file scan mode. + * This method may be used for work folder, file indexes are scanned from the file context. + * In this mode only provided WAL segments will be scanned. New WAL files created during iteration will be ignored. + * @param files files to scan. Order it not important, but is significant to provide all segments without omissions + * @return closable WAL records iterator, should be closed when non needed + * @throws IgniteCheckedException if failed to read files + */ + public WALIterator iteratorWorkFiles(@NotNull final File ...files) throws IgniteCheckedException { + return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), true, files); + } + + /** + * @return fake shared context required for create minimal services for record reading + */ + @NotNull private GridCacheSharedContext prepareSharedCtx() { + final GridKernalContext kernalCtx = new StandaloneGridKernalContext(log); + + final StandaloneIgniteCacheDatabaseSharedManager dbMgr = new StandaloneIgniteCacheDatabaseSharedManager(); + + dbMgr.setPageSize(pageSize); + return new GridCacheSharedContext<>( + kernalCtx, null, null, null, + null, null, dbMgr, null, + null, null, null, null, + null, null, null); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java new file mode 100644 index 0000000000000..df932e600cfb2 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java @@ -0,0 +1,499 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.wal.reader; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.GridComponent; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.GridKernalGateway; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.MarshallerContextImpl; +import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager; +import org.apache.ignite.internal.managers.collision.GridCollisionManager; +import org.apache.ignite.internal.managers.communication.GridIoManager; +import org.apache.ignite.internal.managers.deployment.GridDeploymentManager; +import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; +import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager; +import org.apache.ignite.internal.managers.failover.GridFailoverManager; +import org.apache.ignite.internal.managers.indexing.GridIndexingManager; +import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager; +import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor; +import org.apache.ignite.internal.processors.cache.GridCacheProcessor; +import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; +import org.apache.ignite.internal.processors.closure.GridClosureProcessor; +import org.apache.ignite.internal.processors.cluster.ClusterProcessor; +import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor; +import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor; +import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor; +import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor; +import org.apache.ignite.internal.processors.hadoop.HadoopHelper; +import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter; +import org.apache.ignite.internal.processors.igfs.IgfsHelper; +import org.apache.ignite.internal.processors.igfs.IgfsProcessorAdapter; +import org.apache.ignite.internal.processors.job.GridJobProcessor; +import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor; +import org.apache.ignite.internal.processors.marshaller.GridMarshallerMappingProcessor; +import org.apache.ignite.internal.processors.odbc.SqlListenerProcessor; +import org.apache.ignite.internal.processors.platform.PlatformProcessor; +import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor; +import org.apache.ignite.internal.processors.pool.PoolProcessor; +import org.apache.ignite.internal.processors.port.GridPortProcessor; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; +import org.apache.ignite.internal.processors.resource.GridResourceProcessor; +import org.apache.ignite.internal.processors.rest.GridRestProcessor; +import org.apache.ignite.internal.processors.schedule.IgniteScheduleProcessorAdapter; +import org.apache.ignite.internal.processors.security.GridSecurityProcessor; +import org.apache.ignite.internal.processors.segmentation.GridSegmentationProcessor; +import org.apache.ignite.internal.processors.service.GridServiceProcessor; +import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor; +import org.apache.ignite.internal.processors.task.GridTaskProcessor; +import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor; +import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions; +import org.apache.ignite.internal.util.IgniteExceptionRegistry; +import org.apache.ignite.internal.util.StripedExecutor; +import org.apache.ignite.plugin.PluginNotFoundException; +import org.apache.ignite.plugin.PluginProvider; +import org.apache.ignite.thread.IgniteStripedThreadPoolExecutor; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Dummy grid kernal context + */ +public class StandaloneGridKernalContext implements GridKernalContext { + private IgniteLogger log; + + /** + * @param log Logger. + */ + StandaloneGridKernalContext(IgniteLogger log) { + this.log = log; + } + + /** {@inheritDoc} */ + @Override public List components() { + return null; + } + + /** {@inheritDoc} */ + @Override public UUID localNodeId() { + return null; + } + + /** {@inheritDoc} */ + @Override public String igniteInstanceName() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgniteLogger log(String ctgr) { + return log; + } + + /** {@inheritDoc} */ + @Override public IgniteLogger log(Class cls) { + return log; + } + + /** {@inheritDoc} */ + @Override public boolean isStopping() { + return false; + } + + /** {@inheritDoc} */ + @Override public GridKernalGateway gateway() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgniteEx grid() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgniteConfiguration config() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridTaskProcessor task() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridAffinityProcessor affinity() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridJobProcessor job() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridTimeoutProcessor timeout() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridResourceProcessor resource() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridJobMetricsProcessor jobMetric() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridCacheProcessor cache() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridClusterStateProcessor state() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridTaskSessionProcessor session() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridClosureProcessor closure() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridServiceProcessor service() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridPortProcessor ports() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgniteScheduleProcessorAdapter schedule() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridRestProcessor rest() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridSegmentationProcessor segmentation() { + return null; + } + + /** {@inheritDoc} */ + @Override public DataStreamProcessor dataStream() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgfsProcessorAdapter igfs() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgfsHelper igfsHelper() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridContinuousProcessor continuous() { + return null; + } + + /** {@inheritDoc} */ + @Override public HadoopProcessorAdapter hadoop() { + return null; + } + + /** {@inheritDoc} */ + @Override public PoolProcessor pools() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridMarshallerMappingProcessor mapping() { + return null; + } + + /** {@inheritDoc} */ + @Override public HadoopHelper hadoopHelper() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService utilityCachePool() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgniteStripedThreadPoolExecutor asyncCallbackPool() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgniteCacheObjectProcessor cacheObjects() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridQueryProcessor query() { + return null; + } + + /** {@inheritDoc} */ + @Override public SqlListenerProcessor sqlListener() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgnitePluginProcessor plugins() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridDeploymentManager deploy() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridIoManager io() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridDiscoveryManager discovery() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridCheckpointManager checkpoint() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridEventStorageManager event() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridFailoverManager failover() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridCollisionManager collision() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridSecurityProcessor security() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridLoadBalancerManager loadBalancing() { + return null; + } + + /** {@inheritDoc} */ + @Override public GridIndexingManager indexing() { + return null; + } + + /** {@inheritDoc} */ + @Override public DataStructuresProcessor dataStructures() { + return null; + } + + /** {@inheritDoc} */ + @Override public void markSegmented() { } + + /** {@inheritDoc} */ + @Override public boolean segmented() { + return false; + } + + /** {@inheritDoc} */ + @Override public void printMemoryStats() { } + + /** {@inheritDoc} */ + @Override public boolean isDaemon() { + return false; + } + + /** {@inheritDoc} */ + @Override public GridPerformanceSuggestions performance() { + return null; + } + + /** {@inheritDoc} */ + @Override public String userVersion(ClassLoader ldr) { + return null; + } + + /** {@inheritDoc} */ + @Override public PluginProvider pluginProvider(String name) throws PluginNotFoundException { + return null; + } + + /** {@inheritDoc} */ + @Override public T createComponent(Class cls) { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getServiceExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getSystemExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public StripedExecutor getStripedExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getManagementExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getPeerClassLoadingExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getIgfsExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getDataStreamerExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getRestExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getAffinityExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Nullable @Override public ExecutorService getIndexingExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getQueryExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Nullable @Override public Map customExecutors() { + return null; + } + + /** {@inheritDoc} */ + @Override public ExecutorService getSchemaExecutorService() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgniteExceptionRegistry exceptionRegistry() { + return null; + } + + /** {@inheritDoc} */ + @Override public Object nodeAttribute(String key) { + return null; + } + + /** {@inheritDoc} */ + @Override public boolean hasNodeAttribute(String key) { + return false; + } + + /** {@inheritDoc} */ + @Override public Object addNodeAttribute(String key, Object val) { + return null; + } + + /** {@inheritDoc} */ + @Override public Map nodeAttributes() { + return null; + } + + /** {@inheritDoc} */ + @Override public ClusterProcessor cluster() { + return null; + } + + /** {@inheritDoc} */ + @Override public MarshallerContextImpl marshallerContext() { + return null; + } + + /** {@inheritDoc} */ + @Override public boolean clientNode() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean clientDisconnected() { + return false; + } + + /** {@inheritDoc} */ + @Override public PlatformProcessor platform() { + return null; + } + + /** {@inheritDoc} */ + @NotNull @Override public Iterator iterator() { + return null; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneIgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneIgniteCacheDatabaseSharedManager.java new file mode 100644 index 0000000000000..85a872453e61b --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneIgniteCacheDatabaseSharedManager.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.wal.reader; + +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; + +/** + * Fake implementation for publishing setter and for creation in standalone WAL reader tool + */ +class StandaloneIgniteCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedManager { + /** {@inheritDoc} */ + @Override public void setPageSize(int pageSize) { + super.setPageSize(pageSize); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java new file mode 100644 index 0000000000000..f17c1122aa666 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.wal.reader; + +import java.io.DataInput; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; +import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException; +import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer; +import org.apache.ignite.internal.util.typedef.F; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE; + +/** + * WAL reader iterator, for creation in standalone WAL reader tool + * Operates over one directory, does not provide start and end boundaries + */ +class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { + /** */ + private static final long serialVersionUID = 0L; + + /** Record buffer size */ + private static final int BUF_SIZE = 2 * 1024 * 1024; + + /** + * WAL files directory. Should already contain 'consistent ID' as subfolder. + * null value means file-by-file iteration mode + */ + @Nullable + private File walFilesDir; + + /** + * File descriptors remained to scan. + * null value means directory scan mode + */ + @Nullable + private List walFileDescriptors; + + /** + * True if this iterator used for work dir, false for archive. + * In work dir mode exceptions come from record reading are ignored (file may be not completed). + * Index of file is taken from file itself, not from file name + */ + private boolean workDir; + + /** + * Creates iterator in directory scan mode + * + * @param walFilesDir Wal files directory. Should already contain node consistent ID as subfolder + * @param log Logger. + * @param sharedCtx Shared context. + */ + StandaloneWalRecordsIterator( + @NotNull final File walFilesDir, + @NotNull final IgniteLogger log, + @NotNull final GridCacheSharedContext sharedCtx) throws IgniteCheckedException { + super(log, + sharedCtx, + new RecordV1Serializer(sharedCtx), + BUF_SIZE); + init(walFilesDir, false, null); + advance(); + } + + /** + * Creates iterator in file-by-file iteration mode. Directory + * + * @param log Logger. + * @param sharedCtx Shared context. + * @param workDir Work directory is scanned, false - archive + * @param walFiles Wal files. + */ + StandaloneWalRecordsIterator( + @NotNull final IgniteLogger log, + @NotNull final GridCacheSharedContext sharedCtx, + final boolean workDir, + @NotNull final File... walFiles) throws IgniteCheckedException { + super(log, + sharedCtx, + new RecordV1Serializer(sharedCtx), + BUF_SIZE); + this.workDir = workDir; + init(null, workDir, walFiles); + advance(); + } + + /** + * For directory mode sets oldest file as initial segment, + * for file by file mode, converts all files to descriptors and gets oldest as initial. + * + * @param walFilesDir directory for directory scan mode + * @param workDir work directory, only for file-by-file mode + * @param walFiles files for file-by-file iteration mode + */ + private void init( + @Nullable final File walFilesDir, + final boolean workDir, + @Nullable final File[] walFiles) throws IgniteCheckedException { + if (walFilesDir != null) { + FileWriteAheadLogManager.FileDescriptor[] descs = loadFileDescriptors(walFilesDir); + curWalSegmIdx = !F.isEmpty(descs) ? descs[0].getIdx() : 0; + this.walFilesDir = walFilesDir; + this.workDir = false; + } + else { + this.workDir = workDir; + if (workDir) + walFileDescriptors = scanIndexesFromFileHeaders(walFiles); + else + walFileDescriptors = new ArrayList<>(Arrays.asList(FileWriteAheadLogManager.scan(walFiles))); + curWalSegmIdx = !walFileDescriptors.isEmpty() ? walFileDescriptors.get(0).getIdx() : 0; + } + curWalSegmIdx--; + + if (log.isDebugEnabled()) + log.debug("Initialized WAL cursor [curWalSegmIdx=" + curWalSegmIdx + ']'); + } + + /** + * This methods checks all provided files to be correct WAL segment. + * Header record and its position is checked. WAL position is used to deremine real index. + * File index from file name is ignored. + * + * @param allFiles files to scan + * @return list of file descriptors with checked header records, file index is set + * @throws IgniteCheckedException if IO error occurs + */ + private List scanIndexesFromFileHeaders( + @Nullable final File[] allFiles) throws IgniteCheckedException { + if (allFiles == null || allFiles.length == 0) + return Collections.emptyList(); + + final List resultingDescs = new ArrayList<>(); + + for (File file : allFiles) { + if (file.length() < HEADER_RECORD_SIZE) + continue; + + FileWALPointer ptr; + + try (RandomAccessFile rf = new RandomAccessFile(file, "r");) { + final FileChannel ch = rf.getChannel(); + final ByteBuffer buf = ByteBuffer.allocate(HEADER_RECORD_SIZE); + + buf.order(ByteOrder.nativeOrder()); + + final DataInput in = new FileInput(ch, buf); + // Header record must be agnostic to the serializer version. + final int type = in.readUnsignedByte(); + + if (type == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE) + throw new SegmentEofException("Reached logical end of the segment", null); + ptr = RecordV1Serializer.readPosition(in); + } + catch (IOException e) { + throw new IgniteCheckedException("Failed to scan index from file [" + file + "]", e); + } + + resultingDescs.add(new FileWriteAheadLogManager.FileDescriptor(file, ptr.index())); + } + Collections.sort(resultingDescs); + return resultingDescs; + } + + /** {@inheritDoc} */ + @Override protected FileWriteAheadLogManager.ReadFileHandle advanceSegment( + @Nullable final FileWriteAheadLogManager.ReadFileHandle curWalSegment) throws IgniteCheckedException { + + if (curWalSegment != null) + curWalSegment.close(); + + curWalSegmIdx++; + // curHandle.workDir is false + final FileWriteAheadLogManager.FileDescriptor fd; + + if (walFilesDir != null) { + fd = new FileWriteAheadLogManager.FileDescriptor( + new File(walFilesDir, + FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx))); + } + else { + if (walFileDescriptors.isEmpty()) + return null; //no files to read, stop iteration + + fd = walFileDescriptors.remove(0); + } + + if (log.isDebugEnabled()) + log.debug("Reading next file [absIdx=" + curWalSegmIdx + ", file=" + fd.getAbsolutePath() + ']'); + + assert fd != null; + + curRec = null; + try { + return initReadHandle(fd, null); + } + catch (FileNotFoundException e) { + log.info("Missing WAL segment in the archive: " + e.getMessage()); + return null; + } + } + + /** {@inheritDoc} */ + @Override protected void handleRecordException( + @NotNull final Exception e, + @Nullable final FileWALPointer ptr) { + super.handleRecordException(e, ptr); + final RuntimeException ex = new RuntimeException("Record reading problem occurred at file pointer [" + ptr + "]:" + e.getMessage(), e); + + ex.printStackTrace(); + if (!workDir) + throw ex; + } + + /** {@inheritDoc} */ + @Override protected void onClose() throws IgniteCheckedException { + super.onClose(); + curRec = null; + + closeCurrentWalSegment(); + + curWalSegmIdx = Integer.MAX_VALUE; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index 0ccd3a08218e5..0a7b3dd57fd28 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -103,21 +103,37 @@ /** * Record V1 serializer. + * Stores records in following format: + *
    + *
  • Record type from {@link RecordType#ordinal()} incremented by 1
    • + *
  • WAL pointer to double check consistency
    • + *
  • Data
    • + *
  • CRC or zero padding
    • + *
*/ public class RecordV1Serializer implements RecordSerializer { - /** */ - public static final int HEADER_RECORD_SIZE = /*Type*/1 + /*Pointer */12 + /*Magic*/8 + /*Version*/4 + /*CRC*/4; + /** Length of Type */ + public static final int REC_TYPE_SIZE = 1; + + /** Length of WAL Pointer */ + public static final int FILE_WAL_POINTER_SIZE = 12; + + /** Length of CRC value */ + private static final int CRC_SIZE = 4; /** */ + public static final int HEADER_RECORD_SIZE = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + /*Magic*/8 + /*Version*/4 + CRC_SIZE; + + /** Cache shared context */ private GridCacheSharedContext cctx; - /** */ + /** Size of page used for PageMemory regions */ private int pageSize; - /** */ + /** Cache object processor to reading {@link DataEntry DataEntries} */ private IgniteCacheObjectProcessor co; - /** */ + /** Skip CRC calculation/check flag */ private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false); /** @@ -658,7 +674,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { assert res != null; - res.size((int)(in0.position() - startPos + 4)); // Account for CRC which will be read afterwards. + res.size((int)(in0.position() - startPos + CRC_SIZE)); // Account for CRC which will be read afterwards. return res; } @@ -671,12 +687,16 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { } /** - * @param in In. + * Loads record from input, does not read CRC value + * + * @param in Input to read record from + * @param expPtr expected WAL pointer for record. Used to validate actual position against expected from the file + * @throws SegmentEofException if end of WAL segment reached */ private WALRecord readRecord(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException { int type = in.readUnsignedByte(); - if (type == 0) + if (type == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE) throw new SegmentEofException("Reached logical end of the segment", null); FileWALPointer ptr = readPosition(in); @@ -1212,7 +1232,7 @@ private WALRecord readRecord(ByteBufferBackedDataInput in, WALPointer expPtr) th /** {@inheritDoc} */ @SuppressWarnings("CastConflictsWithInstanceof") @Override public int size(WALRecord record) throws IgniteCheckedException { - int commonFields = /* Type */1 + /* Pointer */12 + /*CRC*/4; + int commonFields = REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE; switch (record.type()) { case PAGE_RECORD: @@ -1371,7 +1391,7 @@ assert record instanceof PageSnapshot; return commonFields + /*cacheId*/ 4 + /*pageId*/ 8; case SWITCH_SEGMENT_RECORD: - return commonFields; + return commonFields - CRC_SIZE; //CRC is not loaded for switch segment, exception is thrown instead default: throw new UnsupportedOperationException("Type: " + record.type()); @@ -1379,10 +1399,11 @@ assert record instanceof PageSnapshot; } /** + * Saves position, WAL pointer (requires {@link #FILE_WAL_POINTER_SIZE} bytes) * @param buf Byte buffer to serialize version to. * @param ptr File WAL pointer to write. */ - private void putPosition(ByteBuffer buf, FileWALPointer ptr) { + public static void putPosition(ByteBuffer buf, FileWALPointer ptr) { buf.putLong(ptr.index()); buf.putInt(ptr.fileOffset()); } @@ -1392,7 +1413,7 @@ private void putPosition(ByteBuffer buf, FileWALPointer ptr) { * @return Read file WAL pointer. * @throws IOException If failed to write. */ - private FileWALPointer readPosition(DataInput in) throws IOException { + public static FileWALPointer readPosition(DataInput in) throws IOException { long idx = in.readLong(); int fileOffset = in.readInt(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java index 5561d95b38ce1..fed87666e6b03 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java @@ -181,6 +181,8 @@ public void testSet() throws Exception { for (int i = 0; i < 100; i++) set.add(i); + assertEquals(100, set.size()); + stopAllGrids(); ignite = startGrids(4); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java index 793806e18b652..48d8c21c48417 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java @@ -297,7 +297,7 @@ public void testNodeLeftDuringExchange() throws Exception { final int entryCnt = 10_000; final int initGridCnt = 4; - final IgniteEx ig0 = (IgniteEx)startGrids(initGridCnt); + final Ignite ig0 = startGrids(initGridCnt); ig0.active(true); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java new file mode 100644 index 0000000000000..06bcf08fa47ec --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db.wal.reader; + +import java.io.File; +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteEvents; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheRebalanceMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.configuration.BinaryConfiguration; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.MemoryConfiguration; +import org.apache.ignite.configuration.MemoryPolicyConfiguration; +import org.apache.ignite.configuration.PersistentStoreConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.events.Event; +import org.apache.ignite.events.EventType; +import org.apache.ignite.internal.pagemem.wal.WALIterator; +import org.apache.ignite.internal.pagemem.wal.WALPointer; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; +import org.apache.ignite.events.WalSegmentArchivedEvent; +import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED; + +/** + * Test suite for WAL segments reader and event generator. + */ +public class IgniteWalReaderTest extends GridCommonAbstractTest { + /** Wal segments count */ + private static final int WAL_SEGMENTS = 10; + + /** Cache name. */ + private static final String CACHE_NAME = "cache0"; + + /** Fill wal with some data before iterating. Should be true for non local run */ + private static final boolean fillWalBeforeTest = true; + + /** Delete DB dir before test. */ + private static final boolean deleteBefore = true; + + /** Delete DB dir after test. */ + private static final boolean deleteAfter = true; + + /** Dump records to logger. Should be false for non local run */ + private static final boolean dumpRecords = false; + + /** Page size to set */ + public static final int PAGE_SIZE = 4 * 1024; + + /** + * Field for transferring setting from test to getConfig method + * Archive incomplete segment after inactivity milliseconds. + */ + private int archiveIncompleteSegmentAfterInactivityMs = 0; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + final IgniteConfiguration cfg = super.getConfiguration(gridName); + + final CacheConfiguration ccfg = new CacheConfiguration<>(CACHE_NAME); + + ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC); + ccfg.setRebalanceMode(CacheRebalanceMode.SYNC); + ccfg.setAffinity(new RendezvousAffinityFunction(false, 32)); + ccfg.setIndexedTypes(Integer.class, IgniteWalReaderTest.IndexedObject.class); + + cfg.setCacheConfiguration(ccfg); + + cfg.setIncludeEventTypes(EventType.EVT_WAL_SEGMENT_ARCHIVED); + + final MemoryConfiguration dbCfg = new MemoryConfiguration(); + + dbCfg.setPageSize(PAGE_SIZE); + + final MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration(); + + memPlcCfg.setName("dfltMemPlc"); + memPlcCfg.setInitialSize(1024 * 1024 * 1024); + memPlcCfg.setMaxSize(1024 * 1024 * 1024); + + dbCfg.setMemoryPolicies(memPlcCfg); + dbCfg.setDefaultMemoryPolicyName("dfltMemPlc"); + + cfg.setMemoryConfiguration(dbCfg); + + final PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration(); + pCfg.setWalHistorySize(1); + pCfg.setWalSegmentSize(1024 * 1024); + pCfg.setWalSegments(WAL_SEGMENTS); + pCfg.setWalMode(WALMode.BACKGROUND); + + if (archiveIncompleteSegmentAfterInactivityMs > 0) + pCfg.setWalAutoArchiveAfterInactivity(archiveIncompleteSegmentAfterInactivityMs); + + cfg.setPersistentStoreConfiguration(pCfg); + + final BinaryConfiguration binCfg = new BinaryConfiguration(); + + binCfg.setCompactFooter(false); + + cfg.setBinaryConfiguration(binCfg); + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + stopAllGrids(); + + if (deleteBefore) + deleteWorkFiles(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + if (deleteAfter) + deleteWorkFiles(); + } + + /** + * @throws IgniteCheckedException If failed. + */ + private void deleteWorkFiles() throws IgniteCheckedException { + if (fillWalBeforeTest) + deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); + } + + /** + * @throws Exception if failed. + */ + public void testFillWalAndReadRecords() throws Exception { + final int cacheObjectsToWrite = 10000; + + if (fillWalBeforeTest) { + final Ignite ignite0 = startGrid("node0"); + + ignite0.active(true); + + putDummyRecords(ignite0, cacheObjectsToWrite); + + stopGrid("node0"); + } + + final File db = U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false); + final File wal = new File(db, "wal"); + final File walArchive = new File(wal, "archive"); + final String consistentId = "127_0_0_1_47500"; + final MockWalIteratorFactory mockItFactory = new MockWalIteratorFactory(log, PAGE_SIZE, consistentId, WAL_SEGMENTS); + final WALIterator it = mockItFactory.iterator(wal, walArchive); + final int cntUsingMockIter = iterateAndCount(it); + + log.info("Total records loaded " + cntUsingMockIter); + assert cntUsingMockIter > 0; + assert cntUsingMockIter > cacheObjectsToWrite; + + final File walArchiveDirWithConsistentId = new File(walArchive, consistentId); + final File walWorkDirWithConsistentId = new File(wal, consistentId); + + final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE); + final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId)); + + log.info("Total records loaded using directory : " + cntArchiveDir); + + final int cntArchiveFileByFile = iterateAndCount( + factory.iteratorArchiveFiles( + walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER))); + + log.info("Total records loaded using archive directory (file-by-file): " + cntArchiveFileByFile); + + assert cntArchiveFileByFile > cacheObjectsToWrite; + assert cntArchiveDir > cacheObjectsToWrite; + assert cntArchiveDir == cntArchiveFileByFile; + //really count2 may be less because work dir correct loading is not supported yet + assert cntUsingMockIter >= cntArchiveDir + : "Mock based reader loaded " + cntUsingMockIter + " records but standalone has loaded only " + cntArchiveDir; + + + final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER); + int cntWork = 0; + + try (WALIterator stIt = factory.iteratorWorkFiles(workFiles)) { + while (stIt.hasNextX()) { + IgniteBiTuple next = stIt.nextX(); + if (dumpRecords) + log.info("Work. Record: " + next.get2()); + cntWork++; + } + } + log.info("Total records loaded from work: " + cntWork); + + assert cntWork + cntArchiveFileByFile == cntUsingMockIter + : "Work iterator loaded [" + cntWork + "] " + + "Archive iterator loaded [" + cntArchiveFileByFile + "]; " + + "mock iterator [" + cntUsingMockIter + "]"; + + } + + /** + * @param walIter iterator to count, will be closed + * @return count of records + * @throws IgniteCheckedException if failed to iterate + */ + private int iterateAndCount(WALIterator walIter) throws IgniteCheckedException { + int cntUsingMockIter = 0; + + try(WALIterator it = walIter) { + while (it.hasNextX()) { + IgniteBiTuple next = it.nextX(); + if (dumpRecords) + log.info("Record: " + next.get2()); + cntUsingMockIter++; + } + } + return cntUsingMockIter; + } + + /** + * Tests archive completed event is fired + * + * @throws Exception if failed + */ + public void testArchiveCompletedEventFired() throws Exception { + final AtomicBoolean evtRecorded = new AtomicBoolean(); + + final Ignite ignite = startGrid("node0"); + + ignite.active(true); + + final IgniteEvents evts = ignite.events(); + + if (!evts.isEnabled(EVT_WAL_SEGMENT_ARCHIVED)) + return; //nothing to test + + evts.localListen(new IgnitePredicate() { + @Override public boolean apply(Event e) { + WalSegmentArchivedEvent archComplEvt = (WalSegmentArchivedEvent)e; + long idx = archComplEvt.getAbsWalSegmentIdx(); + log.info("Finished archive for segment [" + idx + ", " + + archComplEvt.getArchiveFile() + "]: [" + e + "]"); + + evtRecorded.set(true); + return true; + } + }, EVT_WAL_SEGMENT_ARCHIVED); + + putDummyRecords(ignite, 150); + + stopGrid("node0"); + assert evtRecorded.get(); + } + + /** + * Puts provided number of records to fill WAL + * + * @param ignite ignite instance + * @param recordsToWrite count + */ + private void putDummyRecords(Ignite ignite, int recordsToWrite) { + IgniteCache cache0 = ignite.cache(CACHE_NAME); + + for (int i = 0; i < recordsToWrite; i++) + cache0.put(i, new IndexedObject(i)); + } + + /** + * Tests time out based WAL segment archiving + * + * @throws Exception if failure occurs + */ + public void testArchiveIncompleteSegmentAfterInactivity() throws Exception { + final AtomicBoolean waitingForEvt = new AtomicBoolean(); + final CountDownLatch archiveSegmentForInactivity = new CountDownLatch(1); + + archiveIncompleteSegmentAfterInactivityMs = 1000; + + final Ignite ignite = startGrid("node0"); + + ignite.active(true); + + final IgniteEvents evts = ignite.events(); + + evts.localListen(new IgnitePredicate() { + @Override public boolean apply(Event e) { + WalSegmentArchivedEvent archComplEvt = (WalSegmentArchivedEvent)e; + long idx = archComplEvt.getAbsWalSegmentIdx(); + log.info("Finished archive for segment [" + idx + ", " + + archComplEvt.getArchiveFile() + "]: [" + e + "]"); + + if (waitingForEvt.get()) + archiveSegmentForInactivity.countDown(); + return true; + } + }, EVT_WAL_SEGMENT_ARCHIVED); + + putDummyRecords(ignite, 100); + waitingForEvt.set(true); //flag for skipping regular log() and rollOver() + + log.info("Wait for archiving segment for inactive grid started"); + + boolean recordedAfterSleep = + archiveSegmentForInactivity.await(archiveIncompleteSegmentAfterInactivityMs + 1001, TimeUnit.MILLISECONDS); + + stopGrid("node0"); + assert recordedAfterSleep; + } + + /** Test object for placing into grid in this test */ + private static class IndexedObject { + /** */ + @QuerySqlField(index = true) + private int iVal; + + /** Data filled with recognizable pattern */ + private byte[] data; + + /** + * @param iVal Integer value. + */ + private IndexedObject(int iVal) { + this.iVal = iVal; + int sz = 40000; + data = new byte[sz]; + for (int i = 0; i < sz; i++) + data[i] = (byte)('A' + (i % 10)); + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + IndexedObject obj = (IndexedObject)o; + + if (iVal != obj.iVal) + return false; + return Arrays.equals(data, obj.data); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = iVal; + res = 31 * res + Arrays.hashCode(data); + return res; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(IgniteWalReaderTest.IndexedObject.class, this); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java new file mode 100644 index 0000000000000..95079a0049b8f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db.wal.reader; + +import java.io.File; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.PersistentStoreConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.WALIterator; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; +import org.jetbrains.annotations.Nullable; +import org.mockito.Mockito; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; + +/** + * Mockito based WAL iterator provider + */ +public class MockWalIteratorFactory { + /** Logger. */ + private final IgniteLogger log; + + /** Page size. */ + private final int pageSize; + + /** Consistent node id. */ + private final String consistentId; + + /** Segments count in work dir. */ + private int segments; + + /** + * Creates factory + * @param log Logger. + * @param pageSize Page size. + * @param consistentId Consistent id. + * @param segments Segments. + */ + public MockWalIteratorFactory(@Nullable IgniteLogger log, int pageSize, String consistentId, int segments) { + this.log = log == null ? Mockito.mock(IgniteLogger.class) : log; + this.pageSize = pageSize; + this.consistentId = consistentId; + this.segments = segments; + } + + /** + * Creates iterator + * @param wal WAL directory without node id + * @param walArchive WAL archive without node id + * @return iterator + * @throws IgniteCheckedException if IO failed + */ + public WALIterator iterator(File wal, File walArchive) throws IgniteCheckedException { + final PersistentStoreConfiguration persistentCfg1 = Mockito.mock(PersistentStoreConfiguration.class); + + when(persistentCfg1.getWalStorePath()).thenReturn(wal.getAbsolutePath()); + when(persistentCfg1.getWalArchivePath()).thenReturn(walArchive.getAbsolutePath()); + when(persistentCfg1.getWalSegments()).thenReturn(segments); + when(persistentCfg1.getTlbSize()).thenReturn(PersistentStoreConfiguration.DFLT_TLB_SIZE); + when(persistentCfg1.getWalRecordIteratorBufferSize()).thenReturn(PersistentStoreConfiguration.DFLT_WAL_RECORD_ITERATOR_BUFFER_SIZE); + + final IgniteConfiguration cfg = Mockito.mock(IgniteConfiguration.class); + + when(cfg.getPersistentStoreConfiguration()).thenReturn(persistentCfg1); + + final GridKernalContext ctx = Mockito.mock(GridKernalContext.class); + + when(ctx.config()).thenReturn(cfg); + when(ctx.clientNode()).thenReturn(false); + + final GridDiscoveryManager disco = Mockito.mock(GridDiscoveryManager.class); + + when(disco.consistentId()).thenReturn(consistentId); + when(ctx.discovery()).thenReturn(disco); + + final IgniteWriteAheadLogManager mgr = new FileWriteAheadLogManager(ctx); + final GridCacheSharedContext sctx = Mockito.mock(GridCacheSharedContext.class); + + when(sctx.kernalContext()).thenReturn(ctx); + when(sctx.discovery()).thenReturn(disco); + + final GridCacheDatabaseSharedManager database = Mockito.mock(GridCacheDatabaseSharedManager.class); + + when(database.pageSize()).thenReturn(pageSize); + when(sctx.database()).thenReturn(database); + when(sctx.logger(any(Class.class))).thenReturn(log); + + mgr.start(sctx); + + return mgr.replay(null); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index 351f52eb904c2..801870595dcc2 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -18,22 +18,18 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; -import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsAtomicCacheRebalancingTest; -import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheRebalancingTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistenceMetricsSelfTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionDuringPartitionClearTest; +import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest; -import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest; -import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsNoActualWalHistoryTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest; -import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryTest; -import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRecoveryTxLogicalRecordsTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteDataIntegrityTests; +import org.apache.ignite.internal.processors.cache.persistence.db.wal.reader.IgniteWalReaderTest; /** * @@ -69,6 +65,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgnitePersistentStoreDataStructuresTest.class); + suite.addTestSuite(IgniteWalReaderTest.class); return suite; } } From 90b67fa1b72b096943499e26e402988840bdbe97 Mon Sep 17 00:00:00 2001 From: Sergey Chugunov Date: Tue, 4 Jul 2017 20:47:09 +0300 Subject: [PATCH 053/155] IGNITE-4536 two tests unmuted, one test removed as obsolete (IGNITE-5592) - Fixes #2235. Signed-off-by: Alexey Goncharuk --- .../GridCacheAbstractMetricsSelfTest.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java index 2e81b9607ba03..6aed380fd8f53 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -581,8 +580,6 @@ public void testMissHitPercentage() throws Exception { * @throws Exception If failed. */ public void testMisses() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-4536"); - IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); int keyCnt = keyCount(); @@ -625,8 +622,6 @@ public void testMisses() throws Exception { * @throws Exception If failed. */ public void testMissesOnEmptyCache() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-4536"); - IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); assertEquals("Expected 0 read", 0, cache.localMetrics().getCacheGets()); @@ -675,25 +670,6 @@ public void testRemoves() throws Exception { assertEquals(1L, cache.localMetrics().getCacheRemovals()); } - /** - * @throws Exception If failed. - */ - public void testManualEvictions() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-4536"); - - IgniteCache cache = grid(0).cache(DEFAULT_CACHE_NAME); - - if (cache.getConfiguration(CacheConfiguration.class).getCacheMode() == CacheMode.PARTITIONED) - return; - - cache.put(1, 1); - - cache.localEvict(Collections.singleton(1)); - - assertEquals(0L, cache.localMetrics().getCacheRemovals()); - assertEquals(1L, cache.localMetrics().getCacheEvictions()); - } - /** * @throws Exception If failed. */ From 50bb0901255707dc9601159803363a36aeafecab Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Tue, 4 Jul 2017 18:10:25 +0300 Subject: [PATCH 054/155] Muted hanging tests. --- .../IgniteCommunicationBalanceMultipleConnectionsTest.java | 5 +++++ ...IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java | 5 +++++ .../apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java index e95b1ec0b514c..444f086522a8b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/IgniteCommunicationBalanceMultipleConnectionsTest.java @@ -25,4 +25,9 @@ public class IgniteCommunicationBalanceMultipleConnectionsTest extends IgniteCom @Override protected int connectionsPerNode() { return 5; } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-5689"); + } } diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java index 5154642b2a62b..edf02794316e8 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java @@ -30,4 +30,9 @@ public class IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest public IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest() { super(DUAL_ASYNC, true); } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-5691"); + } } \ No newline at end of file diff --git a/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java b/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java index 49bb1ac63ed61..5477d4384246d 100644 --- a/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java +++ b/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java @@ -99,6 +99,11 @@ public JavaEmbeddedIgniteRDDSelfTest() { super(false); } + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-5690"); + } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); From 5b7165ca717de038249570c4213f050ba9b5aee7 Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Tue, 4 Jul 2017 20:57:56 +0300 Subject: [PATCH 055/155] IGNITE-5693 - Fixed semaphore flaky test - Fixes #2239. Signed-off-by: Alexey Goncharuk --- .../IgniteSemaphoreAbstractSelfTest.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteSemaphoreAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteSemaphoreAbstractSelfTest.java index d5f4e0e69aca5..445d469952409 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteSemaphoreAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteSemaphoreAbstractSelfTest.java @@ -35,6 +35,7 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteKernal; +import org.apache.ignite.internal.util.lang.GridAbsPredicateX; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.PA; import org.apache.ignite.lang.IgniteCallable; @@ -437,8 +438,7 @@ private IgniteSemaphore createSemaphore(String semaphoreName, int numPermissions * @param semaphoreName Semaphore name. * @throws Exception If failed. */ - private void removeSemaphore(String semaphoreName) - throws Exception { + private void removeSemaphore(final String semaphoreName) throws Exception { IgniteSemaphore semaphore = grid(RND.nextInt(NODES_CNT)).semaphore(semaphoreName, 10, false, true); assert semaphore != null; @@ -454,8 +454,17 @@ private void removeSemaphore(String semaphoreName) semaphore0.close(); // Ensure semaphore is removed on all nodes. - for (Ignite g : G.allGrids()) - assertNull(((IgniteKernal)g).context().dataStructures().semaphore(semaphoreName, null, 10, true, false)); + assert GridTestUtils.waitForCondition(new GridAbsPredicateX() { + @Override public boolean applyx() throws IgniteCheckedException { + for (Ignite g : G.allGrids()) { + if (((IgniteKernal)g).context().dataStructures().semaphore( + semaphoreName, null, 10, true, false) != null) + return false; + } + + return true; + } + }, 5_000); checkRemovedSemaphore(semaphore); } From 54572c3023dc03a55621fbb4754888b081791e31 Mon Sep 17 00:00:00 2001 From: tledkov-gridgain Date: Wed, 5 Jul 2017 10:30:48 +0300 Subject: [PATCH 056/155] IGNITE-5340: Improved IgniteCacheAbstractQuerySelfTest to better test non-standard index names. This closes #2206. --- .../processors/cache/IgniteCacheAbstractQuerySelfTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java index f3dbbb1ed0766..e58f983213709 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java @@ -179,7 +179,8 @@ protected CacheConfiguration cacheConfiguration() { qryEntity.addQueryField("id", Integer.class.getName(), null); qryEntity.addQueryField("name", String.class.getName(), null); qryEntity.setTableName("Type2"); - qryEntity.setIndexes(Arrays.asList(new QueryIndex("id"))); + qryEntity.setIndexes(Arrays.asList(new QueryIndex("id") + .setName("index~!@#$%^&*()_=-+;[]{}|?,.*`:nameWithNonLetterSymbols"))); entityList.add(qryEntity); From 44fad244aa40f552159af153fefdf25d47c7375e Mon Sep 17 00:00:00 2001 From: tledkov-gridgain Date: Wed, 5 Jul 2017 11:18:48 +0300 Subject: [PATCH 057/155] IGNITE-5426: JdbcThinConnection.readOnly state propagation. This closes #2200. --- .../ignite/internal/jdbc/thin/JdbcThinConnection.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java index 14c34eed7fd81..89ef2fca0ed8a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java @@ -76,6 +76,9 @@ public class JdbcThinConnection implements Connection { /** Auto commit flag. */ private boolean autoCommit; + /** Read-only flag. */ + private boolean readOnly; + /** Current transaction holdability. */ private int holdability; @@ -277,13 +280,15 @@ private void checkCursorOptions(int resSetType, int resSetConcurrency, /** {@inheritDoc} */ @Override public void setReadOnly(boolean readOnly) throws SQLException { ensureNotClosed(); + + this.readOnly = readOnly; } /** {@inheritDoc} */ @Override public boolean isReadOnly() throws SQLException { ensureNotClosed(); - return true; + return readOnly; } /** {@inheritDoc} */ From 1337901f04c866e20093b20449c0872f089fb64b Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 5 Jul 2017 11:19:43 +0300 Subject: [PATCH 058/155] Reworked cluster activation/deactivation. --- .../apache/ignite/internal/GridComponent.java | 4 +- .../ignite/internal/GridPluginComponent.java | 2 +- .../apache/ignite/internal/IgniteKernal.java | 33 +- .../internal/managers/GridManagerAdapter.java | 2 +- .../managers/discovery/DiscoCache.java | 17 +- .../discovery/DiscoveryLocalJoinData.java | 104 ++ .../discovery/GridDiscoveryManager.java | 128 +- .../pagemem/store/IgnitePageStoreManager.java | 3 +- .../processors/GridProcessorAdapter.java | 2 +- .../cache/CacheAffinitySharedManager.java | 67 +- .../processors/cache/CacheGroupContext.java | 4 +- .../processors/cache/CacheGroupData.java | 4 +- .../processors/cache/ClusterCachesInfo.java | 490 +++++-- .../cache/DynamicCacheChangeRequest.java | 52 +- .../processors/cache/ExchangeActions.java | 37 +- .../cache/GridCacheEventManager.java | 2 - .../cache/GridCacheEvictionManager.java | 1 - .../processors/cache/GridCacheIoManager.java | 13 +- .../cache/GridCacheMvccManager.java | 9 +- .../GridCachePartitionExchangeManager.java | 423 +++--- .../processors/cache/GridCacheProcessor.java | 177 ++- .../cache/GridCacheSharedContext.java | 60 +- .../cache/GridCacheSharedManager.java | 6 - .../cache/GridCacheSharedManagerAdapter.java | 16 - ...rState.java => PendingDiscoveryEvent.java} | 37 +- .../processors/cache/StateChangeRequest.java | 77 + .../CacheObjectBinaryProcessorImpl.java | 4 +- .../GridCacheTxRecoveryFuture.java | 1 - .../distributed/dht/GridDhtCacheAdapter.java | 1 - .../distributed/dht/GridDhtGetFuture.java | 1 - .../dht/GridDhtGetSingleFuture.java | 2 - .../dht/GridDhtPartitionTopologyImpl.java | 13 +- .../dht/GridDhtTopologyFutureAdapter.java | 2 +- .../dht/GridPartitionedSingleGetFuture.java | 3 - .../GridNearAtomicAbstractUpdateFuture.java | 1 - .../dht/preloader/GridDhtForceKeysFuture.java | 1 - .../preloader/GridDhtPartitionDemander.java | 2 + .../GridDhtPartitionsExchangeFuture.java | 228 ++- .../GridDhtPartitionsFullMessage.java | 44 +- .../GridDhtPartitionsSingleMessage.java | 38 +- .../dht/preloader/GridDhtPreloader.java | 2 +- .../distributed/near/GridNearGetFuture.java | 2 - .../near/GridNearTxPrepareRequest.java | 1 - .../GridCacheDatabaseSharedManager.java | 105 +- .../persistence/GridCacheOffheapManager.java | 5 +- .../IgniteCacheDatabaseSharedManager.java | 64 +- .../IgniteCacheSnapshotManager.java | 12 +- .../file/FilePageStoreManager.java | 14 +- .../wal/FileWriteAheadLogManager.java | 8 - .../GridCacheDistributedQueryManager.java | 4 +- .../store/GridCacheStoreManagerAdapter.java | 1 - .../version/GridCacheVersionManager.java | 6 - .../IgniteCacheObjectProcessor.java | 5 - .../IgniteCacheObjectProcessorImpl.java | 5 - .../ChangeGlobalStateFinishMessage.java | 86 ++ .../ChangeGlobalStateMessage.java | 78 +- .../processors/cluster/ClusterProcessor.java | 3 +- .../cluster/DiscoveryDataClusterState.java | 157 ++ .../cluster/GridClusterStateProcessor.java | 1122 ++++++-------- .../IgniteChangeGlobalStateSupport.java | 3 +- .../DataStructuresProcessor.java | 6 +- .../GridCacheAtomicLongImpl.java | 2 +- .../GridCacheAtomicReferenceImpl.java | 2 +- .../GridCacheAtomicSequenceImpl.java | 2 +- .../GridCacheAtomicStampedImpl.java | 2 +- .../GridCacheCountDownLatchImpl.java | 2 +- .../datastructures/GridCacheLockImpl.java | 4 +- .../datastructures/GridCacheQueueAdapter.java | 1 - .../GridCacheSemaphoreImpl.java | 2 +- .../datastructures/GridCacheSetImpl.java | 1 - .../internal/processors/igfs/IgfsImpl.java | 2 - .../processors/igfs/IgfsProcessor.java | 2 +- .../processors/query/GridQueryProcessor.java | 4 +- .../processors/rest/GridRestProcessor.java | 2 +- .../GridChangeStateCommandHandler.java | 2 +- .../service/GridServiceProcessor.java | 6 +- .../processors/task/GridTaskProcessor.java | 2 +- .../ignite/spi/discovery/tcp/ClientImpl.java | 12 +- .../ignite/spi/discovery/tcp/ServerImpl.java | 10 +- .../TestRecordingCommunicationSpi.java | 10 + ...idManagerLocalMessageListenerSelfTest.java | 4 +- .../cache/IgniteActiveClusterTest.java | 182 --- .../IgniteClusterActivateDeactivateTest.java | 1284 +++++++++++++++++ ...ActivateDeactivateTestWithPersistence.java | 197 +++ .../IgniteDaemonNodeMarshallerCacheTest.java | 10 - .../pagemem/NoOpPageStoreManager.java | 12 +- .../persistence/pagemem/NoOpWALManager.java | 23 +- .../AbstractNodeJoinTemplate.java | 149 +- .../IgniteChangeGlobalStateAbstractTest.java | 65 +- .../IgniteChangeGlobalStateCacheTest.java | 2 +- ...niteChangeGlobalStateDataStreamerTest.java | 5 +- ...iteChangeGlobalStateDataStructureTest.java | 6 +- .../IgniteChangeGlobalStateFailOverTest.java | 26 +- .../IgniteChangeGlobalStateTest.java | 158 +- .../IgniteStandByClusterTest.java | 17 +- .../join/JoinActiveNodeToActiveCluster.java | 62 +- ...iveNodeToActiveClusterWithPersistence.java | 17 + .../IgniteStandByClientReconnectTest.java | 13 +- ...tandByClientReconnectToNewClusterTest.java | 13 +- ...CommunicationSpiMultithreadedSelfTest.java | 2 +- .../junits/GridAbstractTest.java | 4 +- .../junits/common/GridCommonAbstractTest.java | 3 + .../testsuites/IgniteStandByClusterSuite.java | 5 +- .../processors/hadoop/HadoopProcessor.java | 4 +- 104 files changed, 4058 insertions(+), 2075 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryLocalJoinData.java rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/{ClusterState.java => PendingDiscoveryEvent.java} (52%) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java rename modules/core/src/main/java/org/apache/ignite/internal/processors/{cache => cluster}/ChangeGlobalStateMessage.java (57%) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteActiveClusterTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java b/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java index 0505929ac8f76..93ffe9539f90c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java @@ -84,9 +84,11 @@ enum DiscoveryDataExchangeType { * Callback that notifies that kernal has successfully started, * including all managers and processors. * + * @param active Cluster active flag (note: should be used carefully since state can + * change concurrently). * @throws IgniteCheckedException Thrown in case of any errors. */ - public void onKernalStart() throws IgniteCheckedException; + public void onKernalStart(boolean active) throws IgniteCheckedException; /** * Callback to notify that kernal is about to stop. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java b/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java index cc1ae714d8cf2..fd59d24d18efe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java @@ -61,7 +61,7 @@ public PluginProvider plugin() { } /** {@inheritDoc} */ - @Override public void onKernalStart() throws IgniteCheckedException { + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { plugin.onIgniteStart(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 31ee3e2e30453..0c17b32f7cdc6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -98,6 +98,7 @@ import org.apache.ignite.internal.managers.collision.GridCollisionManager; import org.apache.ignite.internal.managers.communication.GridIoManager; import org.apache.ignite.internal.managers.deployment.GridDeploymentManager; +import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData; import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager; import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager; import org.apache.ignite.internal.managers.failover.GridFailoverManager; @@ -207,7 +208,6 @@ import static org.apache.ignite.internal.IgniteComponentType.IGFS; import static org.apache.ignite.internal.IgniteComponentType.IGFS_HELPER; import static org.apache.ignite.internal.IgniteComponentType.SCHEDULE; -import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_ACTIVE_ON_START; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_BUILD_DATE; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_BUILD_VER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CLIENT_MODE; @@ -818,8 +818,6 @@ public void start( List plugins = U.allPluginProviders(); - final boolean activeOnStart = cfg.isActiveOnStart(); - // Spin out SPIs & managers. try { ctx = new GridKernalContextImpl(log, @@ -994,11 +992,28 @@ public void start( // Suggest Operation System optimizations. ctx.performance().addAll(OsConfigurationSuggestions.getSuggestions()); + DiscoveryLocalJoinData joinData = ctx.discovery().localJoin(); + + IgniteInternalFuture transitionWaitFut = joinData.transitionWaitFuture(); + + boolean active; + + if (transitionWaitFut != null) { + if (log.isInfoEnabled()) { + log.info("Join cluster while cluster state transition is in progress, " + + "waiting when transition finish."); + } + + active = transitionWaitFut.get(); + } + else + active = joinData.active(); + // Notify discovery manager the first to make sure that topology is discovered. - ctx.discovery().onKernalStart(); + ctx.discovery().onKernalStart(active); // Notify IO manager the second so further components can send and receive messages. - ctx.io().onKernalStart(); + ctx.io().onKernalStart(active); // Start plugins. for (PluginProvider provider : ctx.plugins().allProviders()) @@ -1021,7 +1036,7 @@ public void start( if (!skipDaemon(comp)) { try { - comp.onKernalStart(); + comp.onKernalStart(active); } catch (IgniteNeedReconnectException e) { assert ctx.discovery().reconnectSupported(); @@ -1486,7 +1501,6 @@ private void fillNodeAttributes(boolean notifyEnabled) throws IgniteCheckedExcep add(ATTR_MARSHALLER_USE_DFLT_SUID, getBoolean(IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID, OptimizedMarshaller.USE_DFLT_SUID)); add(ATTR_LATE_AFFINITY_ASSIGNMENT, cfg.isLateAffinityAssignment()); - add(ATTR_ACTIVE_ON_START, cfg.isActiveOnStart()); if (cfg.getMarshaller() instanceof BinaryMarshaller) { add(ATTR_MARSHALLER_COMPACT_FOOTER, cfg.getBinaryConfiguration() == null ? @@ -3395,7 +3409,7 @@ public IgniteInternalFuture getOrCreateCacheAsync(String cacheName, boolean c guard(); try { - return context().state().active(); + return context().state().publicApiActiveState(); } finally { unguard(); @@ -3694,10 +3708,11 @@ private void unguard() { * @throws IgniteException if cluster in inActive state */ private void checkClusterState() throws IgniteException { - if (!ctx.state().active()) + if (!ctx.state().publicApiActiveState()) { throw new IgniteException("Can not perform the operation because the cluster is inactive. Note, that " + "the cluster is considered inactive by default if Ignite Persistent Store is used to let all the nodes " + "join the cluster. To activate the cluster call Ignite.activate(true)."); + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java index 7dfeffbcf76cf..a151eb5d5fa4a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java @@ -362,7 +362,7 @@ protected final String stopInfo() { } /** {@inheritDoc} */ - @Override public final void onKernalStart() throws IgniteCheckedException { + @Override public final void onKernalStart(boolean active) throws IgniteCheckedException { for (final IgniteSpi spi : spis) { try { spi.onContextInitialized(new IgniteSpiContext() { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java index 2b3c4fc8f9539..4c1077b039544 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java @@ -25,6 +25,7 @@ import java.util.Set; import java.util.UUID; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; @@ -37,6 +38,9 @@ * */ public class DiscoCache { + /** */ + private final DiscoveryDataClusterState state; + /** Local node. */ private final ClusterNode loc; @@ -78,6 +82,7 @@ public class DiscoCache { private final Set alives = new GridConcurrentHashSet<>(); /** + * @param state Current cluster state. * @param loc Local node. * @param rmtNodes Remote nodes. * @param allNodes All nodes. @@ -91,7 +96,9 @@ public class DiscoCache { * @param nodeMap Node map. * @param alives Alive nodes. */ - DiscoCache(ClusterNode loc, + DiscoCache( + DiscoveryDataClusterState state, + ClusterNode loc, List rmtNodes, List allNodes, List srvNodes, @@ -103,6 +110,7 @@ public class DiscoCache { Map> cacheGrpAffNodes, Map nodeMap, Set alives) { + this.state = state; this.loc = loc; this.rmtNodes = rmtNodes; this.allNodes = allNodes; @@ -117,6 +125,13 @@ public class DiscoCache { this.alives.addAll(alives); } + /** + * @return Current cluster state. + */ + public DiscoveryDataClusterState state() { + return state; + } + /** @return Local node. */ public ClusterNode localNode() { return loc; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryLocalJoinData.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryLocalJoinData.java new file mode 100644 index 0000000000000..a1f2aa72713ae --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryLocalJoinData.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.managers.discovery; + +import org.apache.ignite.events.DiscoveryEvent; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.jetbrains.annotations.Nullable; + +/** + * Information about local join event. + */ +public class DiscoveryLocalJoinData { + /** */ + private final DiscoveryEvent evt; + + /** */ + private final DiscoCache discoCache; + + /** */ + private final AffinityTopologyVersion joinTopVer; + + /** */ + private final IgniteInternalFuture transitionWaitFut; + + /** */ + private final boolean active; + + /** + * @param evt Event. + * @param discoCache Discovery data cache. + * @param transitionWaitFut Future if cluster state transition is in progress. + * @param active Cluster active status. + */ + public DiscoveryLocalJoinData(DiscoveryEvent evt, + DiscoCache discoCache, + @Nullable IgniteInternalFuture transitionWaitFut, + boolean active) { + assert evt != null && evt.topologyVersion() > 0 : evt; + + this.evt = evt; + this.discoCache = discoCache; + this.transitionWaitFut = transitionWaitFut; + this.active = active; + + joinTopVer = new AffinityTopologyVersion(evt.topologyVersion(), 0); + } + + /** + * @return Future if cluster state transition is in progress. + */ + @Nullable public IgniteInternalFuture transitionWaitFuture() { + return transitionWaitFut; + } + + /** + * @return Cluster state. + */ + public boolean active() { + return active; + } + + /** + * @return Event. + */ + public DiscoveryEvent event() { + return evt; + } + + /** + * @return Discovery data cache. + */ + public DiscoCache discoCache() { + return discoCache; + } + + /** + * @return Join topology version. + */ + public AffinityTopologyVersion joinTopologyVersion() { + return joinTopVer; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(DiscoveryLocalJoinData.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index c38e37a74d4db..9f5bd3f02272e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -75,8 +75,11 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; -import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor; import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; +import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; +import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor; import org.apache.ignite.internal.processors.jobmetrics.GridJobMetrics; import org.apache.ignite.internal.processors.security.SecurityContext; import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor; @@ -90,7 +93,6 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.P1; -import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.S; @@ -133,7 +135,6 @@ import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED; import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; -import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_ACTIVE_ON_START; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; @@ -144,6 +145,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME; import static org.apache.ignite.internal.IgniteVersionUtils.VER; +import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT; import static org.apache.ignite.internal.processors.security.SecurityUtils.SERVICE_PERMISSIONS_SINCE; import static org.apache.ignite.internal.processors.security.SecurityUtils.isSecurityCompatibilityMode; import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.NOOP; @@ -238,7 +240,7 @@ public class GridDiscoveryManager extends GridManagerAdapter { private long segChkFreq; /** Local node join to topology event. */ - private GridFutureAdapter> locJoin = new GridFutureAdapter<>(); + private GridFutureAdapter locJoin = new GridFutureAdapter<>(); /** GC CPU load. */ private volatile double gcCpuLoad; @@ -570,7 +572,7 @@ private void updateClientNodes(UUID leftNodeId) { if (type != EVT_NODE_SEGMENTED && type != EVT_CLIENT_NODE_DISCONNECTED && type != EVT_CLIENT_NODE_RECONNECTED && - type != DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT) { + type != EVT_DISCOVERY_CUSTOM_EVT) { minorTopVer = 0; verChanged = true; @@ -586,15 +588,50 @@ private void updateClientNodes(UUID leftNodeId) { updateClientNodes(node.id()); } + DiscoCache discoCache = null; + + boolean locJoinEvt = type == EVT_NODE_JOINED && node.id().equals(locNode.id()); + + IgniteInternalFuture transitionWaitFut = null; + + ChangeGlobalStateFinishMessage stateFinishMsg = null; + + if (locJoinEvt) { + discoCache = createDiscoCache(ctx.state().clusterState(), locNode, topSnapshot); + + transitionWaitFut = ctx.state().onLocalJoin(discoCache); + } + else if (type == EVT_NODE_FAILED || type == EVT_NODE_LEFT) + stateFinishMsg = ctx.state().onNodeLeft(node); + final AffinityTopologyVersion nextTopVer; - if (type == DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT) { + if (type == EVT_DISCOVERY_CUSTOM_EVT) { assert customMsg != null; - boolean incMinorTopVer = ctx.cache().onCustomEvent( - customMsg, - new AffinityTopologyVersion(topVer, minorTopVer), - node); + boolean incMinorTopVer; + + if (customMsg instanceof ChangeGlobalStateMessage) { + incMinorTopVer = ctx.state().onStateChangeMessage( + new AffinityTopologyVersion(topVer, minorTopVer), + (ChangeGlobalStateMessage)customMsg, + discoCache()); + } + else if (customMsg instanceof ChangeGlobalStateFinishMessage) { + ctx.state().onStateFinishMessage((ChangeGlobalStateFinishMessage)customMsg); + + discoCache = createDiscoCache(ctx.state().clusterState(), locNode, topSnapshot); + + topSnap.set(new Snapshot(topSnap.get().topVer, discoCache)); + + incMinorTopVer = false; + } + else { + incMinorTopVer = ctx.cache().onCustomEvent( + customMsg, + new AffinityTopologyVersion(topVer, minorTopVer), + node); + } if (incMinorTopVer) { minorTopVer++; @@ -603,17 +640,13 @@ private void updateClientNodes(UUID leftNodeId) { } nextTopVer = new AffinityTopologyVersion(topVer, minorTopVer); - - if (verChanged) - ctx.cache().onDiscoveryEvent(type, node, nextTopVer); } - else { + else nextTopVer = new AffinityTopologyVersion(topVer, minorTopVer); - ctx.cache().onDiscoveryEvent(type, node, nextTopVer); - } + ctx.cache().onDiscoveryEvent(type, customMsg, node, nextTopVer, ctx.state().clusterState()); - if (type == DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT) { + if (type == EVT_DISCOVERY_CUSTOM_EVT) { for (Class cls = customMsg.getClass(); cls != null; cls = cls.getSuperclass()) { List> list = customEvtLsnrs.get(cls); @@ -630,13 +663,12 @@ private void updateClientNodes(UUID leftNodeId) { } } - final DiscoCache discoCache; - // Put topology snapshot into discovery history. // There is no race possible between history maintenance and concurrent discovery // event notifications, since SPI notifies manager about all events from this listener. if (verChanged) { - discoCache = createDiscoCache(locNode, topSnapshot); + if (discoCache == null) + discoCache = createDiscoCache(ctx.state().clusterState(), locNode, topSnapshot); discoCacheHist.put(nextTopVer, discoCache); @@ -650,8 +682,10 @@ private void updateClientNodes(UUID leftNodeId) { // Current version. discoCache = discoCache(); + final DiscoCache discoCache0 = discoCache; + // If this is a local join event, just save it and do not notify listeners. - if (type == EVT_NODE_JOINED && node.id().equals(locNode.id())) { + if (locJoinEvt) { if (gridStartTime == 0) gridStartTime = getSpi().getGridStartTime(); @@ -668,7 +702,15 @@ private void updateClientNodes(UUID leftNodeId) { discoEvt.topologySnapshot(topVer, new ArrayList<>(F.view(topSnapshot, FILTER_DAEMON))); - locJoin.onDone(new T2<>(discoEvt, discoCache)); + discoWrk.discoCache = discoCache; + + if (!isLocDaemon && !ctx.clientDisconnected()) + ctx.cache().context().exchange().onLocalJoin(discoEvt, discoCache); + + locJoin.onDone(new DiscoveryLocalJoinData(discoEvt, + discoCache, + transitionWaitFut, + ctx.state().clusterState().active())); return; } @@ -697,7 +739,7 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) { topHist.clear(); topSnap.set(new Snapshot(AffinityTopologyVersion.ZERO, - createDiscoCache(locNode, Collections.emptySet()))); + createDiscoCache(ctx.state().clusterState(), locNode, Collections.emptySet()))); } else if (type == EVT_CLIENT_NODE_RECONNECTED) { assert locNode.isClient() : locNode; @@ -709,12 +751,14 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) { ((IgniteKernal)ctx.grid()).onReconnected(clusterRestarted); + ctx.cache().context().exchange().onLocalJoin(localJoinEvent(), discoCache); + ctx.cluster().clientReconnectFuture().listen(new CI1>() { @Override public void apply(IgniteFuture fut) { try { fut.get(); - discoWrk.addEvent(type, nextTopVer, node, discoCache, topSnapshot, null); + discoWrk.addEvent(type, nextTopVer, node, discoCache0, topSnapshot, null); } catch (IgniteException ignore) { // No-op. @@ -727,6 +771,9 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) { if (type == EVT_CLIENT_NODE_DISCONNECTED || type == EVT_NODE_SEGMENTED || !ctx.clientDisconnected()) discoWrk.addEvent(type, nextTopVer, node, discoCache, topSnapshot, customMsg); + + if (stateFinishMsg != null) + discoWrk.addEvent(EVT_DISCOVERY_CUSTOM_EVT, nextTopVer, node, discoCache, topSnapshot, stateFinishMsg); } }); @@ -826,7 +873,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) { * @return {@code True} if should not process message. */ private boolean skipMessage(int type, @Nullable DiscoveryCustomMessage customMsg) { - if (type == DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT) { + if (type == EVT_DISCOVERY_CUSTOM_EVT) { assert customMsg != null && customMsg.id() != null : customMsg; if (rcvdCustomMsgs.contains(customMsg.id())) { @@ -1157,7 +1204,6 @@ private void checkAttributes(Iterable nodes) throws IgniteCheckedEx locMarshStrSerVer2; boolean locDelayAssign = locNode.attribute(ATTR_LATE_AFFINITY_ASSIGNMENT); - boolean locActiveOnStart = locNode.attribute(ATTR_ACTIVE_ON_START); Boolean locSrvcCompatibilityEnabled = locNode.attribute(ATTR_SERVICES_COMPATIBILITY_MODE); Boolean locSecurityCompatibilityEnabled = locNode.attribute(ATTR_SECURITY_COMPATIBILITY_MODE); @@ -1971,7 +2017,7 @@ public AffinityTopologyVersion topologyVersionEx() { /** @return Event that represents a local node joined to topology. */ public DiscoveryEvent localJoinEvent() { try { - return locJoin.get().get1(); + return locJoin.get().event(); } catch (IgniteCheckedException e) { throw new IgniteException(e); @@ -1981,7 +2027,7 @@ public DiscoveryEvent localJoinEvent() { /** * @return Tuple that consists of a local join event and discovery cache at the join time. */ - public T2 localJoin() { + public DiscoveryLocalJoinData localJoin() { try { return locJoin.get(); } @@ -2016,7 +2062,7 @@ public void sendCustomEvent(DiscoveryCustomMessage msg) throws IgniteCheckedExce public void clientCacheStartEvent(UUID reqId, @Nullable Map startReqs, @Nullable Set cachesToClose) { - discoWrk.addEvent(DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT, + discoWrk.addEvent(EVT_DISCOVERY_CUSTOM_EVT, AffinityTopologyVersion.NONE, localNode(), null, @@ -2098,11 +2144,14 @@ public void reconnect() { /** * Called from discovery thread. * + * @param state Current state. * @param loc Local node. * @param topSnapshot Topology snapshot. * @return Newly created discovery cache. */ - @NotNull private DiscoCache createDiscoCache(ClusterNode loc, Collection topSnapshot) { + @NotNull private DiscoCache createDiscoCache(DiscoveryDataClusterState state, + ClusterNode loc, + Collection topSnapshot) { HashSet alives = U.newHashSet(topSnapshot.size()); HashMap nodeMap = U.newHashMap(topSnapshot.size()); @@ -2177,6 +2226,7 @@ public void reconnect() { } return new DiscoCache( + state, loc, Collections.unmodifiableList(rmtNodes), Collections.unmodifiableList(allNodes), @@ -2318,7 +2368,7 @@ public void scheduleSegmentCheck() { discoWrk.addEvent(EVT_NODE_SEGMENTED, AffinityTopologyVersion.NONE, node, - createDiscoCache(node, empty), + createDiscoCache(null, node, empty), empty, null); @@ -2339,6 +2389,9 @@ public void scheduleSegmentCheck() { /** Worker for discovery events. */ private class DiscoveryWorker extends GridWorker { + /** */ + private DiscoCache discoCache; + /** Event queue. */ private final BlockingQueue, DiscoveryCustomMessage>> evts = new LinkedBlockingQueue<>(); @@ -2457,6 +2510,9 @@ private void body0() throws InterruptedException { boolean segmented = false; + if (evt.get4() != null) + discoCache = evt.get4(); + switch (type) { case EVT_NODE_JOINED: { assert !discoOrdered || topVer.topologyVersion() == node.order() : "Invalid topology version [topVer=" + topVer + @@ -2570,8 +2626,8 @@ else if (log.isDebugEnabled()) break; } - case DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT: { - if (ctx.event().isRecordable(DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT)) { + case EVT_DISCOVERY_CUSTOM_EVT: { + if (ctx.event().isRecordable(EVT_DISCOVERY_CUSTOM_EVT)) { DiscoveryCustomEvent customEvt = new DiscoveryCustomEvent(); customEvt.node(ctx.discovery().localNode()); @@ -2581,6 +2637,12 @@ else if (log.isDebugEnabled()) customEvt.affinityTopologyVersion(topVer); customEvt.customMessage(evt.get6()); + if (evt.get4() == null) { + assert discoCache != null : evt.get6(); + + evt.set4(discoCache); + } + ctx.event().record(customEvt, evt.get4()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java index 468d35dfc93ff..fa6e9e410e7f8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java @@ -183,11 +183,10 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, public Map readCacheConfigurations() throws IgniteCheckedException; /** - * @param grpDesc Cache group descriptor. * @param cacheData Cache configuration. * @throws IgniteCheckedException If failed. */ - public void storeCacheData(CacheGroupDescriptor grpDesc, StoredCacheData cacheData) throws IgniteCheckedException; + public void storeCacheData(StoredCacheData cacheData) throws IgniteCheckedException; /** * @param grpId Cache group ID. * @return {@code True} if index store for given cache group existed before node started. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java index 690ba0ee1110b..d6f78ab61770c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java @@ -65,7 +65,7 @@ protected GridProcessorAdapter(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void onKernalStart() throws IgniteCheckedException { + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 9516f848e5ee8..8d08c3f064c53 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -41,6 +41,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.managers.discovery.DiscoCache; +import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache; @@ -52,6 +53,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; +import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.util.future.GridCompoundFuture; import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; @@ -108,6 +111,9 @@ public class CacheAffinitySharedManager extends GridCacheSharedManagerAdap /** */ private final ThreadLocal clientCacheChanges = new ThreadLocal<>(); + /** Caches initialized flag (initialized when join activate cluster or after activation. */ + private boolean cachesInitialized; + /** Discovery listener. */ private final GridLocalEventListener discoLsnr = new GridLocalEventListener() { @Override public void onEvent(Event evt) { @@ -140,10 +146,19 @@ public class CacheAffinitySharedManager extends GridCacheSharedManagerAdap * Callback invoked from discovery thread when discovery message is received. * * @param type Event type. + * @param customMsg Custom message instance. * @param node Event node. * @param topVer Topology version. + * @param state Cluster state. */ - void onDiscoveryEvent(int type, ClusterNode node, AffinityTopologyVersion topVer) { + void onDiscoveryEvent(int type, + @Nullable DiscoveryCustomMessage customMsg, + ClusterNode node, + AffinityTopologyVersion topVer, + DiscoveryDataClusterState state) { + if (state.transition() || !state.active()) + return; + if (type == EVT_NODE_JOINED && node.isLocal()) { // Clean-up in case of client reconnect. caches.clear(); @@ -153,6 +168,15 @@ void onDiscoveryEvent(int type, ClusterNode node, AffinityTopologyVersion topVer lastAffVer = null; caches.init(cctx.cache().cacheGroupDescriptors(), cctx.cache().cacheDescriptors()); + + cachesInitialized = true; + } + else if (customMsg instanceof ChangeGlobalStateFinishMessage) { + if (!cachesInitialized && ((ChangeGlobalStateFinishMessage)customMsg).clusterActive()) { + caches.init(cctx.cache().cacheGroupDescriptors(), cctx.cache().cacheDescriptors()); + + cachesInitialized = true; + } } if (!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) { @@ -404,7 +428,10 @@ void onCacheGroupCreated(CacheGroupContext grp) { DynamicCacheChangeRequest startReq = startReqs.get(desc.cacheName()); - cctx.cache().prepareCacheStart(desc, startReq.nearCacheConfiguration(), topVer); + cctx.cache().prepareCacheStart(desc.cacheConfiguration(), + desc, + startReq.nearCacheConfiguration(), + topVer); startedInfos.put(desc.cacheId(), startReq.nearCacheConfiguration() != null); @@ -683,19 +710,7 @@ public void onCacheChangeRequest( NearCacheConfiguration nearCfg = null; - if (exchActions.newClusterState() == ClusterState.ACTIVE) { - if (CU.isSystemCache(req.cacheName())) - startCache = true; - else if (!cctx.localNode().isClient()) { - startCache = cctx.cacheContext(action.descriptor().cacheId()) == null && - CU.affinityNode(cctx.localNode(), req.startCacheConfiguration().getNodeFilter()); - - nearCfg = req.nearCacheConfiguration(); - } - else // Only static cache configured on client must be started. - startCache = cctx.kernalContext().state().isLocallyConfigured(req.cacheName()); - } - else if (cctx.localNodeId().equals(req.initiatingNodeId())) { + if (req.locallyConfigured() || (cctx.localNodeId().equals(req.initiatingNodeId()) && !exchActions.activate())) { startCache = true; nearCfg = req.nearCacheConfiguration(); @@ -703,7 +718,7 @@ else if (cctx.localNodeId().equals(req.initiatingNodeId())) { else { // Cache should not be started assert cctx.cacheContext(cacheDesc.cacheId()) == null - : "Starting cache has not null context: " + cacheDesc.cacheName(); + : "Starting cache has not null context: " + cacheDesc.cacheName(); IgniteCacheProxy cacheProxy = cctx.cache().jcacheProxy(req.cacheName()); @@ -711,27 +726,29 @@ else if (cctx.localNodeId().equals(req.initiatingNodeId())) { if (cacheProxy != null) { // Cache should be in restarting mode assert cacheProxy.isRestarting() - : "Cache has non restarting proxy " + cacheProxy; + : "Cache has non restarting proxy " + cacheProxy; startCache = true; } - else - startCache = CU.affinityNode(cctx.localNode(), cacheDesc.groupDescriptor().config().getNodeFilter()); + else { + startCache = CU.affinityNode(cctx.localNode(), + cacheDesc.groupDescriptor().config().getNodeFilter()); + } } try { // Save configuration before cache started. - if (cctx.pageStore() != null && !cctx.localNode().isClient()) + if (cctx.pageStore() != null && !cctx.kernalContext().clientNode()) { cctx.pageStore().storeCacheData( - cacheDesc.groupDescriptor(), new StoredCacheData(req.startCacheConfiguration()) ); + } if (startCache) { - cctx.cache().prepareCacheStart(cacheDesc, nearCfg, fut.topologyVersion()); - - if (exchActions.newClusterState() == null) - cctx.kernalContext().state().onCacheStart(req); + cctx.cache().prepareCacheStart(req.startCacheConfiguration(), + cacheDesc, + nearCfg, + fut.topologyVersion()); if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) { if (fut.discoCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index c3ddc5f175e8c..14eb3628e5036 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -683,6 +683,8 @@ void stopGroup() { aff.cancelFutures(err); + preldr.onKernalStop(); + offheapMgr.stop(); ctx.io().removeCacheGroupHandlers(grpId); @@ -853,8 +855,6 @@ public void start() throws IgniteCheckedException { preldr = new GridCachePreloaderAdapter(this); if (ctx.kernalContext().config().getPersistentStoreConfiguration() != null) { - ClassLoader clsLdr = U.gridClassLoader(); - try { offheapMgr = new GridCacheOffheapManager(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupData.java index a290caf8e51a2..99b7b1e321a7a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupData.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupData.java @@ -78,8 +78,8 @@ public class CacheGroupData implements Serializable { Map caches, long flags) { assert cacheCfg != null; - assert grpId != 0; - assert deploymentId != null; + assert grpId != 0 : cacheCfg.getName(); + assert deploymentId != null : cacheCfg.getName(); this.cacheCfg = cacheCfg; this.grpName = grpName; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java index 8f124b2b92062..5452bd20fcb19 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java @@ -40,6 +40,9 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; +import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.processors.query.QuerySchema; import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.processors.query.schema.SchemaOperationException; @@ -93,10 +96,13 @@ class ClusterCachesInfo { private List> locJoinStartCaches; /** */ - private Map clientReconnectReqs; + private Map> locCfgsForActivation; /** */ - private volatile Exception onJoinCacheException; + private Map clientReconnectReqs; + + /** {@code True} if joined cluster while cluster state change was in progress. */ + private boolean joinOnTransition; /** * @param ctx Context. @@ -113,14 +119,25 @@ class ClusterCachesInfo { */ void onStart(CacheJoinNodeDiscoveryData joinDiscoData) throws IgniteCheckedException { this.joinDiscoData = joinDiscoData; - } - /** - * - * @return Exception if cache has conflict. - */ - Exception onJoinCacheException(){ - return onJoinCacheException; + Map grpCfgs = new HashMap<>(); + + for (CacheJoinNodeDiscoveryData.CacheInfo info : joinDiscoData.caches().values()) { + if (info.cacheData().config().getGroupName() == null) + continue; + + CacheConfiguration ccfg = grpCfgs.get(info.cacheData().config().getGroupName()); + + if (ccfg == null) + grpCfgs.put(info.cacheData().config().getGroupName(), info.cacheData().config()); + else + validateCacheGroupConfiguration(ccfg, info.cacheData().config()); + } + + String conflictErr = processJoiningNode(joinDiscoData, ctx.localNodeId(), true); + + if (conflictErr != null) + throw new IgniteCheckedException("Failed to start configured cache. " + conflictErr); } /** @@ -142,7 +159,9 @@ void onKernalStart(boolean checkConsistency) throws IgniteCheckedException { if (gridData != null && gridData.conflictErr != null) throw new IgniteCheckedException(gridData.conflictErr); - if (joinDiscoData != null && gridData != null) { + if (gridData != null && gridData.joinDiscoData != null) { + CacheJoinNodeDiscoveryData joinDiscoData = gridData.joinDiscoData; + for (CacheJoinNodeDiscoveryData.CacheInfo locCacheInfo : joinDiscoData.caches().values()) { CacheConfiguration locCfg = locCacheInfo.cacheData().config(); @@ -165,9 +184,9 @@ void onKernalStart(boolean checkConsistency) throws IgniteCheckedException { } } - joinDiscoData = null; gridData = null; } + /** * Checks that remote caches has configuration compatible with the local. * @@ -308,22 +327,64 @@ void onClientCacheChange(ClientCacheChangeDiscoveryMessage msg, ClusterNode node } } } - /** * @param batch Cache change request. * @param topVer Topology version. * @return {@code True} if minor topology version should be increased. */ boolean onCacheChangeRequested(DynamicCacheChangeBatch batch, AffinityTopologyVersion topVer) { - ExchangeActions exchangeActions = new ExchangeActions(); + DiscoveryDataClusterState state = ctx.state().clusterState(); + + if (state.active() && !state.transition()) { + ExchangeActions exchangeActions = new ExchangeActions(); + + CacheChangeProcessResult res = processCacheChangeRequests(exchangeActions, + batch.requests(), + topVer, + false); - boolean incMinorTopVer = false; + if (res.needExchange) { + assert !exchangeActions.empty() : exchangeActions; - List addedDescs = new ArrayList<>(); + batch.exchangeActions(exchangeActions); + } + + return res.needExchange; + } + else { + IgniteCheckedException err = new IgniteCheckedException("Failed to start/stop cache, cluster state change " + + "is in progress."); + + for (DynamicCacheChangeRequest req : batch.requests()) { + if (req.template()) { + ctx.cache().completeTemplateAddFuture(req.startCacheConfiguration().getName(), + req.deploymentId()); + } + else + ctx.cache().completeCacheStartFuture(req, false, err); + } + + return false; + } + } + + /** + * @param exchangeActions Exchange actions to update. + * @param reqs Requests. + * @param topVer Topology version. + * @param persistedCfgs {@code True} if process start of persisted caches during cluster activation. + * @return Process result. + */ + private CacheChangeProcessResult processCacheChangeRequests( + ExchangeActions exchangeActions, + Collection reqs, + AffinityTopologyVersion topVer, + boolean persistedCfgs) { + CacheChangeProcessResult res = new CacheChangeProcessResult(); final List> reqsToComplete = new ArrayList<>(); - for (DynamicCacheChangeRequest req : batch.requests()) { + for (DynamicCacheChangeRequest req : reqs) { if (req.template()) { CacheConfiguration ccfg = req.startCacheConfiguration(); @@ -347,17 +408,18 @@ boolean onCacheChangeRequested(DynamicCacheChangeBatch batch, AffinityTopologyVe assert old == null; - addedDescs.add(templateDesc); + res.addedDescs.add(templateDesc); } - ctx.cache().completeTemplateAddFuture(ccfg.getName(), req.deploymentId()); + if (!persistedCfgs) + ctx.cache().completeTemplateAddFuture(ccfg.getName(), req.deploymentId()); continue; } assert !req.clientStartOnly() : req; - DynamicCacheDescriptor desc = req.globalStateChange() ? null : registeredCaches.get(req.cacheName()); + DynamicCacheDescriptor desc = registeredCaches.get(req.cacheName()); boolean needExchange = false; @@ -373,22 +435,32 @@ boolean onCacheChangeRequested(DynamicCacheChangeBatch batch, AffinityTopologyVe if (conflictErr != null) { U.warn(log, "Ignore cache start request. " + conflictErr); - ctx.cache().completeCacheStartFuture(req, false, new IgniteCheckedException("Failed to start " + - "cache. " + conflictErr)); + IgniteCheckedException err = new IgniteCheckedException("Failed to start " + + "cache. " + conflictErr); + + if (persistedCfgs) + res.errs.add(err); + else + ctx.cache().completeCacheStartFuture(req, false, err); continue; } if (req.clientStartOnly()) { + assert !persistedCfgs; + ctx.cache().completeCacheStartFuture(req, false, new IgniteCheckedException("Failed to start " + "client cache (a cache with the given name is not started): " + req.cacheName())); } else { SchemaOperationException err = QueryUtils.checkQueryEntityConflicts( - req.startCacheConfiguration(), ctx.cache().cacheDescriptors().values()); + req.startCacheConfiguration(), registeredCaches.values()); if (err != null) { - ctx.cache().completeCacheStartFuture(req, false, err); + if (persistedCfgs) + res.errs.add(err); + else + ctx.cache().completeCacheStartFuture(req, false, err); continue; } @@ -430,11 +502,13 @@ boolean onCacheChangeRequested(DynamicCacheChangeBatch batch, AffinityTopologyVe ccfg.getName(), ccfg.getNearConfiguration() != null); - ctx.discovery().addClientNode(req.cacheName(), - req.initiatingNodeId(), - req.nearCacheConfiguration() != null); + if (!persistedCfgs) { + ctx.discovery().addClientNode(req.cacheName(), + req.initiatingNodeId(), + req.nearCacheConfiguration() != null); + } - addedDescs.add(startDesc); + res.addedDescs.add(startDesc); exchangeActions.addCacheToStart(req, startDesc); @@ -442,6 +516,7 @@ boolean onCacheChangeRequested(DynamicCacheChangeBatch batch, AffinityTopologyVe } } else { + assert !persistedCfgs; assert req.initiatingNodeId() != null : req; if (req.failIfExists()) { @@ -489,8 +564,6 @@ boolean onCacheChangeRequested(DynamicCacheChangeBatch batch, AffinityTopologyVe } } } - else if (req.globalStateChange()) - exchangeActions.newClusterState(req.state()); else if (req.resetLostPartitions()) { if (desc != null) { needExchange = true; @@ -559,18 +632,18 @@ else if (req.stop()) { assert false : req; if (!needExchange) { - if (!clientCacheStart && req.initiatingNodeId().equals(ctx.localNodeId())) + if (!clientCacheStart && ctx.localNodeId().equals(req.initiatingNodeId())) reqsToComplete.add(new T2<>(req, waitTopVer)); } else - incMinorTopVer = true; + res.needExchange = true; } - if (!F.isEmpty(addedDescs)) { - AffinityTopologyVersion startTopVer = incMinorTopVer ? topVer.nextMinorVersion() : topVer; + if (!F.isEmpty(res.addedDescs)) { + AffinityTopologyVersion startTopVer = res.needExchange ? topVer.nextMinorVersion() : topVer; - for (DynamicCacheDescriptor desc : addedDescs) { - assert desc.template() || incMinorTopVer; + for (DynamicCacheDescriptor desc : res.addedDescs) { + assert desc.template() || res.needExchange; desc.startTopologyVersion(startTopVer); } @@ -602,13 +675,7 @@ else if (req.stop()) { }); } - if (incMinorTopVer) { - assert !exchangeActions.empty() : exchangeActions; - - batch.exchangeActions(exchangeActions); - } - - return incMinorTopVer; + return res; } /** @@ -669,7 +736,7 @@ private Serializable joinDiscoveryData() { return new CacheClientReconnectDiscoveryData(cacheGrpsInfo, cachesInfo); } else { - assert ctx.config().isDaemon() || joinDiscoData != null || !ctx.state().active(); + assert ctx.config().isDaemon() || joinDiscoData != null; return joinDiscoData; } @@ -720,31 +787,6 @@ List cachesReceivedFromJoin(UUID joinedNodeId) { return started != null ? started : Collections.emptyList(); } - public void addJoinInfo() { - try { - Map grpCfgs = new HashMap<>(); - - for (CacheJoinNodeDiscoveryData.CacheInfo info : joinDiscoData.caches().values()) { - if (info.cacheData().config().getGroupName() == null) - continue; - - CacheConfiguration ccfg = grpCfgs.get(info.cacheData().config().getGroupName()); - - if (ccfg == null) - grpCfgs.put(info.cacheData().config().getGroupName(), info.cacheData().config()); - else - validateCacheGroupConfiguration(ccfg, info.cacheData().config()); - } - - String conflictErr = processJoiningNode(joinDiscoData, ctx.localNodeId(), true); - - if (conflictErr != null) - onJoinCacheException = new IgniteCheckedException("Failed to start configured cache. " + conflictErr); - }catch (IgniteCheckedException e){ - onJoinCacheException = e; - } - } - /** * Discovery event callback, executed from discovery thread. * @@ -771,10 +813,7 @@ void onDiscoveryEvent(int type, ClusterNode node, AffinityTopologyVersion topVer if (node.id().equals(ctx.discovery().localNode().id())) { if (gridData == null) { // First node starts. - assert joinDiscoData != null || !ctx.state().active(); - - if (ctx.state().active()) - addJoinInfo(); + assert joinDiscoData != null; initStartCachesForLocalJoin(true); } @@ -864,7 +903,7 @@ void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) { if (ctx.isDaemon() || data.commonData() == null) return; - assert joinDiscoData != null || disconnectedState() || !ctx.state().active(); + assert joinDiscoData != null || disconnectedState(); assert data.commonData() instanceof CacheNodeCommonDiscoveryData : data; CacheNodeCommonDiscoveryData cachesData = (CacheNodeCommonDiscoveryData)data.commonData(); @@ -965,7 +1004,7 @@ void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) { } } - gridData = new GridData(cachesData, conflictErr); + gridData = new GridData(joinDiscoData, cachesData, conflictErr); if (!disconnectedState()) initStartCachesForLocalJoin(false); @@ -977,11 +1016,20 @@ void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) { * @param firstNode {@code True} if first node in cluster starts. */ private void initStartCachesForLocalJoin(boolean firstNode) { - assert locJoinStartCaches == null; + assert F.isEmpty(locJoinStartCaches) : locJoinStartCaches; + + if (ctx.state().clusterState().transition()) { + joinOnTransition = true; - locJoinStartCaches = new ArrayList<>(); + return; + } if (joinDiscoData != null) { + locJoinStartCaches = new ArrayList<>(); + locCfgsForActivation = new HashMap<>(); + + boolean active = ctx.state().clusterState().active(); + for (DynamicCacheDescriptor desc : registeredCaches.values()) { if (firstNode && !joinDiscoData.caches().containsKey(desc.cacheName())) continue; @@ -997,13 +1045,13 @@ private void initStartCachesForLocalJoin(boolean firstNode) { DynamicCacheDescriptor desc0 = new DynamicCacheDescriptor(ctx, locCfg.cacheData().config(), - desc.cacheType(), - desc.groupDescriptor(), - desc.template(), - desc.receivedFrom(), - desc.staticallyConfigured(), - desc.sql(), - desc.deploymentId(), + desc.cacheType(), + desc.groupDescriptor(), + desc.template(), + desc.receivedFrom(), + desc.staticallyConfigured(), + desc.sql(), + desc.deploymentId(), new QuerySchema(locCfg.cacheData().queryEntities())); desc0.startTopologyVersion(desc.startTopologyVersion()); @@ -1016,14 +1064,126 @@ private void initStartCachesForLocalJoin(boolean firstNode) { if (locCfg != null || joinDiscoData.startCaches() || CU.affinityNode(ctx.discovery().localNode(), desc.groupDescriptor().config().getNodeFilter())) { - // Move system and internal caches first. - if (desc.cacheType().userCache()) - locJoinStartCaches.add(new T2<>(desc, nearCfg)); + if (active) { + // Move system and internal caches first. + if (desc.cacheType().userCache()) + locJoinStartCaches.add(new T2<>(desc, nearCfg)); + else + locJoinStartCaches.add(0, new T2<>(desc, nearCfg)); + } else - locJoinStartCaches.add(0, new T2<>(desc, nearCfg)); + locCfgsForActivation.put(desc.cacheName(), new T2<>(desc.cacheConfiguration(), nearCfg)); + } + } + } + } + + /** + * @param msg Message. + */ + void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) { + if (joinOnTransition) { + initStartCachesForLocalJoin(false); + + joinOnTransition = false; + } + } + + /** + * @param msg Message. + * @param topVer Current topology version. + * @return Exchange action. + * @throws IgniteCheckedException If configuration validation failed. + */ + ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer) + throws IgniteCheckedException { + ExchangeActions exchangeActions = new ExchangeActions(); + + if (msg.activate()) { + for (DynamicCacheDescriptor desc : registeredCaches.values()) { + desc.startTopologyVersion(topVer); + + T2 locCfg = !F.isEmpty(locCfgsForActivation) ? + locCfgsForActivation.get(desc.cacheName()) : null; + + DynamicCacheChangeRequest req = new DynamicCacheChangeRequest(msg.requestId(), + desc.cacheName(), + msg.initiatorNodeId()); + + req.startCacheConfiguration(desc.cacheConfiguration()); + req.cacheType(desc.cacheType()); + + if (locCfg != null) { + if (locCfg.get1() != null) + req.startCacheConfiguration(locCfg.get1()); + + req.nearCacheConfiguration(locCfg.get2()); + + req.locallyConfigured(true); + } + + exchangeActions.addCacheToStart(req, desc); + } + + for (CacheGroupDescriptor grpDesc : registeredCacheGroups().values()) + exchangeActions.addCacheGroupToStart(grpDesc); + + List storedCfgs = msg.storedCacheConfigurations(); + + if (storedCfgs != null) { + List reqs = new ArrayList<>(); + + IgniteUuid deplymentId = IgniteUuid.fromUuid(msg.requestId()); + + for (StoredCacheData storedCfg : storedCfgs) { + CacheConfiguration ccfg = storedCfg.config(); + + if (!registeredCaches.containsKey(ccfg.getName())) { + DynamicCacheChangeRequest req = new DynamicCacheChangeRequest(msg.requestId(), + ccfg.getName(), + msg.initiatorNodeId()); + + req.deploymentId(deplymentId); + req.startCacheConfiguration(ccfg); + req.cacheType(ctx.cache().cacheType(ccfg.getName())); + req.schema(new QuerySchema(storedCfg.queryEntities())); + + reqs.add(req); + } + } + + CacheChangeProcessResult res = processCacheChangeRequests(exchangeActions, reqs, topVer, true); + + if (!res.errs.isEmpty()) { + IgniteCheckedException err = new IgniteCheckedException("Failed to activate cluster."); + + for (IgniteCheckedException err0 : res.errs) + err.addSuppressed(err0); + + throw err; } } } + else { + locCfgsForActivation = new HashMap<>(); + + for (DynamicCacheDescriptor desc : registeredCaches.values()) { + DynamicCacheChangeRequest req = DynamicCacheChangeRequest.stopRequest(ctx, + desc.cacheName(), + desc.sql(), + false); + + exchangeActions.addCacheToStop(req, desc); + + if (ctx.discovery().cacheClientNode(ctx.discovery().localNode(), desc.cacheName())) + locCfgsForActivation.put(desc.cacheName(), new T2<>((CacheConfiguration)null, (NearCacheConfiguration)null)); + } + + for (CacheGroupDescriptor grpDesc : registeredCacheGroups().values()) + exchangeActions.addCacheGroupToStop(grpDesc, false); + } + + return exchangeActions; } /** @@ -1053,16 +1213,20 @@ else if (joiningNodeData instanceof CacheJoinNodeDiscoveryData) * @param clientNodeId Client node ID. */ private void processClientReconnectData(CacheClientReconnectDiscoveryData clientData, UUID clientNodeId) { - for (CacheClientReconnectDiscoveryData.CacheInfo cacheInfo : clientData.clientCaches().values()) { - String cacheName = cacheInfo.config().getName(); + DiscoveryDataClusterState state = ctx.state().clusterState(); + + if (state.active() && !state.transition()) { + for (CacheClientReconnectDiscoveryData.CacheInfo cacheInfo : clientData.clientCaches().values()) { + String cacheName = cacheInfo.config().getName(); - if (surviveReconnect(cacheName)) - ctx.discovery().addClientNode(cacheName, clientNodeId, false); - else { - DynamicCacheDescriptor desc = registeredCaches.get(cacheName); + if (surviveReconnect(cacheName)) + ctx.discovery().addClientNode(cacheName, clientNodeId, false); + else { + DynamicCacheDescriptor desc = registeredCaches.get(cacheName); - if (desc != null && desc.deploymentId().equals(cacheInfo.deploymentId())) - ctx.discovery().addClientNode(cacheName, clientNodeId, cacheInfo.nearCache()); + if (desc != null && desc.deploymentId().equals(cacheInfo.deploymentId())) + ctx.discovery().addClientNode(cacheName, clientNodeId, cacheInfo.nearCache()); + } } } } @@ -1371,6 +1535,7 @@ ConcurrentMap registeredTemplates() { */ void onDisconnect() { cachesOnDisconnect = new CachesOnDisconnect( + ctx.state().clusterState(), new HashMap<>(registeredCacheGrps), new HashMap<>(registeredCaches)); @@ -1382,57 +1547,82 @@ void onDisconnect() { } /** + * @param active {@code True} if reconnected to active cluster. + * @param transition {@code True} if reconnected while state transition in progress. * @return Information about stopped caches and cache groups. */ - ClusterCachesReconnectResult onReconnected() { + ClusterCachesReconnectResult onReconnected(boolean active, boolean transition) { assert disconnectedState(); Set stoppedCaches = new HashSet<>(); Set stoppedCacheGrps = new HashSet<>(); - for (Map.Entry e : cachesOnDisconnect.cacheGrps.entrySet()) { - CacheGroupDescriptor locDesc = e.getValue(); - - CacheGroupDescriptor desc; - boolean stopped = true; + if (!active) { + joinOnTransition = transition; - if (locDesc.sharedGroup()) { - desc = cacheGroupByName(locDesc.groupName()); + if (F.isEmpty(locCfgsForActivation)) { + locCfgsForActivation = new HashMap<>(); - if (desc != null && desc.deploymentId().equals(locDesc.deploymentId())) - stopped = false; + for (IgniteInternalCache cache : ctx.cache().caches()) { + locCfgsForActivation.put(cache.name(), + new T2<>((CacheConfiguration)null, cache.configuration().getNearConfiguration())); + } } - else { - desc = nonSharedCacheGroupByCacheName(locDesc.config().getName()); - if (desc != null && - (surviveReconnect(locDesc.config().getName()) || desc.deploymentId().equals(locDesc.deploymentId()))) - stopped = false; - } + for (Map.Entry e : cachesOnDisconnect.cacheGrps.entrySet()) + stoppedCacheGrps.add(e.getValue().groupId()); - if (stopped) - stoppedCacheGrps.add(locDesc.groupId()); - else - assert locDesc.groupId() == desc.groupId(); + for (Map.Entry e : cachesOnDisconnect.caches.entrySet()) + stoppedCaches.add(e.getKey()); } + else { + for (Map.Entry e : cachesOnDisconnect.cacheGrps.entrySet()) { + CacheGroupDescriptor locDesc = e.getValue(); - for (Map.Entry e : cachesOnDisconnect.caches.entrySet()) { - DynamicCacheDescriptor desc = e.getValue(); + CacheGroupDescriptor desc; + boolean stopped = true; - String cacheName = e.getKey(); + if (locDesc.sharedGroup()) { + desc = cacheGroupByName(locDesc.groupName()); - boolean stopped; + if (desc != null && desc.deploymentId().equals(locDesc.deploymentId())) + stopped = false; + } + else { + desc = nonSharedCacheGroupByCacheName(locDesc.config().getName()); - if (!surviveReconnect(cacheName) || !ctx.state().active()) { - DynamicCacheDescriptor newDesc = registeredCaches.get(cacheName); + if (desc != null && + (surviveReconnect(locDesc.config().getName()) || desc.deploymentId().equals(locDesc.deploymentId()))) + stopped = false; + } - stopped = newDesc == null || !desc.deploymentId().equals(newDesc.deploymentId()); + if (stopped) + stoppedCacheGrps.add(locDesc.groupId()); + else + assert locDesc.groupId() == desc.groupId(); } - else - stopped = false; - if (stopped) - stoppedCaches.add(cacheName); + for (Map.Entry e : cachesOnDisconnect.caches.entrySet()) { + DynamicCacheDescriptor desc = e.getValue(); + + String cacheName = e.getKey(); + + boolean stopped; + + if (!surviveReconnect(cacheName)) { + DynamicCacheDescriptor newDesc = registeredCaches.get(cacheName); + + stopped = newDesc == null || !desc.deploymentId().equals(newDesc.deploymentId()); + } + else + stopped = false; + + if (stopped) + stoppedCaches.add(cacheName); + } + + if (!cachesOnDisconnect.clusterActive()) + initStartCachesForLocalJoin(false); } if (clientReconnectReqs != null) { @@ -1450,7 +1640,7 @@ ClusterCachesReconnectResult onReconnected() { /** * @return {@code True} if client node is currently in disconnected state. */ - public boolean disconnectedState() { + private boolean disconnectedState() { return cachesOnDisconnect != null; } @@ -1462,19 +1652,13 @@ private boolean surviveReconnect(String cacheName) { return CU.isUtilityCache(cacheName); } - /** - * - */ - void clearCaches() { - registeredCacheGrps.clear(); - - registeredCaches.clear(); - } - /** * */ private static class GridData { + /** */ + private final CacheJoinNodeDiscoveryData joinDiscoData; + /** */ private final CacheNodeCommonDiscoveryData gridData; @@ -1482,10 +1666,12 @@ private static class GridData { private final String conflictErr; /** + * @param joinDiscoData Discovery data collected for local node join. * @param gridData Grid data. * @param conflictErr Cache configuration conflict error. */ - GridData(CacheNodeCommonDiscoveryData gridData, String conflictErr) { + GridData(CacheJoinNodeDiscoveryData joinDiscoData, CacheNodeCommonDiscoveryData gridData, String conflictErr) { + this.joinDiscoData = joinDiscoData; this.gridData = gridData; this.conflictErr = conflictErr; } @@ -1495,6 +1681,9 @@ private static class GridData { * */ private static class CachesOnDisconnect { + /** */ + final DiscoveryDataClusterState state; + /** */ final Map cacheGrps; @@ -1502,12 +1691,37 @@ private static class CachesOnDisconnect { final Map caches; /** + * @param state Cluster state. * @param cacheGrps Cache groups. * @param caches Caches. */ - CachesOnDisconnect(Map cacheGrps, Map caches) { + CachesOnDisconnect(DiscoveryDataClusterState state, + Map cacheGrps, + Map caches) { + this.state = state; this.cacheGrps = cacheGrps; this.caches = caches; } + + /** + * @return {@code True} if cluster was in active state. + */ + boolean clusterActive() { + return state.active() && !state.transition(); + } + } + + /** + * + */ + private static class CacheChangeProcessResult { + /** */ + private boolean needExchange; + + /** */ + private final List addedDescs = new ArrayList<>(); + + /** */ + private final List errs = new ArrayList<>(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeRequest.java index 6d5eaf32ed2a7..2fd878084577e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeRequest.java @@ -83,15 +83,15 @@ public class DynamicCacheChangeRequest implements Serializable { /** */ private UUID rcvdFrom; - /** Cache state. Set to non-null when global state is changed. */ - private ClusterState state; - /** Reset lost partitions flag. */ private boolean resetLostPartitions; /** Dynamic schema. */ private QuerySchema schema; + /** */ + private transient boolean locallyConfigured; + /** * @param reqId Unique request ID. * @param cacheName Cache stop name. @@ -100,28 +100,12 @@ public class DynamicCacheChangeRequest implements Serializable { public DynamicCacheChangeRequest(UUID reqId, String cacheName, UUID initiatingNodeId) { assert reqId != null; assert cacheName != null; - assert initiatingNodeId != null; this.reqId = reqId; this.cacheName = cacheName; this.initiatingNodeId = initiatingNodeId; } - /** - * @param reqId Unique request ID. - * @param state New cluster state. - * @param initiatingNodeId Initiating node ID. - */ - public DynamicCacheChangeRequest(UUID reqId, ClusterState state, UUID initiatingNodeId) { - assert reqId != null; - assert state != null; - assert initiatingNodeId != null; - - this.reqId = reqId; - this.state = state; - this.initiatingNodeId = initiatingNodeId; - } - /** * @param ctx Context. * @param cacheName Cache name. @@ -182,20 +166,6 @@ public UUID requestId() { return reqId; } - /** - * @return State. - */ - public ClusterState state() { - return state; - } - - /** - * @return {@code True} if global caches state is changes. - */ - public boolean globalStateChange() { - return state != null; - } - /** * @param template {@code True} if this is request for adding template configuration. */ @@ -253,7 +223,7 @@ public boolean stop() { } /** - * + * @return Destroy flag. */ public boolean destroy(){ return destroy; @@ -420,6 +390,20 @@ public void schema(QuerySchema schema) { this.schema = schema != null ? schema.copy() : null; } + /** + * @return Locally configured flag. + */ + public boolean locallyConfigured() { + return locallyConfigured; + } + + /** + * @param locallyConfigured Locally configured flag. + */ + public void locallyConfigured(boolean locallyConfigured) { + this.locallyConfigured = locallyConfigured; + } + /** {@inheritDoc} */ @Override public String toString() { return "DynamicCacheChangeRequest [cacheName=" + cacheName() + diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java index 9caf9aa333160..e9ece5afb5576 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java @@ -50,7 +50,7 @@ public class ExchangeActions { private Map cachesToResetLostParts; /** */ - private ClusterState newState; + private StateChangeRequest stateChangeReq; /** * @param grpId Group ID. @@ -89,7 +89,7 @@ public boolean clientOnlyExchange() { /** * @return New caches start requests. */ - Collection cacheStartRequests() { + public Collection cacheStartRequests() { return cachesToStart != null ? cachesToStart.values() : Collections.emptyList(); } @@ -184,19 +184,31 @@ public boolean cacheStarted(int cacheId) { } /** - * @param state New cluster state. + * @param stateChange Cluster state change request. */ - void newClusterState(ClusterState state) { - assert state != null; + public void stateChangeRequest(StateChangeRequest stateChange) { + this.stateChangeReq = stateChange; + } + + /** + * @return {@code True} if has deactivate request. + */ + public boolean deactivate() { + return stateChangeReq != null && !stateChangeReq.activate(); + } - newState = state; + /** + * @return {@code True} if has activate request. + */ + public boolean activate() { + return stateChangeReq != null && stateChangeReq.activate(); } /** - * @return New cluster state if state change was requested. + * @return Cluster state change request. */ - @Nullable public ClusterState newClusterState() { - return newState; + @Nullable public StateChangeRequest stateChangeRequest() { + return stateChangeReq; } /** @@ -328,13 +340,14 @@ public boolean empty() { F.isEmpty(cachesToStop) && F.isEmpty(cacheGrpsToStart) && F.isEmpty(cacheGrpsToStop) && - F.isEmpty(cachesToResetLostParts); + F.isEmpty(cachesToResetLostParts) && + stateChangeReq == null; } /** * */ - static class ActionData { + public static class ActionData { /** */ private final DynamicCacheChangeRequest req; @@ -429,6 +442,6 @@ public boolean destroy() { ", startGrps=" + startGrps + ", stopGrps=" + stopGrps + ", resetParts=" + (cachesToResetLostParts != null ? cachesToResetLostParts.keySet() : null) + - ", newState=" + newState + ']'; + ", stateChangeRequest=" + stateChangeReq + ']'; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java index a96730556b940..a9692f8c3cc2b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java @@ -21,7 +21,6 @@ import java.util.UUID; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.CacheEvent; -import org.apache.ignite.events.CacheRebalancingEvent; import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.util.typedef.F; @@ -32,7 +31,6 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_READ; -import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_UNLOADED; import static org.apache.ignite.events.EventType.EVT_CACHE_STARTED; import static org.apache.ignite.events.EventType.EVT_CACHE_STOPPED; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java index 8ba10a2225f1b..7735f74cf2354 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.processors.cache; import java.util.Collection; -import java.util.Set; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.eviction.EvictionFilter; import org.apache.ignite.cache.eviction.EvictionPolicy; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java index 2de3808458698..f9d1114fd8515 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java @@ -1405,30 +1405,33 @@ private int messageIndex(Class msgCls) { } /** + * @param cctx Context. * @param topic Topic. * @param c Handler. */ - public void addOrderedCacheHandler(Object topic, IgniteBiInClosure c) { - addOrderedHandler(false, topic, c); + public void addOrderedCacheHandler(GridCacheSharedContext cctx, Object topic, IgniteBiInClosure c) { + addOrderedHandler(cctx, false, topic, c); } /** + * @param cctx Context. * @param topic Topic. * @param c Handler. */ - public void addOrderedCacheGroupHandler(Object topic, IgniteBiInClosure c) { - addOrderedHandler(true, topic, c); + public void addOrderedCacheGroupHandler(GridCacheSharedContext cctx, Object topic, IgniteBiInClosure c) { + addOrderedHandler(cctx, true, topic, c); } /** * Adds ordered message handler. * + * @param cctx Context. * @param cacheGrp {@code True} if cache group message, {@code false} if cache message. * @param topic Topic. * @param c Handler. */ @SuppressWarnings({"unchecked"}) - private void addOrderedHandler(boolean cacheGrp, Object topic, IgniteBiInClosure c) { + private void addOrderedHandler(GridCacheSharedContext cctx, boolean cacheGrp, Object topic, IgniteBiInClosure c) { MessageHandlers msgHandlers = cacheGrp ? grpHandlers : cacheHandlers; IgniteLogger log0 = log; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java index 24433ded259b6..a6907b98d4dea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java @@ -274,10 +274,11 @@ else if (log.isDebugEnabled()) pendingExplicit = GridConcurrentFactory.newMap(); } - /** {@inheritDoc} */ - @Override protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException { - if (!reconnect) - cctx.gridEvents().addLocalEventListener(discoLsnr, EVT_NODE_FAILED, EVT_NODE_LEFT); + /** + * Cache futures listener must be registered after communication listener. + */ + public void registerEventListener() { + cctx.gridEvents().addLocalEventListener(discoLsnr, EVT_NODE_FAILED, EVT_NODE_LEFT); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 93310e3adbefb..22345d27dc4c6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -59,6 +59,7 @@ import org.apache.ignite.internal.events.DiscoveryCustomEvent; import org.apache.ignite.internal.managers.discovery.DiscoCache; import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; +import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData; import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener; import org.apache.ignite.internal.pagemem.snapshot.StartSnapshotOperationAckDiscoveryMessage; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; @@ -81,6 +82,8 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; import org.apache.ignite.internal.processors.query.schema.SchemaNodeLeaveExchangeWorkerTask; import org.apache.ignite.internal.processors.timeout.GridTimeoutObject; import org.apache.ignite.internal.util.GridListSet; @@ -192,6 +195,9 @@ public class GridCachePartitionExchangeManager extends GridCacheSharedMana /** */ private DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS"); + /** Events received while cluster state transition was in progress. */ + private final List pendingEvts = new ArrayList<>(); + /** Discovery listener. */ private final DiscoveryEventListener discoLsnr = new DiscoveryEventListener() { @Override public void onEvent(DiscoveryEvent evt, DiscoCache cache) { @@ -199,109 +205,53 @@ public class GridCachePartitionExchangeManager extends GridCacheSharedMana return; try { - ClusterNode loc = cctx.localNode(); - - assert evt.type() == EVT_NODE_JOINED || evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED || - evt.type() == EVT_DISCOVERY_CUSTOM_EVT; - - final ClusterNode n = evt.eventNode(); - - GridDhtPartitionExchangeId exchId = null; - GridDhtPartitionsExchangeFuture exchFut = null; - - if (evt.type() != EVT_DISCOVERY_CUSTOM_EVT) { - assert !loc.id().equals(n.id()); - - if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) { - assert cctx.discovery().node(n.id()) == null; - - // Avoid race b/w initial future add and discovery event. - GridDhtPartitionsExchangeFuture initFut = null; - - if (readyTopVer.get().equals(AffinityTopologyVersion.NONE)) { - initFut = exchangeFuture(initialExchangeId(), null, null, null, null); - - initFut.onNodeLeft(n); - } - - for (GridDhtPartitionsExchangeFuture f : exchFuts.values()) { - if (f != initFut) - f.onNodeLeft(n); - } - } + if (evt.type() == EVT_DISCOVERY_CUSTOM_EVT && + (((DiscoveryCustomEvent)evt).customMessage() instanceof ChangeGlobalStateMessage)) { + ChangeGlobalStateMessage stateChangeMsg = + (ChangeGlobalStateMessage)((DiscoveryCustomEvent)evt).customMessage(); - assert evt.type() != EVT_NODE_JOINED || n.order() > loc.order() : - "Node joined with smaller-than-local " + - "order [newOrder=" + n.order() + ", locOrder=" + loc.order() + ']'; + if (stateChangeMsg.exchangeActions() == null) + return; - exchId = exchangeId(n.id(), - affinityTopologyVersion(evt), - evt.type()); + onDiscoveryEvent(evt, cache); - exchFut = exchangeFuture(exchId, evt, cache,null, null); + return; } - else { - DiscoveryCustomMessage customMsg = ((DiscoveryCustomEvent)evt).customMessage(); + if (evt.type() == EVT_DISCOVERY_CUSTOM_EVT && + (((DiscoveryCustomEvent)evt).customMessage() instanceof ChangeGlobalStateFinishMessage)) { + ChangeGlobalStateFinishMessage stateFinishMsg = + (ChangeGlobalStateFinishMessage)((DiscoveryCustomEvent)evt).customMessage(); - if (customMsg instanceof DynamicCacheChangeBatch) { - DynamicCacheChangeBatch batch = (DynamicCacheChangeBatch)customMsg; - - ExchangeActions exchActions = batch.exchangeActions(); - - if (exchActions != null) { - exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type()); + if (stateFinishMsg.clusterActive()) { + for (PendingDiscoveryEvent pendingEvt : pendingEvts) { + if (log.isDebugEnabled()) + log.debug("Process pending event: " + pendingEvt.event()); - exchFut = exchangeFuture(exchId, evt, cache, exchActions, null); + onDiscoveryEvent(pendingEvt.event(), pendingEvt.discoCache()); } } - else if (customMsg instanceof CacheAffinityChangeMessage) { - CacheAffinityChangeMessage msg = (CacheAffinityChangeMessage)customMsg; - - if (msg.exchangeId() == null) { - if (msg.exchangeNeeded()) { - exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type()); - - exchFut = exchangeFuture(exchId, evt, cache, null, msg); - } - } - else if (msg.exchangeId().topologyVersion().topologyVersion() >= cctx.discovery().localJoinEvent().topologyVersion()) - exchangeFuture(msg.exchangeId(), null, null, null, null) - .onAffinityChangeMessage(evt.eventNode(), msg); + else { + for (PendingDiscoveryEvent pendingEvt : pendingEvts) + processEventInactive(pendingEvt.event(), pendingEvt.discoCache()); } - else if (customMsg instanceof StartSnapshotOperationAckDiscoveryMessage - && ((StartSnapshotOperationAckDiscoveryMessage)customMsg).needExchange()) { - exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type()); - exchFut = exchangeFuture(exchId, evt, null, null, null); - } - else { - // Process event as custom discovery task if needed. - CachePartitionExchangeWorkerTask task = - cctx.cache().exchangeTaskForCustomDiscoveryMessage(customMsg); + pendingEvts.clear(); - if (task != null) - exchWorker.addCustomTask(task); - } + return; } - if (exchId != null) { + if (cache.state().transition()) { if (log.isDebugEnabled()) - log.debug("Discovery event (will start exchange): " + exchId); - - // Event callback - without this callback future will never complete. - exchFut.onEvent(exchId, evt, cache); + log.debug("Add pending event: " + evt); - // Start exchange process. - addFuture(exchFut); - } - else { - if (log.isDebugEnabled()) - log.debug("Do not start exchange for discovery event: " + evt); + pendingEvts.add(new PendingDiscoveryEvent(evt, cache)); } + else if (cache.state().active()) + onDiscoveryEvent(evt, cache); + else + processEventInactive(evt, cache); - // Notify indexing engine about node leave so that we can re-map coordinator accordingly. - if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) - exchWorker.addCustomTask(new SchemaNodeLeaveExchangeWorkerTask(evt.eventNode())); + notifyNodeFail(evt); } finally { leaveBusy(); @@ -309,6 +259,29 @@ else if (customMsg instanceof StartSnapshotOperationAckDiscoveryMessage } }; + /** + * @param evt Event. + */ + private void notifyNodeFail(DiscoveryEvent evt) { + if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) { + final ClusterNode n = evt.eventNode(); + + assert cctx.discovery().node(n.id()) == null; + + for (GridDhtPartitionsExchangeFuture f : exchFuts.values()) + f.onNodeLeft(n); + } + } + + /** + * @param evt Event. + * @param cache Discovery data cache. + */ + private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) { + if (log.isDebugEnabled()) + log.debug("Ignore event, cluster is inactive: " + evt); + } + /** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { super.start0(); @@ -338,12 +311,158 @@ else if (customMsg instanceof StartSnapshotOperationAckDiscoveryMessage processSinglePartitionRequest(node, msg); } }); + + if (!cctx.kernalContext().clientNode()) { + for (int cnt = 0; cnt < cctx.gridConfig().getRebalanceThreadPoolSize(); cnt++) { + final int idx = cnt; + + cctx.io().addOrderedCacheGroupHandler(cctx, rebalanceTopic(cnt), new CI2() { + @Override public void apply(final UUID id, final GridCacheGroupIdMessage m) { + if (!enterBusy()) + return; + + try { + CacheGroupContext grp = cctx.cache().cacheGroup(m.groupId()); + + if (grp != null) { + if (m instanceof GridDhtPartitionSupplyMessage) { + grp.preloader().handleSupplyMessage(idx, id, (GridDhtPartitionSupplyMessage) m); + + return; + } + else if (m instanceof GridDhtPartitionDemandMessage) { + grp.preloader().handleDemandMessage(idx, id, (GridDhtPartitionDemandMessage) m); + + return; + } + } + + U.error(log, "Unsupported message type: " + m.getClass().getName()); + } + finally { + leaveBusy(); + } + } + }); + } + } + } + + /** + * Callback for local join event (needed since regular event for local join is not generated). + * + * @param evt Event. + * @param cache Cache. + */ + public void onLocalJoin(DiscoveryEvent evt, DiscoCache cache) { + discoLsnr.onEvent(evt, cache); + } + + /** + * @param evt Event. + * @param cache Discovery data cache. + */ + private void onDiscoveryEvent(DiscoveryEvent evt, DiscoCache cache) { + ClusterNode loc = cctx.localNode(); + + assert evt.type() == EVT_NODE_JOINED || evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED || + evt.type() == EVT_DISCOVERY_CUSTOM_EVT; + + final ClusterNode n = evt.eventNode(); + + GridDhtPartitionExchangeId exchId = null; + GridDhtPartitionsExchangeFuture exchFut = null; + + if (evt.type() != EVT_DISCOVERY_CUSTOM_EVT) { + assert evt.type() != EVT_NODE_JOINED || n.isLocal() || n.order() > loc.order() : + "Node joined with smaller-than-local " + + "order [newOrder=" + n.order() + ", locOrder=" + loc.order() + ']'; + + exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type()); + + exchFut = exchangeFuture(exchId, evt, cache,null, null); + } + else { + DiscoveryCustomMessage customMsg = ((DiscoveryCustomEvent)evt).customMessage(); + + if (customMsg instanceof ChangeGlobalStateMessage) { + ChangeGlobalStateMessage stateChangeMsg = (ChangeGlobalStateMessage)customMsg; + + ExchangeActions exchActions = stateChangeMsg.exchangeActions(); + + if (exchActions != null) { + exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type()); + + exchFut = exchangeFuture(exchId, evt, cache, exchActions, null); + } + } + else if (customMsg instanceof DynamicCacheChangeBatch) { + DynamicCacheChangeBatch batch = (DynamicCacheChangeBatch)customMsg; + + ExchangeActions exchActions = batch.exchangeActions(); + + if (exchActions != null) { + exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type()); + + exchFut = exchangeFuture(exchId, evt, cache, exchActions, null); + } + } + else if (customMsg instanceof CacheAffinityChangeMessage) { + CacheAffinityChangeMessage msg = (CacheAffinityChangeMessage)customMsg; + + if (msg.exchangeId() == null) { + if (msg.exchangeNeeded()) { + exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type()); + + exchFut = exchangeFuture(exchId, evt, cache, null, msg); + } + } + else if (msg.exchangeId().topologyVersion().topologyVersion() >= cctx.discovery().localJoinEvent().topologyVersion()) + exchangeFuture(msg.exchangeId(), null, null, null, null) + .onAffinityChangeMessage(evt.eventNode(), msg); + } + else if (customMsg instanceof StartSnapshotOperationAckDiscoveryMessage + && ((StartSnapshotOperationAckDiscoveryMessage)customMsg).needExchange()) { + exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type()); + + exchFut = exchangeFuture(exchId, evt, null, null, null); + } + else { + // Process event as custom discovery task if needed. + CachePartitionExchangeWorkerTask task = + cctx.cache().exchangeTaskForCustomDiscoveryMessage(customMsg); + + if (task != null) + exchWorker.addCustomTask(task); + } + } + + if (exchId != null) { + if (log.isDebugEnabled()) + log.debug("Discovery event (will start exchange): " + exchId); + + // Event callback - without this callback future will never complete. + exchFut.onEvent(exchId, evt, cache); + + // Start exchange process. + addFuture(exchFut); + } + else { + if (log.isDebugEnabled()) + log.debug("Do not start exchange for discovery event: " + evt); + } + + notifyNodeFail(evt); + + // Notify indexing engine about node leave so that we can re-map coordinator accordingly. + if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) + exchWorker.addCustomTask(new SchemaNodeLeaveExchangeWorkerTask(evt.eventNode())); } /** * @param task Task to run in exchange worker thread. */ - public void addCustomTask(CachePartitionExchangeWorkerTask task) { + void addCustomTask(CachePartitionExchangeWorkerTask task) { assert task != null; exchWorker.addCustomTask(task); @@ -371,9 +490,14 @@ private GridDhtPartitionExchangeId initialExchangeId() { return exchangeId(cctx.localNode().id(), startTopVer, EVT_NODE_JOINED); } - /** {@inheritDoc} */ - @Override protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException { - super.onKernalStart0(reconnect); + /** + * @param active Cluster state. + * @param reconnect Reconnect flag. + * @throws IgniteCheckedException If failed. + */ + public void onKernalStart(boolean active, boolean reconnect) throws IgniteCheckedException { + for (ClusterNode n : cctx.discovery().remoteNodes()) + cctx.versions().onReceived(n.id(), n.metrics().getLastDataVersion()); ClusterNode loc = cctx.localNode(); @@ -381,79 +505,49 @@ private GridDhtPartitionExchangeId initialExchangeId() { assert startTime > 0; - // Generate dummy discovery event for local node joining. - T2 locJoin = cctx.discovery().localJoin(); - - DiscoveryEvent discoEvt = locJoin.get1(); - DiscoCache discoCache = locJoin.get2(); - - GridDhtPartitionExchangeId exchId = initialExchangeId(); + DiscoveryLocalJoinData locJoin = cctx.discovery().localJoin(); - GridDhtPartitionsExchangeFuture fut = exchangeFuture(exchId, discoEvt, discoCache, null, null); + GridDhtPartitionsExchangeFuture fut = null; if (reconnect) reconnectExchangeFut = new GridFutureAdapter<>(); - exchWorker.addFirstExchangeFuture(fut); - - if (!cctx.kernalContext().clientNode()) { - for (int cnt = 0; cnt < cctx.gridConfig().getRebalanceThreadPoolSize(); cnt++) { - final int idx = cnt; - - cctx.io().addOrderedCacheGroupHandler(rebalanceTopic(cnt), new CI2() { - @Override public void apply(final UUID id, final GridCacheGroupIdMessage m) { - if (!enterBusy()) - return; - - try { - CacheGroupContext grp = cctx.cache().cacheGroup(m.groupId()); - - if (grp != null) { - if (m instanceof GridDhtPartitionSupplyMessage) { - grp.preloader().handleSupplyMessage(idx, id, (GridDhtPartitionSupplyMessage) m); - - return; - } - else if (m instanceof GridDhtPartitionDemandMessage) { - grp.preloader().handleDemandMessage(idx, id, (GridDhtPartitionDemandMessage) m); + if (active) { + DiscoveryEvent discoEvt = locJoin.event(); + DiscoCache discoCache = locJoin.discoCache(); - return; - } - } + GridDhtPartitionExchangeId exchId = initialExchangeId(); - U.error(log, "Unsupported message type: " + m.getClass().getName()); - } - finally { - leaveBusy(); - } - } - }); - } + fut = exchangeFuture(exchId, discoEvt, discoCache, null, null); } + else if (reconnect) + reconnectExchangeFut.onDone(); new IgniteThread(cctx.igniteInstanceName(), "exchange-worker", exchWorker).start(); if (reconnect) { - fut.listen(new CI1>() { - @Override public void apply(IgniteInternalFuture fut) { - try { - fut.get(); + if (fut != null) { + fut.listen(new CI1>() { + @Override public void apply(IgniteInternalFuture fut) { + try { + fut.get(); - for (CacheGroupContext grp : cctx.cache().cacheGroups()) - grp.preloader().onInitialExchangeComplete(null); + for (CacheGroupContext grp : cctx.cache().cacheGroups()) + grp.preloader().onInitialExchangeComplete(null); - reconnectExchangeFut.onDone(); - } - catch (IgniteCheckedException e) { - for (CacheGroupContext grp : cctx.cache().cacheGroups()) - grp.preloader().onInitialExchangeComplete(e); + reconnectExchangeFut.onDone(); + } + catch (IgniteCheckedException e) { + for (CacheGroupContext grp : cctx.cache().cacheGroups()) + grp.preloader().onInitialExchangeComplete(e); - reconnectExchangeFut.onDone(e); + reconnectExchangeFut.onDone(e); + } } - } - }); + }); + } } - else { + else if (fut != null) { if (log.isDebugEnabled()) log.debug("Beginning to wait on local exchange future: " + fut); @@ -489,10 +583,8 @@ else if (m instanceof GridDhtPartitionDemandMessage) { } } - AffinityTopologyVersion nodeStartVer = new AffinityTopologyVersion(discoEvt.topologyVersion(), 0); - for (CacheGroupContext grp : cctx.cache().cacheGroups()) { - if (nodeStartVer.equals(grp.localStartVersion())) + if (locJoin.joinTopologyVersion().equals(grp.localStartVersion())) grp.preloader().onInitialExchangeComplete(null); } @@ -1668,28 +1760,6 @@ private void dumpDiagnosticInfo(IgniteInternalFuture fut, ((IgniteDiagnosticAware)fut).addDiagnosticRequest(ctx); } - /** - * @param deque Deque to poll from. - * @param time Time to wait. - * @param w Worker. - * @return Polled item. - * @throws InterruptedException If interrupted. - */ - @Nullable private T poll(BlockingQueue deque, long time, GridWorker w) throws InterruptedException { - assert w != null; - - // There is currently a case where {@code interrupted} - // flag on a thread gets flipped during stop which causes the pool to hang. This check - // will always make sure that interrupted flag gets reset before going into wait conditions. - // The true fix should actually make sure that interrupted flag does not get reset or that - // interrupted exception gets propagated. Until we find a real fix, this method should - // always work to make sure that there is no hanging during stop. - if (w.isCancelled()) - Thread.currentThread().interrupt(); - - return deque.poll(time, MILLISECONDS); - } - /** * Exchange future thread. All exchanges happen only by one thread and next * exchange will not start until previous one completes. @@ -1709,15 +1779,6 @@ private ExchangeWorker() { super(cctx.igniteInstanceName(), "partition-exchanger", GridCachePartitionExchangeManager.this.log); } - /** - * Add first exchange future. - * - * @param exchFut Exchange future. - */ - void addFirstExchangeFuture(GridDhtPartitionsExchangeFuture exchFut) { - futQ.addFirst(exchFut); - } - /** * @param exchFut Exchange future. */ @@ -1946,7 +2007,7 @@ void dumpExchangeDebugInfo() { } } - if (!exchFut.skipPreload() && cctx.kernalContext().state().active()) { + if (!exchFut.skipPreload() ) { assignsMap = new HashMap<>(); for (CacheGroupContext grp : cctx.cache().cacheGroups()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 0f859eb828e2b..624dec0e8914c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -107,6 +107,9 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager; import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; +import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor; import org.apache.ignite.internal.processors.plugin.CachePluginManager; import org.apache.ignite.internal.processors.query.QuerySchema; @@ -692,36 +695,27 @@ private void cleanup(CacheConfiguration cfg, @Nullable Object rsrc, boolean near for (GridCacheSharedManager mgr : sharedCtx.managers()) mgr.start(sharedCtx); - if (ctx.config().isDaemon()) { - ctx.state().cacheProcessorStarted(new CacheJoinNodeDiscoveryData( - IgniteUuid.randomUuid(), - Collections.emptyMap(), - Collections.emptyMap(), - false - )); - - return; - } - - Map caches = new HashMap<>(); + if (!ctx.isDaemon()) { + Map caches = new HashMap<>(); - Map templates = new HashMap<>(); + Map templates = new HashMap<>(); - addCacheOnJoinFromConfig(caches, templates); + addCacheOnJoinFromConfig(caches, templates); - CacheJoinNodeDiscoveryData discoData = new CacheJoinNodeDiscoveryData( - IgniteUuid.randomUuid(), - caches, - templates, - startAllCachesOnClientStart() - ); + CacheJoinNodeDiscoveryData discoData = new CacheJoinNodeDiscoveryData( + IgniteUuid.randomUuid(), + caches, + templates, + startAllCachesOnClientStart() + ); - cachesInfo.onStart(discoData); + cachesInfo.onStart(discoData); - if (log.isDebugEnabled()) - log.debug("Started cache processor."); + if (log.isDebugEnabled()) + log.debug("Started cache processor."); + } - ctx.state().cacheProcessorStarted(discoData); + ctx.state().cacheProcessorStarted(); } /** @@ -830,51 +824,38 @@ public Collection cacheGroups() { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public void onKernalStart() throws IgniteCheckedException { - boolean active = ctx.state().active(); + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { + if (ctx.isDaemon()) + return; try { - boolean checkConsistency = - !ctx.config().isDaemon() && !getBoolean(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK); + boolean checkConsistency = !getBoolean(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK); if (checkConsistency) checkConsistency(); - if (active && cachesInfo.onJoinCacheException() != null) - throw new IgniteCheckedException(cachesInfo.onJoinCacheException()); - cachesInfo.onKernalStart(checkConsistency); - if (active && !ctx.clientNode() && !ctx.isDaemon()) - sharedCtx.database().lock(); - - // Must start database before start first cache. - sharedCtx.database().onKernalStart(false); - ctx.query().onCacheKernalStart(); - // In shared context, we start exchange manager and wait until processed local join - // event, all caches which we get on join will be start. - for (GridCacheSharedManager mgr : sharedCtx.managers()) { - if (sharedCtx.database() != mgr) - mgr.onKernalStart(false); - } + sharedCtx.mvcc().registerEventListener(); + + sharedCtx.exchange().onKernalStart(active, false); } finally { cacheStartedLatch.countDown(); } + if (!ctx.clientNode()) + addRemovedItemsCleanupTask(Long.getLong(IGNITE_CACHE_REMOVED_ENTRIES_TTL, 10_000)); + // Escape if cluster inactive. if (!active) return; - if (!ctx.config().isDaemon()) - ctx.cacheObjects().onUtilityCacheStarted(); - ctx.service().onUtilityCacheStarted(); - final AffinityTopologyVersion startTopVer = - new AffinityTopologyVersion(ctx.discovery().localJoinEvent().topologyVersion(), 0); + final AffinityTopologyVersion startTopVer = ctx.discovery().localJoin().joinTopologyVersion(); final List syncFuts = new ArrayList<>(caches.size()); @@ -894,15 +875,8 @@ public Collection cacheGroups() { } }); - // Avoid iterator creation. - //noinspection ForLoopReplaceableByForEach for (int i = 0, size = syncFuts.size(); i < size; i++) syncFuts.get(i).get(); - - assert ctx.config().isDaemon() || caches.containsKey(CU.UTILITY_CACHE_NAME) : "Utility cache should be started"; - - if (!ctx.clientNode() && !ctx.isDaemon()) - addRemovedItemsCleanupTask(Long.getLong(IGNITE_CACHE_REMOVED_ENTRIES_TTL, 10_000)); } /** @@ -969,8 +943,6 @@ public void stopCaches(boolean cancel) { for (CacheGroupContext grp : cacheGrps.values()) stopCacheGroup(grp.groupId()); - - cachesInfo.clearCaches(); } /** @@ -1097,7 +1069,11 @@ private void stopCacheOnReconnect(GridCacheContext cctx, List @Override public IgniteInternalFuture onReconnected(boolean clusterRestarted) throws IgniteCheckedException { List reconnected = new ArrayList<>(caches.size()); - ClusterCachesReconnectResult reconnectRes = cachesInfo.onReconnected(); + DiscoveryDataClusterState state = ctx.state().clusterState(); + + boolean active = state.active() && !state.transition(); + + ClusterCachesReconnectResult reconnectRes = cachesInfo.onReconnected(active, state.transition()); final List stoppedCaches = new ArrayList<>(); @@ -1135,7 +1111,7 @@ private void stopCacheOnReconnect(GridCacheContext cctx, List grp.onReconnected(); } - sharedCtx.onReconnected(); + sharedCtx.onReconnected(active); for (GridCacheAdapter cache : reconnected) cache.context().gate().reconnected(false); @@ -1750,17 +1726,26 @@ public CacheMode cacheMode(String cacheName) { } /** + * @return Caches to be started when this node starts. + */ + public List> cachesToStartOnLocalJoin() { + return cachesInfo.cachesToStartOnLocalJoin(); + } + + /** + * @param caches Caches to start. * @param exchTopVer Current exchange version. * @throws IgniteCheckedException If failed. */ - public void startCachesOnLocalJoin(AffinityTopologyVersion exchTopVer) throws IgniteCheckedException { - List> caches = cachesInfo.cachesToStartOnLocalJoin(); - + public void startCachesOnLocalJoin(List> caches, + AffinityTopologyVersion exchTopVer) + throws IgniteCheckedException { if (!F.isEmpty(caches)) { for (T2 t : caches) { DynamicCacheDescriptor desc = t.get1(); prepareCacheStart( + desc.cacheConfiguration(), desc, t.get2(), exchTopVer @@ -1787,6 +1772,7 @@ public Collection startReceivedCaches(UUID nodeId, Affin if (CU.affinityNode(ctx.discovery().localNode(), filter)) { prepareCacheStart( + desc.cacheConfiguration(), desc, null, exchTopVer @@ -1799,17 +1785,18 @@ public Collection startReceivedCaches(UUID nodeId, Affin } /** + * @param startCfg Cache configuration to use. * @param desc Cache descriptor. * @param reqNearCfg Near configuration if specified for client cache start request. * @param exchTopVer Current exchange version. * @throws IgniteCheckedException If failed. */ - public void prepareCacheStart( + void prepareCacheStart( + CacheConfiguration startCfg, DynamicCacheDescriptor desc, @Nullable NearCacheConfiguration reqNearCfg, AffinityTopologyVersion exchTopVer ) throws IgniteCheckedException { - CacheConfiguration startCfg = desc.cacheConfiguration(); assert !caches.containsKey(startCfg.getName()) : startCfg.getName(); CacheConfiguration ccfg = new CacheConfiguration(startCfg); @@ -2003,7 +1990,7 @@ private void stopGateway(DynamicCacheChangeRequest req) { sharedCtx.removeCacheContext(ctx); - onKernalStop(cache, destroy); + onKernalStop(cache, true); stopCache(cache, true, destroy); @@ -2017,9 +2004,7 @@ private void stopGateway(DynamicCacheChangeRequest req) { * @param startTopVer Cache start version. * @param err Cache start error if any. */ - void initCacheProxies( - AffinityTopologyVersion startTopVer, @Nullable - Throwable err) { + void initCacheProxies(AffinityTopologyVersion startTopVer, @Nullable Throwable err) { for (GridCacheAdapter cache : caches.values()) { GridCacheContext cacheCtx = cache.context(); @@ -2122,7 +2107,7 @@ public void onExchangeDone( if (exchActions == null) return; - if (exchActions.systemCachesStarting() && exchActions.newClusterState() == null) + if (exchActions.systemCachesStarting() && exchActions.stateChangeRequest() == null) ctx.dataStructures().restoreStructuresState(ctx); if (err == null) { @@ -2143,9 +2128,6 @@ public void onExchangeDone( try { prepareCacheStop(action.request().cacheName(), action.request().destroy()); - - if (exchActions.newClusterState() == null) - ctx.state().onCacheStop(action.request()); } finally { sharedCtx.database().checkpointReadUnlock(); @@ -2166,6 +2148,9 @@ public void onExchangeDone( if (!sharedCtx.kernalContext().clientNode()) sharedCtx.database().onCacheGroupsStopped(stoppedGroups); + + if (exchActions.deactivate()) + sharedCtx.deactivate(); } } @@ -2204,10 +2189,11 @@ void completeTemplateAddFuture(String cacheName, IgniteUuid deploymentId) { /** * @param req Request to complete future for. + * @param success Future result. * @param err Error if any. */ void completeCacheStartFuture(DynamicCacheChangeRequest req, boolean success, @Nullable Throwable err) { - if (req.initiatingNodeId().equals(ctx.localNodeId())) { + if (ctx.localNodeId().equals(req.initiatingNodeId())) { DynamicCacheStartFuture fut = (DynamicCacheStartFuture)pendingFuts.get(req.requestId()); if (fut != null) @@ -2304,30 +2290,35 @@ private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx, /** {@inheritDoc} */ @Override public void collectGridNodeData(DiscoveryDataBag dataBag) { - if (ctx.state().active()) - cachesInfo.collectGridNodeData(dataBag); - else - ctx.state().collectGridNodeData0(dataBag); + cachesInfo.collectGridNodeData(dataBag); } /** {@inheritDoc} */ @Override public void onJoiningNodeDataReceived(JoiningNodeDiscoveryData data) { - if (ctx.state().active()) - cachesInfo.onJoiningNodeDataReceived(data); - - ctx.state().onJoiningNodeDataReceived0(data); + cachesInfo.onJoiningNodeDataReceived(data); } /** {@inheritDoc} */ @Override public void onGridDataReceived(GridDiscoveryData data) { - if (ctx.state().active()) { - if (!cachesInfo.disconnectedState()) - cachesInfo.addJoinInfo(); + cachesInfo.onGridDataReceived(data); + } - cachesInfo.onGridDataReceived(data); - } + /** + * @param msg Message. + */ + public void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) { + cachesInfo.onStateChangeFinish(msg); + } - ctx.state().onGridDataReceived0(data); + /** + * @param msg Message. + * @param topVer Current topology version. + * @throws IgniteCheckedException If configuration validation failed. + * @return Exchange actions. + */ + public ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer) + throws IgniteCheckedException { + return cachesInfo.onStateChangeRequest(msg, topVer); } /** @@ -2929,13 +2920,19 @@ else if (ctx.clientDisconnected()) { /** * @param type Event type. + * @param customMsg Custom message instance. * @param node Event node. * @param topVer Topology version. + * @param state Cluster state. */ - public void onDiscoveryEvent(int type, ClusterNode node, AffinityTopologyVersion topVer) { + public void onDiscoveryEvent(int type, + @Nullable DiscoveryCustomMessage customMsg, + ClusterNode node, + AffinityTopologyVersion topVer, + DiscoveryDataClusterState state) { cachesInfo.onDiscoveryEvent(type, node, topVer); - sharedCtx.affinity().onDiscoveryEvent(type, node, topVer); + sharedCtx.affinity().onDiscoveryEvent(type, customMsg, node, topVer, state); } /** @@ -3214,7 +3211,7 @@ private IgniteInternalCache internalCacheEx(String name) { proxy = new IgniteCacheProxy(cacheAdapter.context(), cacheAdapter, null, false); } - assert proxy != null; + assert proxy != null : name; return proxy.internalProxy(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java index 75d03d75e6558..9adca8d284942 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cache; +import java.util.ArrayList; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -45,7 +46,6 @@ import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheSnapshotManager; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal; import org.apache.ignite.internal.processors.cache.jta.CacheJtaManagerAdapter; import org.apache.ignite.internal.processors.cache.store.CacheStoreManager; @@ -54,6 +54,7 @@ import org.apache.ignite.internal.processors.cache.transactions.TransactionMetricsAdapter; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager; +import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor; import org.apache.ignite.internal.util.GridIntList; import org.apache.ignite.internal.util.future.GridCompoundFuture; @@ -156,6 +157,9 @@ public class GridCacheSharedContext { /** Concurrent DHT atomic updates counters. */ private AtomicIntegerArray dhtAtomicUpdCnt; + /** */ + private final List stateAwareMgrs; + /** * @param kernalCtx Context. * @param txMgr Transaction manager. @@ -207,6 +211,49 @@ public GridCacheSharedContext( txFinishMsgLog = kernalCtx.log(CU.TX_MSG_FINISH_LOG_CATEGORY); txLockMsgLog = kernalCtx.log(CU.TX_MSG_LOCK_LOG_CATEGORY); txRecoveryMsgLog = kernalCtx.log(CU.TX_MSG_RECOVERY_LOG_CATEGORY); + + stateAwareMgrs = new ArrayList<>(); + + if (pageStoreMgr != null) + stateAwareMgrs.add(pageStoreMgr); + + if (walMgr != null) + stateAwareMgrs.add(walMgr); + + stateAwareMgrs.add(dbMgr); + + stateAwareMgrs.add(snpMgr); + } + + /** + * @throws IgniteCheckedException If failed. + */ + public void activate() throws IgniteCheckedException { + if (!kernalCtx.clientNode()) + dbMgr.lock(); + + boolean success = false; + + try { + for (IgniteChangeGlobalStateSupport mgr : stateAwareMgrs) + mgr.onActivate(kernalCtx); + + success = true; + } + finally { + if (!success) { + if (!kernalCtx.clientNode()) + dbMgr.unLock(); + } + } + } + + /** + * + */ + public void deactivate() { + for (int i = stateAwareMgrs.size() - 1; i >= 0; i--) + stateAwareMgrs.get(i).onDeActivate(kernalCtx); } /** @@ -272,12 +319,15 @@ void onDisconnected(IgniteFuture reconnectFut) throws IgniteCheckedException if (restartOnDisconnect(mgr)) mgr.stop(true); } + + deactivate(); } /** + * @param active Active flag. * @throws IgniteCheckedException If failed. */ - void onReconnected() throws IgniteCheckedException { + void onReconnected(boolean active) throws IgniteCheckedException { List> mgrs = new LinkedList<>(); setManagers(mgrs, txMgr, @@ -303,8 +353,10 @@ void onReconnected() throws IgniteCheckedException { kernalCtx.query().onCacheReconnect(); - for (GridCacheSharedManager mgr : mgrs) - mgr.onKernalStart(true); + if (!active) + affinity().removeAllCacheInfo(); + + exchMgr.onKernalStart(active, true); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManager.java index e0e40905d333b..bc1bbb97b9e43 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManager.java @@ -39,12 +39,6 @@ public interface GridCacheSharedManager { */ public void stop(boolean cancel); - /** - * @param reconnect {@code True} if manager restarted after client reconnect. - * @throws IgniteCheckedException If failed. - */ - public void onKernalStart(boolean reconnect) throws IgniteCheckedException; - /** * @param cancel Cancel flag. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManagerAdapter.java index f6f79e487de8c..90ae6702b703d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManagerAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedManagerAdapter.java @@ -111,14 +111,6 @@ protected void stop0(boolean cancel) { // No-op. } - /** {@inheritDoc} */ - @Override public final void onKernalStart(boolean reconnect) throws IgniteCheckedException { - onKernalStart0(reconnect); - - if (!reconnect && log != null && log.isDebugEnabled()) - log.debug(kernalStartInfo()); - } - /** {@inheritDoc} */ @Override public final void onKernalStop(boolean cancel) { if (!starting.get()) @@ -131,14 +123,6 @@ protected void stop0(boolean cancel) { log.debug(kernalStopInfo()); } - /** - * @param reconnect {@code True} if manager restarted after client reconnect. - * @throws IgniteCheckedException If failed. - */ - protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException { - // No-op. - } - /** * @param cancel Cancel flag. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PendingDiscoveryEvent.java similarity index 52% rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterState.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PendingDiscoveryEvent.java index 1e1ef718555a2..b4274f7e2b423 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterState.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PendingDiscoveryEvent.java @@ -17,22 +17,45 @@ package org.apache.ignite.internal.processors.cache; +import org.apache.ignite.events.DiscoveryEvent; +import org.apache.ignite.internal.managers.discovery.DiscoCache; +import org.apache.ignite.internal.util.typedef.internal.S; + /** * */ -public enum ClusterState { +public class PendingDiscoveryEvent { + /** */ + private final DiscoveryEvent evt; + + /** */ + private final DiscoCache cache; + /** - * Cache is inactive. No operations are allowed, no partition assignments or rebalancing is performed. + * @param evt Event. + * @param cache Discovery data cache. */ - INACTIVE, + public PendingDiscoveryEvent(DiscoveryEvent evt, DiscoCache cache) { + this.evt = evt; + this.cache = cache; + } /** - * Cache is active and operations. There are no lost partitions. + * @return Event. */ - ACTIVE, + public DiscoveryEvent event() { + return evt; + } /** - * Cache is inactive. But process of it activation in progress. + * @return Discovery data cache. */ - TRANSITION + public DiscoCache discoCache() { + return cache; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(PendingDiscoveryEvent.class, this); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java new file mode 100644 index 0000000000000..2d35e810916c4 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.UUID; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; +import org.apache.ignite.internal.util.typedef.internal.S; + +/** + * + */ +public class StateChangeRequest { + /** */ + private final ChangeGlobalStateMessage msg; + + /** */ + private final AffinityTopologyVersion topVer; + + /** + * @param msg Message. + * @param topVer State change topology versoin. + */ + public StateChangeRequest(ChangeGlobalStateMessage msg, + AffinityTopologyVersion topVer) { + this.msg = msg; + this.topVer = topVer; + } + + /** + * @return State change exchange version. + */ + public AffinityTopologyVersion topologyVersion() { + return topVer; + } + + /** + * @return State change request ID. + */ + public UUID requestId() { + return msg.requestId(); + } + + /** + * @return New state. + */ + public boolean activate() { + return msg.activate(); + } + + /** + * @return Node initiated state change process. + */ + public UUID initiatorNodeId() { + return msg.initiatorNodeId(); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(StateChangeRequest.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java index c2c71ea505cec..0065e4156b51a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java @@ -258,8 +258,8 @@ public void addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener lsnr) } /** {@inheritDoc} */ - @Override public void onKernalStart() throws IgniteCheckedException { - super.onKernalStart(); + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { + super.onKernalStart(active); discoveryStarted = true; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java index 1c97de2125d16..d6b45b342fc18 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java @@ -29,7 +29,6 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture; -import org.apache.ignite.internal.processors.cache.GridCacheFuture; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.util.GridLeanMap; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java index 38d0108b359b8..960b91a39b72c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java @@ -50,7 +50,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo; import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException; import org.apache.ignite.internal.processors.cache.GridCacheMapEntry; -import org.apache.ignite.internal.processors.cache.GridCacheMapEntryFactory; import org.apache.ignite.internal.processors.cache.GridCachePreloader; import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy; import org.apache.ignite.internal.processors.cache.KeyCacheObject; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java index 123d26b57677b..0ea48e38c4fd4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java @@ -27,7 +27,6 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; -import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheObject; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java index 6392d0a44b5ca..439bb9d7fb702 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java @@ -19,13 +19,11 @@ import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.Map; import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; -import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.NodeStoppingException; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index c205c3b515333..57ce3231d73c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -41,7 +41,6 @@ import org.apache.ignite.internal.processors.affinity.AffinityAssignment; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; -import org.apache.ignite.internal.processors.cache.ClusterState; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap; @@ -424,11 +423,8 @@ else if (localNode(p, aff)) throws IgniteCheckedException { DiscoveryEvent discoEvt = exchFut.discoveryEvent(); - ClusterState newState = exchFut.newClusterState(); - - treatAllPartAsLoc = (newState != null && newState == ClusterState.ACTIVE) - || (ctx.kernalContext().state().active() - && discoEvt.type() == EventType.EVT_NODE_JOINED + treatAllPartAsLoc = exchFut.activateCluster() + || (discoEvt.type() == EventType.EVT_NODE_JOINED && discoEvt.eventNode().isLocal() && !ctx.kernalContext().clientNode() ); @@ -611,7 +607,7 @@ else if (log.isDebugEnabled()) if (locPart != null) { GridDhtPartitionState state = locPart.state(); - if (state == MOVING && ctx.kernalContext().state().active()) { + if (state == MOVING) { locPart.rent(false); updateSeq = updateLocal(p, locPart.state(), updateSeq); @@ -1773,9 +1769,6 @@ private boolean checkEvictions(long updateSeq) { * @return Checks if any of the local partitions need to be evicted. */ private boolean checkEvictions(long updateSeq, List> aff) { - if (!ctx.kernalContext().state().active()) - return false; - boolean changed = false; UUID locId = ctx.localNodeId(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java index e70f38335777f..d04870ace9c48 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java @@ -80,7 +80,7 @@ protected final CacheValidation validateCacheGroup(CacheGroupContext grp, Collec if (err != null) return err; - if (!cctx.shared().kernalContext().state().active()) + if (!cctx.shared().kernalContext().state().publicApiActiveState()) return new CacheInvalidStateException( "Failed to perform cache operation (cluster is not activated): " + cctx.name()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java index cfecb1ccfcc6c..d66afca93acf2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java @@ -18,16 +18,13 @@ package org.apache.ignite.internal.processors.cache.distributed.dht; import java.util.Collection; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.IgniteDiagnosticAware; -import org.apache.ignite.internal.IgniteDiagnosticMessage; import org.apache.ignite.internal.IgniteDiagnosticPrepareContext; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java index 1bd8ec568120a..6fe96a437a56e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateFuture.java @@ -46,7 +46,6 @@ import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.tostring.GridToStringInclude; -import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java index 763b43b6ab7f6..fe216a00379ac 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java index 774f0cef651ba..e7e95b2537e58 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java @@ -161,6 +161,8 @@ void stop() { lastExchangeFut = null; lastTimeoutObj.set(null); + + syncFut.onDone(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index a1926ee4fd2a7..cea758a2cd68b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -36,6 +36,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.events.CacheEvent; import org.apache.ignite.events.DiscoveryEvent; import org.apache.ignite.events.Event; @@ -57,15 +58,14 @@ import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache; import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage; import org.apache.ignite.internal.processors.cache.CacheGroupContext; -import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; import org.apache.ignite.internal.processors.cache.CachePartitionExchangeWorkerTask; -import org.apache.ignite.internal.processors.cache.ClusterState; import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.ExchangeActions; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.StateChangeRequest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter; import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; @@ -73,7 +73,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; -import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; +import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage; import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter; import org.apache.ignite.internal.util.future.GridCompoundFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; @@ -214,9 +215,6 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte /** Change global state exceptions. */ private final Map changeGlobalStateExceptions = new ConcurrentHashMap8<>(); - /** This exchange for change global state. */ - private boolean exchangeOnChangeGlobalState; - /** */ private ConcurrentMap msgs = new ConcurrentHashMap8<>(); @@ -463,10 +461,24 @@ public void onEvent(GridDhtPartitionExchangeId exchId, DiscoveryEvent discoEvt, } /** - * + * @return {@code True} if cluster state change exchange. + */ + private boolean stateChangeExchange() { + return exchActions != null && exchActions.stateChangeRequest() != null; + } + + /** + * @return {@code True} if activate cluster exchange. */ - public ClusterState newClusterState() { - return exchActions != null ? exchActions.newClusterState() : null; + public boolean activateCluster() { + return exchActions != null && exchActions.activate(); + } + + /** + * @return {@code True} if deactivate cluster exchange. + */ + boolean deactivateCluster() { + return exchActions != null && exchActions.deactivate(); } /** @@ -519,6 +531,8 @@ public void init() throws IgniteInterruptedCheckedException { if (isDone()) return; + assert !cctx.kernalContext().isDaemon(); + initTs = U.currentTimeMillis(); U.await(evtLatch); @@ -557,7 +571,12 @@ public void init() throws IgniteInterruptedCheckedException { if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) { DiscoveryCustomMessage msg = ((DiscoveryCustomEvent)discoEvt).customMessage(); - if (msg instanceof DynamicCacheChangeBatch) { + if (msg instanceof ChangeGlobalStateMessage) { + assert exchActions != null && !exchActions.empty(); + + exchange = onClusterStateChangeRequest(crdNode); + } + else if (msg instanceof DynamicCacheChangeBatch) { assert exchActions != null && !exchActions.empty(); exchange = onCacheChangeRequest(crdNode); @@ -582,8 +601,26 @@ else if (msg instanceof StartSnapshotOperationAckDiscoveryMessage) { cctx.affinity().initStartedCaches(crdNode, this, receivedCaches); } - else - cctx.cache().startCachesOnLocalJoin(topVer); + else { + cctx.activate(); + + List> caches = + cctx.cache().cachesToStartOnLocalJoin(); + + if (cctx.database().persistenceEnabled() && + !cctx.kernalContext().clientNode()) { + List startDescs = new ArrayList<>(); + + if (caches != null) { + for (T2 c : caches) + startDescs.add(c.get1()); + } + + cctx.database().readCheckpointAndRestoreMemory(startDescs); + } + + cctx.cache().startCachesOnLocalJoin(caches, topVer); + } } exchange = CU.clientNode(discoEvt.eventNode()) ? @@ -710,21 +747,94 @@ private void updateTopologies(boolean crd) throws IgniteCheckedException { * @return Exchange type. * @throws IgniteCheckedException If failed. */ - private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedException { + private ExchangeType onClusterStateChangeRequest(boolean crd) throws IgniteCheckedException { assert exchActions != null && !exchActions.empty() : this; - GridClusterStateProcessor stateProc = cctx.kernalContext().state(); + StateChangeRequest req = exchActions.stateChangeRequest(); + + assert req != null : exchActions; + + if (req.activate()) { + if (log.isInfoEnabled()) { + log.info("Start activation process [nodeId=" + cctx.localNodeId() + + ", client=" + cctx.kernalContext().clientNode() + + ", topVer=" + topologyVersion() + "]"); + } + + try { + cctx.activate(); + + if (cctx.database().persistenceEnabled() && !cctx.kernalContext().clientNode()) { + List startDescs = new ArrayList<>(); + + for (ExchangeActions.ActionData startReq : exchActions.cacheStartRequests()) + startDescs.add(startReq.descriptor()); + + cctx.database().readCheckpointAndRestoreMemory(startDescs); + } + + cctx.affinity().onCacheChangeRequest(this, crd, exchActions); - if (exchangeOnChangeGlobalState = stateProc.changeGlobalState(exchActions, topologyVersion())) { - changeGlobalStateE = stateProc.onChangeGlobalState(); + if (log.isInfoEnabled()) { + log.info("Successfully activated caches [nodeId=" + cctx.localNodeId() + + ", client=" + cctx.kernalContext().clientNode() + + ", topVer=" + topologyVersion() + "]"); + } + } + catch (Exception e) { + U.error(log, "Failed to activate node components [nodeId=" + cctx.localNodeId() + + ", client=" + cctx.kernalContext().clientNode() + + ", topVer=" + topologyVersion() + "]", e); - if (changeGlobalStateE != null) { - if (crd) - changeGlobalStateExceptions.put(cctx.localNodeId(), changeGlobalStateE); + changeGlobalStateE = e; - return cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL; + if (crd) { + synchronized (this) { + changeGlobalStateExceptions.put(cctx.localNodeId(), e); + } + } } } + else { + if (log.isInfoEnabled()) { + log.info("Start deactivation process [nodeId=" + cctx.localNodeId() + + ", client=" + cctx.kernalContext().clientNode() + + ", topVer=" + topologyVersion() + "]"); + } + + try { + cctx.kernalContext().dataStructures().onDeActivate(cctx.kernalContext()); + + cctx.kernalContext().service().onDeActivate(cctx.kernalContext()); + + cctx.affinity().onCacheChangeRequest(this, crd, exchActions); + + if (log.isInfoEnabled()) { + log.info("Successfully deactivated data structures, services and caches [" + + "nodeId=" + cctx.localNodeId() + + ", client=" + cctx.kernalContext().clientNode() + + ", topVer=" + topologyVersion() + "]"); + } + } + catch (Exception e) { + U.error(log, "Failed to deactivate node components [nodeId=" + cctx.localNodeId() + + ", client=" + cctx.kernalContext().clientNode() + + ", topVer=" + topologyVersion() + "]", e); + + changeGlobalStateE = e; + } + } + + return cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL; + } + + /** + * @param crd Coordinator flag. + * @return Exchange type. + * @throws IgniteCheckedException If failed. + */ + private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedException { + assert exchActions != null && !exchActions.empty() : this; assert !exchActions.clientOnlyExchange() : exchActions; @@ -1133,8 +1243,8 @@ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException if (partHistReserved0 != null) m.partitionHistoryCounters(partHistReserved0); - if (exchangeOnChangeGlobalState && changeGlobalStateE != null) - m.setException(changeGlobalStateE); + if (stateChangeExchange() && changeGlobalStateE != null) + m.setError(changeGlobalStateE); if (log.isDebugEnabled()) log.debug("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + m + ']'); @@ -1160,8 +1270,8 @@ private GridDhtPartitionsFullMessage createPartitionsMessage() { partHistSuppliers, partsToReload); - if (exchangeOnChangeGlobalState && !F.isEmpty(changeGlobalStateExceptions)) - m.setExceptionsMap(changeGlobalStateExceptions); + if (stateChangeExchange() && !F.isEmpty(changeGlobalStateExceptions)) + m.setErrorsMap(changeGlobalStateExceptions); return m; } @@ -1175,9 +1285,10 @@ private void sendAllPartitions(Collection nodes) throws IgniteCheck assert !nodes.contains(cctx.localNode()); - if (log.isDebugEnabled()) + if (log.isDebugEnabled()) { log.debug("Sending full partition map [nodeIds=" + F.viewReadOnly(nodes, F.node2id()) + ", exchId=" + exchId + ", msg=" + m + ']'); + } for (ClusterNode node : nodes) { try { @@ -1291,8 +1402,8 @@ public boolean serverNodeDiscoveryEvent() { if (exchActions != null && err == null) exchActions.completeRequestFutures(cctx); - if (exchangeOnChangeGlobalState && err == null) - cctx.kernalContext().state().onExchangeDone(); + if (stateChangeExchange() && err == null) + cctx.kernalContext().state().onStateChangeExchangeDone(exchActions.stateChangeRequest()); Map, Long> localReserved = partHistSuppliers.getReservations(cctx.localNodeId()); @@ -1368,6 +1479,7 @@ public void cleanUp() { crd = null; partReleaseFut = null; changeGlobalStateE = null; + exchActions = null; } /** @@ -1444,8 +1556,8 @@ private void processMessage(ClusterNode node, GridDhtPartitionsSingleMessage msg pendingSingleUpdates++; - if (exchangeOnChangeGlobalState && msg.getException() != null) - changeGlobalStateExceptions.put(node.id(), msg.getException()); + if (stateChangeExchange() && msg.getError() != null) + changeGlobalStateExceptions.put(node.id(), msg.getError()); allReceived = remaining.isEmpty(); } @@ -1457,7 +1569,7 @@ private void processMessage(ClusterNode node, GridDhtPartitionsSingleMessage msg if (updateSingleMap) { try { // Do not update partition map, in case cluster transitioning to inactive state. - if (!exchangeOnChangeGlobalState || exchActions.newClusterState() != ClusterState.INACTIVE) + if (!deactivateCluster()) updatePartitionSingleMap(node, msg); } finally { @@ -1735,18 +1847,16 @@ private void onAllReceived() { } } - if (discoEvt.type() == EVT_NODE_JOINED) { - if (cctx.kernalContext().state().active()) - assignPartitionsStates(); - } + if (discoEvt.type() == EVT_NODE_JOINED) + assignPartitionsStates(); else if (discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) { assert discoEvt instanceof DiscoveryCustomEvent; + if (activateCluster()) + assignPartitionsStates(); + if (((DiscoveryCustomEvent)discoEvt).customMessage() instanceof DynamicCacheChangeBatch) { if (exchActions != null) { - if (exchActions.newClusterState() == ClusterState.ACTIVE) - assignPartitionsStates(); - Set caches = exchActions.cachesToResetLostPartitions(); if (!F.isEmpty(caches)) @@ -1783,13 +1893,34 @@ else if (discoEvt.type() == EVT_NODE_LEFT || discoEvt.type() == EVT_NODE_FAILED) nodes = new ArrayList<>(srvNodes); } + IgniteCheckedException err = null; + + if (stateChangeExchange()) { + StateChangeRequest req = exchActions.stateChangeRequest(); + + assert req != null : exchActions; + + boolean stateChangeErr = false; + + if (!F.isEmpty(changeGlobalStateExceptions)) { + stateChangeErr = true; + + err = new IgniteCheckedException("Cluster state change failed."); + + cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req); + } + + boolean active = !stateChangeErr && req.activate(); + + ChangeGlobalStateFinishMessage msg = new ChangeGlobalStateFinishMessage(req.requestId(), active); + + cctx.discovery().sendCustomEvent(msg); + } + if (!nodes.isEmpty()) sendAllPartitions(nodes); - if (exchangeOnChangeGlobalState && !F.isEmpty(changeGlobalStateExceptions)) - cctx.kernalContext().state().onFullResponseMessage(changeGlobalStateExceptions); - - onDone(exchangeId().topologyVersion()); + onDone(exchangeId().topologyVersion(), err); } } catch (IgniteCheckedException e) { @@ -1898,7 +2029,7 @@ public void onReceive(final ClusterNode node, final GridDhtPartitionsFullMessage * @param msg Message. */ private void processMessage(ClusterNode node, GridDhtPartitionsFullMessage msg) { - assert msg.exchangeId().equals(exchId) : msg; + assert exchId.equals(msg.exchangeId()) : msg; assert msg.lastVersion() != null : msg; synchronized (this) { @@ -1919,10 +2050,15 @@ private void processMessage(ClusterNode node, GridDhtPartitionsFullMessage msg) updatePartitionFullMap(msg); - if (exchangeOnChangeGlobalState && !F.isEmpty(msg.getExceptionsMap())) - cctx.kernalContext().state().onFullResponseMessage(msg.getExceptionsMap()); + IgniteCheckedException err = null; - onDone(exchId.topologyVersion()); + if (stateChangeExchange() && !F.isEmpty(msg.getErrorsMap())) { + err = new IgniteCheckedException("Cluster state change failed"); + + cctx.kernalContext().state().onStateChangeError(msg.getErrorsMap(), exchActions.stateChangeRequest()); + } + + onDone(exchId.topologyVersion(), err); } /** @@ -2143,7 +2279,7 @@ public void onNodeLeft(final ClusterNode node) { } if (crd0.isLocal()) { - if (exchangeOnChangeGlobalState && changeGlobalStateE != null) + if (stateChangeExchange() && changeGlobalStateE != null) changeGlobalStateExceptions.put(crd0.id(), changeGlobalStateE); if (allReceived) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java index 0beb93503be7a..75609b8ec49d0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java @@ -90,10 +90,10 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa /** Exceptions. */ @GridToStringInclude @GridDirectTransient - private Map exs; + private Map errs; /** */ - private byte[] exsBytes; + private byte[] errsBytes; /** */ @GridDirectTransient @@ -224,17 +224,17 @@ public Set partsToReload(UUID nodeId, int grpId) { } /** - * + * @return Errors map. */ - public Map getExceptionsMap() { - return exs; + @Nullable Map getErrorsMap() { + return errs; } /** - * @param exs Exs. + * @param errs Errors map. */ - public void setExceptionsMap(Map exs) { - this.exs = new HashMap<>(exs); + void setErrorsMap(Map errs) { + this.errs = new HashMap<>(errs); } /** {@inheritDoc} */ @@ -245,14 +245,14 @@ public void setExceptionsMap(Map exs) { (partCntrs != null && partCntrsBytes == null) || (partHistSuppliers != null && partHistSuppliersBytes == null) || (partsToReload != null && partsToReloadBytes == null) || - (exs != null && exsBytes == null); + (errs != null && errsBytes == null); if (marshal) { byte[] partsBytes0 = null; byte[] partCntrsBytes0 = null; byte[] partHistSuppliersBytes0 = null; byte[] partsToReloadBytes0 = null; - byte[] exsBytes0 = null; + byte[] errsBytes0 = null; if (parts != null && partsBytes == null) partsBytes0 = U.marshal(ctx, parts); @@ -266,8 +266,8 @@ public void setExceptionsMap(Map exs) { if (partsToReload != null && partsToReloadBytes == null) partsToReloadBytes0 = U.marshal(ctx, partsToReload); - if (exs != null && exsBytes == null) - exsBytes0 = U.marshal(ctx, exs); + if (errs != null && errsBytes == null) + errsBytes0 = U.marshal(ctx, errs); if (compress) { assert !compressed(); @@ -277,13 +277,13 @@ public void setExceptionsMap(Map exs) { byte[] partCntrsBytesZip = U.zip(partCntrsBytes0); byte[] partHistSuppliersBytesZip = U.zip(partHistSuppliersBytes0); byte[] partsToReloadBytesZip = U.zip(partsToReloadBytes0); - byte[] exsBytesZip = U.zip(exsBytes0); + byte[] exsBytesZip = U.zip(errsBytes0); partsBytes0 = partsBytesZip; partCntrsBytes0 = partCntrsBytesZip; partHistSuppliersBytes0 = partHistSuppliersBytesZip; partsToReloadBytes0 = partsToReloadBytesZip; - exsBytes0 = exsBytesZip; + errsBytes0 = exsBytesZip; compressed(true); } @@ -296,7 +296,7 @@ public void setExceptionsMap(Map exs) { partCntrsBytes = partCntrsBytes0; partHistSuppliersBytes = partHistSuppliersBytes0; partsToReloadBytes = partsToReloadBytes0; - exsBytes = exsBytes0; + errsBytes = errsBytes0; } } @@ -379,15 +379,15 @@ public void topologyVersion(AffinityTopologyVersion topVer) { if (partCntrs == null) partCntrs = new IgniteDhtPartitionCountersMap(); - if (exsBytes != null && exs == null) { + if (errsBytes != null && errs == null) { if (compressed()) - exs = U.unmarshalZip(ctx.marshaller(), exsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); + errs = U.unmarshalZip(ctx.marshaller(), errsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); else - exs = U.unmarshal(ctx, exsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); + errs = U.unmarshal(ctx, errsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); } - if (exs == null) - exs = new HashMap<>(); + if (errs == null) + errs = new HashMap<>(); } /** {@inheritDoc} */ @@ -412,7 +412,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { writer.incrementState(); case 6: - if (!writer.writeByteArray("exsBytes", exsBytes)) + if (!writer.writeByteArray("errsBytes", errsBytes)) return false; writer.incrementState(); @@ -472,7 +472,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); case 6: - exsBytes = reader.readByteArray("exsBytes"); + errsBytes = reader.readByteArray("errsBytes"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java index 1e5ea142d375d..b4d25c41f4780 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java @@ -76,10 +76,10 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes /** Exception. */ @GridToStringInclude @GridDirectTransient - private Exception ex; + private Exception err; /** */ - private byte[] exBytes; + private byte[] errBytes; /** */ private boolean client; @@ -220,15 +220,15 @@ public Map partitions() { /** * @param ex Exception. */ - public void setException(Exception ex) { - this.ex = ex; + public void setError(Exception ex) { + this.err = ex; } /** - * + * @return Not null exception if exchange processing failed. */ - public Exception getException() { - return ex; + @Nullable public Exception getError() { + return err; } /** {@inheritDoc} @@ -239,13 +239,13 @@ public Exception getException() { boolean marshal = (parts != null && partsBytes == null) || (partCntrs != null && partCntrsBytes == null) || (partHistCntrs != null && partHistCntrsBytes == null) || - (ex != null && exBytes == null); + (err != null && errBytes == null); if (marshal) { byte[] partsBytes0 = null; byte[] partCntrsBytes0 = null; byte[] partHistCntrsBytes0 = null; - byte[] exBytes0 = null; + byte[] errBytes0 = null; if (parts != null && partsBytes == null) partsBytes0 = U.marshal(ctx, parts); @@ -256,8 +256,8 @@ public Exception getException() { if (partHistCntrs != null && partHistCntrsBytes == null) partHistCntrsBytes0 = U.marshal(ctx, partHistCntrs); - if (ex != null && exBytes == null) - exBytes0 = U.marshal(ctx, ex); + if (err != null && errBytes == null) + errBytes0 = U.marshal(ctx, err); if (compress) { assert !compressed(); @@ -266,12 +266,12 @@ public Exception getException() { byte[] partsBytesZip = U.zip(partsBytes0); byte[] partCntrsBytesZip = U.zip(partCntrsBytes0); byte[] partHistCntrsBytesZip = U.zip(partHistCntrsBytes0); - byte[] exBytesZip = U.zip(exBytes0); + byte[] exBytesZip = U.zip(errBytes0); partsBytes0 = partsBytesZip; partCntrsBytes0 = partCntrsBytesZip; partHistCntrsBytes0 = partHistCntrsBytesZip; - exBytes0 = exBytesZip; + errBytes0 = exBytesZip; compressed(true); } @@ -283,7 +283,7 @@ public Exception getException() { partsBytes = partsBytes0; partCntrsBytes = partCntrsBytes0; partHistCntrsBytes = partHistCntrsBytes0; - exBytes = exBytes0; + errBytes = errBytes0; } } @@ -312,11 +312,11 @@ public Exception getException() { partHistCntrs = U.unmarshal(ctx, partHistCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); } - if (exBytes != null && ex == null) { + if (errBytes != null && err == null) { if (compressed()) - ex = U.unmarshalZip(ctx.marshaller(), exBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); + err = U.unmarshalZip(ctx.marshaller(), errBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); else - ex = U.unmarshal(ctx, exBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); + err = U.unmarshal(ctx, errBytes, U.resolveClassLoader(ldr, ctx.gridConfig())); } if (dupPartsData != null) { @@ -368,7 +368,7 @@ public Exception getException() { writer.incrementState(); case 7: - if (!writer.writeByteArray("exBytes", exBytes)) + if (!writer.writeByteArray("errBytes", errBytes)) return false; writer.incrementState(); @@ -424,7 +424,7 @@ public Exception getException() { reader.incrementState(); case 7: - exBytes = reader.readByteArray("exBytes"); + errBytes = reader.readByteArray("errBytes"); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java index 66b59876ab444..2b18c2461af4c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java @@ -183,7 +183,7 @@ private IgniteCheckedException stopError() { // No assignments for disabled preloader. GridDhtPartitionTopology top = grp.topology(); - if (!grp.rebalanceEnabled() || !grp.shared().kernalContext().state().active()) + if (!grp.rebalanceEnabled()) return new GridDhtPreloaderAssignments(exchFut, top.topologyVersion()); int partCnt = grp.affinity().partitions(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java index 062ff6cb5926c..a49812e6d9aa5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java @@ -45,8 +45,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.GridLeanMap; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java index 41e6d702c1a68..29c7aad873bb1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java @@ -26,7 +26,6 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxPrepareRequest; -import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.F; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index b3ab1cd4b0b9e..c700ef4ad7e46 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -65,7 +65,6 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.PersistenceMetrics; -import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.MemoryConfiguration; @@ -100,11 +99,13 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; -import org.apache.ignite.internal.processors.cache.ClusterState; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.StoredCacheData; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; @@ -113,9 +114,6 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.port.GridPortRecord; import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.GridMultiCollectionWrapper; @@ -352,9 +350,9 @@ public IgniteInternalFuture enableCheckpoints(boolean enable) { /** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { - snapshotMgr = cctx.snapshot(); + super.start0(); - assert !cctx.kernalContext().state().active() : "Cluster with persistent must starting as inactive."; + snapshotMgr = cctx.snapshot(); if (!cctx.kernalContext().clientNode()) { IgnitePageStoreManager store = cctx.pageStore(); @@ -371,15 +369,13 @@ public IgniteInternalFuture enableCheckpoints(boolean enable) { fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), cctx.kernalContext(), log); persStoreMetrics.wal(cctx.wal()); - - registrateMetricsMBean(); } } /** - * + * @throws IgniteCheckedException If failed. */ - @Override public void initDataBase() throws IgniteCheckedException { + private void initDataBase() throws IgniteCheckedException { Long cpBufSize = persistenceCfg.getCheckpointingPageBufferSize(); if (persistenceCfg.getCheckpointingThreads() > 1) @@ -432,8 +428,6 @@ public IgniteInternalFuture enableCheckpoints(boolean enable) { } checkpointPageBufSize = cpBufSize; - - super.start0(); } /** {@inheritDoc} */ @@ -442,58 +436,7 @@ public IgniteInternalFuture enableCheckpoints(boolean enable) { } /** {@inheritDoc} */ - @Override protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException { - if (reconnect || cctx.kernalContext().clientNode() || !cctx.kernalContext().state().active()) - return; - - initDataBase(); - - initCachesAndRestoreMemory(); - } - - /** - * - */ - private void initCachesAndRestoreMemory() throws IgniteCheckedException { - Collection cacheNames = new HashSet<>(); - - // TODO IGNITE-5075 group descriptors. - for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration()) { - if (CU.isSystemCache(ccfg.getName())) { - storeMgr.initializeForCache( - cctx.cache().cacheDescriptors().get(ccfg.getName()).groupDescriptor(), - new StoredCacheData(ccfg) - ); - - cacheNames.add(ccfg.getName()); - } - } - - for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration()) - if (!CU.isSystemCache(ccfg.getName())) { - DynamicCacheDescriptor cacheDesc = cctx.cache().cacheDescriptors().get(ccfg.getName()); - - if (cacheDesc != null) - storeMgr.initializeForCache( - cacheDesc.groupDescriptor(), - new StoredCacheData(ccfg) - ); - - cacheNames.add(ccfg.getName()); - } - - for (StoredCacheData cacheData : cctx.pageStore().readCacheConfigurations().values()) { - if (!cacheNames.contains(cacheData.config().getName())) - storeMgr.initializeForCache( - cctx.cache().cacheDescriptors().get( - cacheData.config().getName()).groupDescriptor(), cacheData); - } - - readCheckpointAndRestoreMemory(); - } - - /** {@inheritDoc} */ - @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onActivate(GridKernalContext ctx) throws IgniteCheckedException { if (log.isDebugEnabled()) log.debug("Activate database manager [id=" + cctx.localNodeId() + " topVer=" + cctx.discovery().topologyVersionEx() + " ]"); @@ -504,16 +447,13 @@ private void initCachesAndRestoreMemory() throws IgniteCheckedException { initDataBase(); registrateMetricsMBean(); - - initCachesAndRestoreMemory(); } - if (log.isDebugEnabled()) - log.debug("Restore state after activation [nodeId=" + cctx.localNodeId() + " ]"); + super.onActivate(ctx); } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext kctx) { if (log.isDebugEnabled()) log.debug("DeActivate database manager [id=" + cctx.localNodeId() + " topVer=" + cctx.discovery().topologyVersionEx() + " ]"); @@ -530,7 +470,7 @@ private void initCachesAndRestoreMemory() throws IgniteCheckedException { } /** - * + * @throws IgniteCheckedException If failed. */ private void registrateMetricsMBean() throws IgniteCheckedException { try { @@ -564,13 +504,18 @@ private void unRegistrateMetricsMBean() { } } - /** - * - */ - private void readCheckpointAndRestoreMemory() throws IgniteCheckedException { + /** {@inheritDoc} */ + @Override public void readCheckpointAndRestoreMemory(List cachesToStart) throws IgniteCheckedException { checkpointReadLock(); try { + if (!F.isEmpty(cachesToStart)) { + for (DynamicCacheDescriptor desc : cachesToStart) { + if (CU.affinityNode(cctx.localNode(), desc.cacheConfiguration().getNodeFilter())) + storeMgr.initializeForCache(desc.groupDescriptor(), new StoredCacheData(desc.cacheConfiguration())); + } + } + CheckpointStatus status = readCheckpointStatus(); // First, bring memory to the last consistent checkpoint state if needed. @@ -774,13 +719,10 @@ private void shutdownCheckpointer(boolean cancel) { boolean isSrvNode = !cctx.kernalContext().clientNode(); - boolean clusterStatusActive = cctx.kernalContext().state().active(); - - boolean clusterInTransitionStateToActive = fut.newClusterState() == ClusterState.ACTIVE; + boolean clusterInTransitionStateToActive = fut.activateCluster(); // Before local node join event. - if (clusterInTransitionStateToActive || - (joinEvt && locNode && isSrvNode && clusterStatusActive)) + if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode)) restoreState(); if (cctx.kernalContext().query().moduleEnabled()) { @@ -1579,9 +1521,10 @@ private void restorePartitionState( Map, T2> partStates ) throws IgniteCheckedException { for (CacheGroupContext grp : cctx.cache().cacheGroups()) { - if (grp.isLocal()) + if (grp.isLocal() || !grp.affinityNode()) { // Local cache has no partitions and its states. continue; + } int grpId = grp.groupId(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 4e322b96a3dce..7a38b61a52144 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -135,10 +135,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple /** {@inheritDoc} */ @Override protected CacheDataStore createCacheDataStore0(final int p) throws IgniteCheckedException { - GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ctx.database(); - - boolean exists = ctx.pageStore() != null - && ctx.pageStore().exists(grp.groupId(), p); + boolean exists = ctx.pageStore() != null && ctx.pageStore().exists(grp.groupId(), p); return new GridCacheDataStore(p, exists); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index f04c27898f4e0..c5f174cea626c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import javax.management.JMException; @@ -41,8 +42,10 @@ import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl; import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheMapEntry; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.persistence.evict.FairFifoPageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker; @@ -51,7 +54,6 @@ import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl; import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.LT; @@ -100,13 +102,6 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getMemoryConfiguration() == null) return; - init(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - public void init() throws IgniteCheckedException { MemoryConfiguration memCfg = cctx.kernalContext().config().getMemoryConfiguration(); assert memCfg != null; @@ -114,14 +109,6 @@ public void init() throws IgniteCheckedException { validateConfiguration(memCfg); pageSize = memCfg.getPageSize(); - - initPageMemoryPolicies(memCfg); - - registerMetricsMBeans(); - - startMemoryPolicies(); - - initPageMemoryDataStructures(memCfg); } /** @@ -149,12 +136,12 @@ private void registerMetricsMBean( ) { try { U.registerMBean( - cfg.getMBeanServer(), - cfg.getIgniteInstanceName(), - "MemoryMetrics", - memPlcCfg.getName(), - new MemoryMetricsMXBeanImpl(memMetrics, memPlcCfg), - MemoryMetricsMXBean.class); + cfg.getMBeanServer(), + cfg.getIgniteInstanceName(), + "MemoryMetrics", + memPlcCfg.getName(), + new MemoryMetricsMXBeanImpl(memMetrics, memPlcCfg), + MemoryMetricsMXBean.class); } catch (JMException e) { U.error(log, "Failed to register MBean for MemoryMetrics with name: '" + memMetrics.getName() + "'", e); @@ -163,6 +150,7 @@ private void registerMetricsMBean( /** * @param dbCfg Database config. + * @throws IgniteCheckedException If failed. */ protected void initPageMemoryDataStructures(MemoryConfiguration dbCfg) throws IgniteCheckedException { freeListMap = U.newHashMap(memPlcMap.size()); @@ -553,13 +541,6 @@ public void dumpStatistics(IgniteLogger log) { } } - /** - * @throws IgniteCheckedException If failed. - */ - public void initDataBase() throws IgniteCheckedException { - // No-op. - } - /** * @return collection of all configured {@link MemoryPolicy policies}. */ @@ -591,6 +572,14 @@ public PersistenceMetrics persistentStoreMetrics() { return null; } + /** + * @param cachesToStart Started caches. + * @throws IgniteCheckedException If failed. + */ + public void readCheckpointAndRestoreMemory(List cachesToStart) throws IgniteCheckedException { + // No-op. + } + /** * @param memPlcName Name of {@link MemoryPolicy} to obtain {@link MemoryMetrics} for. * @return {@link MemoryMetrics} snapshot for specified {@link MemoryPolicy} or {@code null} if @@ -947,11 +936,24 @@ protected File buildPath(String path, String consId) { /** {@inheritDoc} */ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException { - start0(); + if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getMemoryConfiguration() == null) + return; + + MemoryConfiguration memCfg = cctx.kernalContext().config().getMemoryConfiguration(); + + assert memCfg != null; + + initPageMemoryPolicies(memCfg); + + registerMetricsMBeans(); + + startMemoryPolicies(); + + initPageMemoryDataStructures(memCfg); } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext kctx) { stop0(false); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java index ad804cb3b3cdb..cce6f55d4a494 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java @@ -124,7 +124,7 @@ public void onChangeTrackerPage( FullPageId fullId, PageMemory pageMem ) throws IgniteCheckedException { - + // No-op. } /** @@ -135,14 +135,16 @@ public void flushDirtyPageHandler( ByteBuffer pageBuf, Integer tag ) throws IgniteCheckedException { - + // No-op. } + /** {@inheritDoc} */ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException { - + // No-op. } - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { - + /** {@inheritDoc} */ + @Override public void onDeActivate(GridKernalContext kctx) { + // No-op. } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index f908512488f1d..28bf6e4e6dad8 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -162,7 +162,7 @@ public FilePageStoreManager(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext kctx) { if (log.isDebugEnabled()) log.debug("DeActivate page store manager [id=" + cctx.localNodeId() + " topVer=" + cctx.discovery().topologyVersionEx() + " ]"); @@ -208,18 +208,17 @@ public FilePageStoreManager(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void storeCacheData( - CacheGroupDescriptor grpDesc, StoredCacheData cacheData ) throws IgniteCheckedException { - File cacheWorkDir = cacheWorkDirectory(grpDesc, cacheData.config()); + File cacheWorkDir = cacheWorkDirectory(cacheData.config()); File file; checkAndInitCacheWorkDir(cacheWorkDir); assert cacheWorkDir.exists() : "Work directory does not exist: " + cacheWorkDir; - if (grpDesc.sharedGroup()) + if (cacheData.config().getGroupName() != null) file = new File(cacheWorkDir, cacheData.config().getName() + CACHE_DATA_FILENAME); else file = new File(cacheWorkDir, CACHE_DATA_FILENAME); @@ -333,14 +332,13 @@ public PageStore writeInternal(int cacheId, long pageId, ByteBuffer pageBuf, int } /** - * @param grpDesc Cache group descriptor. * @param ccfg Cache configuration. * @return Cache work directory. */ - private File cacheWorkDirectory(CacheGroupDescriptor grpDesc, CacheConfiguration ccfg) { + private File cacheWorkDirectory(CacheConfiguration ccfg) { String dirName; - if (grpDesc.sharedGroup()) + if (ccfg.getGroupName() != null) dirName = CACHE_GRP_DIR_PREFIX + ccfg.getGroupName(); else dirName = CACHE_DIR_PREFIX + ccfg.getName(); @@ -357,7 +355,7 @@ private File cacheWorkDirectory(CacheGroupDescriptor grpDesc, CacheConfiguration private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfiguration ccfg) throws IgniteCheckedException { assert !grpDesc.sharedGroup() || ccfg.getGroupName() != null : ccfg.getName(); - File cacheWorkDir = cacheWorkDirectory(grpDesc, ccfg); + File cacheWorkDir = cacheWorkDirectory(ccfg); boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index f877a14c5d087..8993112b23735 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -288,14 +288,6 @@ private void checkWalConfiguration() throws IgniteCheckedException { } } - /** {@inheritDoc} */ - @Override protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException { - super.onKernalStart0(reconnect); - - if (!cctx.kernalContext().clientNode() && cctx.kernalContext().state().active()) - archiver.start(); - } - /** * @return Consistent ID. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java index 63228a0c9627a..7f859a204e138 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java @@ -560,7 +560,7 @@ else if (!cancelled.contains(res.requestId())) final Object topic = topic(cctx.nodeId(), req.id()); - cctx.io().addOrderedCacheHandler(topic, resHnd); + cctx.io().addOrderedCacheHandler(cctx.shared(), topic, resHnd); fut.listen(new CI1>() { @Override public void apply(IgniteInternalFuture fut) { @@ -744,7 +744,7 @@ else if (!cancelled.contains(res.requestId())) final Object topic = topic(cctx.nodeId(), req.id()); - cctx.io().addOrderedCacheHandler(topic, resHnd); + cctx.io().addOrderedCacheHandler(cctx.shared(), topic, resHnd); fut.listen(new CI1>() { @Override public void apply(IgniteInternalFuture fut) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java index 62ead23f8540d..8ff2f5aafa0cc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java @@ -34,7 +34,6 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.store.CacheStore; -import org.apache.ignite.cache.store.CacheStore; import org.apache.ignite.cache.store.CacheStoreSession; import org.apache.ignite.cache.store.CacheStoreSessionListener; import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java index bc2e49abdeb05..269925d42671f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheVersionManager.java @@ -87,12 +87,6 @@ public class GridCacheVersionManager extends GridCacheSharedManagerAdapter { cctx.gridEvents().addLocalEventListener(discoLsnr, EVT_NODE_METRICS_UPDATED); } - /** {@inheritDoc} */ - @Override protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException { - for (ClusterNode n : cctx.discovery().remoteNodes()) - onReceived(n.id(), n.metrics().getLastDataVersion()); - } - /** {@inheritDoc} */ @Override protected void stop0(boolean cancel) { cctx.gridEvents().removeLocalEventListener(discoLsnr, EVT_NODE_METRICS_UPDATED); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java index 1dd47edc24bca..dad6728573cfc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java @@ -43,11 +43,6 @@ public interface IgniteCacheObjectProcessor extends GridProcessor { */ public void onContinuousProcessorStarted(GridKernalContext ctx) throws IgniteCheckedException; - /** - * @throws IgniteCheckedException If failed. - */ - public void onUtilityCacheStarted() throws IgniteCheckedException; - /** * @param typeName Type name. * @return Type ID. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java index 67e14dc084d81..70711e512518a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java @@ -306,11 +306,6 @@ protected final int partition(CacheObjectContext ctx, @Nullable GridCacheContext // No-op. } - /** {@inheritDoc} */ - @Override public void onUtilityCacheStarted() throws IgniteCheckedException { - // No-op. - } - /** {@inheritDoc} */ @Override public int typeId(String typeName) { return 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java new file mode 100644 index 0000000000000..07711984372a2 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cluster; + +import java.util.UUID; +import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.lang.IgniteUuid; +import org.jetbrains.annotations.Nullable; + +/** + * + */ +public class ChangeGlobalStateFinishMessage implements DiscoveryCustomMessage { + /** */ + private static final long serialVersionUID = 0L; + + /** Custom message ID. */ + private IgniteUuid id = IgniteUuid.randomUuid(); + + /** State change request ID. */ + private UUID reqId; + + /** New cluster state. */ + private boolean clusterActive; + + /** + * @param reqId State change request ID. + * @param clusterActive New cluster state. + */ + public ChangeGlobalStateFinishMessage(UUID reqId, boolean clusterActive) { + assert reqId != null; + + this.reqId = reqId; + this.clusterActive = clusterActive; + } + + /** + * @return State change request ID. + */ + public UUID requestId() { + return reqId; + } + + /** + * @return New cluster state. + */ + public boolean clusterActive() { + return clusterActive; + } + + /** {@inheritDoc} */ + @Override public IgniteUuid id() { + return id; + } + + /** {@inheritDoc} */ + @Nullable @Override public DiscoveryCustomMessage ackMessage() { + return null; + } + + /** {@inheritDoc} */ + @Override public boolean isMutable() { + return false; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(ChangeGlobalStateFinishMessage.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ChangeGlobalStateMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java similarity index 57% rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ChangeGlobalStateMessage.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java index 4d1a50b1c09e7..6579399c72f6b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ChangeGlobalStateMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java @@ -15,10 +15,14 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache; +package org.apache.ignite.internal.processors.cluster; +import java.util.List; import java.util.UUID; import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; +import org.apache.ignite.internal.processors.cache.ExchangeActions; +import org.apache.ignite.internal.processors.cache.StoredCacheData; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; @@ -34,7 +38,7 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage { private IgniteUuid id = IgniteUuid.randomUuid(); /** Request ID */ - private UUID requestId; + private UUID reqId; /** Initiator node ID. */ private UUID initiatingNodeId; @@ -42,32 +46,55 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage { /** If true activate else deactivate. */ private boolean activate; - /** Batch contains all requests for start or stop caches. */ - private DynamicCacheChangeBatch changeGlobalStateBatch; + /** Configurations read from persistent store. */ + private List storedCfgs; - /** If happened concurrent activate/deactivate then processed only first message, other message must be skip. */ - private boolean concurrentChangeState; + /** */ + @GridToStringExclude + private transient ExchangeActions exchangeActions; /** - * + * @param reqId State change request ID. + * @param initiatingNodeId Node initiated state change. + * @param storedCfgs Configurations read from persistent store. + * @param activate New cluster state. */ public ChangeGlobalStateMessage( - UUID requestId, + UUID reqId, UUID initiatingNodeId, - boolean activate, - DynamicCacheChangeBatch changeGlobalStateBatch + @Nullable List storedCfgs, + boolean activate ) { - this.requestId = requestId; + assert reqId != null; + assert initiatingNodeId != null; + + this.reqId = reqId; this.initiatingNodeId = initiatingNodeId; + this.storedCfgs = storedCfgs; this.activate = activate; - this.changeGlobalStateBatch = changeGlobalStateBatch; } /** - * + * @return Configurations read from persistent store.. */ - public DynamicCacheChangeBatch getDynamicCacheChangeBatch() { - return changeGlobalStateBatch; + @Nullable public List storedCacheConfigurations() { + return storedCfgs; + } + + /** + * @return Cache updates to be executed on exchange. If {@code null} exchange is not needed. + */ + @Nullable public ExchangeActions exchangeActions() { + return exchangeActions; + } + + /** + * @param exchangeActions Cache updates to be executed on exchange. + */ + void exchangeActions(ExchangeActions exchangeActions) { + assert exchangeActions != null && !exchangeActions.empty() : exchangeActions; + + this.exchangeActions = exchangeActions; } /** {@inheritDoc} */ @@ -77,7 +104,7 @@ public DynamicCacheChangeBatch getDynamicCacheChangeBatch() { /** {@inheritDoc} */ @Nullable @Override public DiscoveryCustomMessage ackMessage() { - return !concurrentChangeState ? changeGlobalStateBatch : null; + return null; } /** {@inheritDoc} */ @@ -85,32 +112,25 @@ public DynamicCacheChangeBatch getDynamicCacheChangeBatch() { return false; } - /** - * - */ + /** + * @return Node initiated state change. + */ public UUID initiatorNodeId() { return initiatingNodeId; } /** - * + * @return New cluster state. */ public boolean activate() { return activate; } /** - * + * @return State change request ID. */ public UUID requestId() { - return requestId; - } - - /** - * - */ - public void concurrentChangeState() { - this.concurrentChangeState = true; + return reqId; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java index ad95a7846f290..dc503fb264838 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java @@ -270,7 +270,6 @@ public IgniteFuture clientReconnectFuture() { dataBag.addNodeSpecificData(CLUSTER_PROC.ordinal(), getDiscoveryData()); } - /** * @return Discovery data. */ @@ -314,7 +313,7 @@ private Boolean findLastFlag(Collection vals) { } /** {@inheritDoc} */ - @Override public void onKernalStart() throws IgniteCheckedException { + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { if (notifyEnabled.get()) { try { verChecker = new GridUpdateNotifier(ctx.igniteInstanceName(), diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java new file mode 100644 index 0000000000000..71bf90b111bf0 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cluster; + +import java.io.Serializable; +import java.util.Set; +import java.util.UUID; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.jetbrains.annotations.Nullable; + +/** + * Discovery data related to cluster state. + */ +public class DiscoveryDataClusterState implements Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private final boolean active; + + /** */ + private final UUID transitionReqId; + + /** Topology version for state change exchange. */ + @GridToStringInclude + private final AffinityTopologyVersion transitionTopVer; + + /** Nodes participating in state change exchange. */ + @GridToStringExclude + private final Set transitionNodes; + + /** Local flag for state transition result (global state is updated asynchronously by custom message). */ + private transient volatile Boolean transitionRes; + + /** + * @param active Current status. + * @return State instance. + */ + static DiscoveryDataClusterState createState(boolean active) { + return new DiscoveryDataClusterState(active, null, null, null); + } + + /** + * @param active New status. + * @param transitionReqId State change request ID. + * @param transitionTopVer State change topology version. + * @param transitionNodes Nodes participating in state change exchange. + * @return State instance. + */ + static DiscoveryDataClusterState createTransitionState(boolean active, + UUID transitionReqId, + AffinityTopologyVersion transitionTopVer, + Set transitionNodes) { + assert transitionReqId != null; + assert transitionTopVer != null; + assert !F.isEmpty(transitionNodes) : transitionNodes; + + return new DiscoveryDataClusterState(active, transitionReqId, transitionTopVer, transitionNodes); + } + + /** + * @param active New state. + * @param transitionReqId State change request ID. + * @param transitionTopVer State change topology version. + * @param transitionNodes Nodes participating in state change exchange. + */ + private DiscoveryDataClusterState(boolean active, + @Nullable UUID transitionReqId, + @Nullable AffinityTopologyVersion transitionTopVer, + @Nullable Set transitionNodes) { + this.active = active; + this.transitionReqId = transitionReqId; + this.transitionTopVer = transitionTopVer; + this.transitionNodes = transitionNodes; + } + + /** + * @return Local flag for state transition result (global state is updated asynchronously by custom message). + */ + @Nullable public Boolean transitionResult() { + return transitionRes; + } + + /** + * Discovery cluster state is changed asynchronously by discovery message, this methods changes local status + * for public API calls. + * + * @param reqId Request ID. + * @param active New cluster state. + */ + public void setTransitionResult(UUID reqId, boolean active) { + if (reqId.equals(transitionReqId)) { + assert transitionRes == null : this; + + transitionRes = active; + } + } + + /** + * @return State change request ID. + */ + public UUID transitionRequestId() { + return transitionReqId; + } + + /** + * @return {@code True} if state change is in progress. + */ + public boolean transition() { + return transitionReqId != null; + } + + /** + * @return State change exchange version. + */ + public AffinityTopologyVersion transitionTopologyVersion() { + return transitionTopVer; + } + + /** + * @return Current cluster state (or new state in case when transition is in progress). + */ + public boolean active() { + return active; + } + + /** + * @return Nodes participating in state change exchange. + */ + public Set transitionNodes() { + return transitionNodes; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(DiscoveryDataClusterState.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java index d57c720aff92c..8cea13f6dd801 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java @@ -18,49 +18,34 @@ package org.apache.ignite.internal.processors.cluster; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteCompute; -import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.events.DiscoveryEvent; import org.apache.ignite.events.Event; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.cluster.ClusterGroupAdapter; -import org.apache.ignite.internal.managers.discovery.CustomEventListener; +import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.managers.discovery.DiscoCache; import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; -import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.CacheClientReconnectDiscoveryData; -import org.apache.ignite.internal.processors.cache.CacheData; -import org.apache.ignite.internal.processors.cache.CacheJoinNodeDiscoveryData; -import org.apache.ignite.internal.processors.cache.CacheJoinNodeDiscoveryData.CacheInfo; -import org.apache.ignite.internal.processors.cache.CacheNodeCommonDiscoveryData; -import org.apache.ignite.internal.processors.cache.ChangeGlobalStateMessage; -import org.apache.ignite.internal.processors.cache.ClusterState; -import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch; -import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest; import org.apache.ignite.internal.processors.cache.ExchangeActions; import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.GridChangeGlobalStateMessageResponse; +import org.apache.ignite.internal.processors.cache.StateChangeRequest; import org.apache.ignite.internal.processors.cache.StoredCacheData; -import org.apache.ignite.internal.processors.query.QuerySchema; import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -72,34 +57,27 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteRunnable; -import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.resources.IgniteInstanceResource; import org.apache.ignite.spi.discovery.DiscoveryDataBag; -import org.apache.ignite.spi.discovery.DiscoveryDataBag.JoiningNodeDiscoveryData; import org.jetbrains.annotations.Nullable; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; -import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.CACHE_PROC; import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.STATE_PROC; import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL; -import static org.apache.ignite.internal.processors.cache.ClusterState.ACTIVE; -import static org.apache.ignite.internal.processors.cache.ClusterState.INACTIVE; -import static org.apache.ignite.internal.processors.cache.ClusterState.TRANSITION; -import static org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest.stopRequest; /** * */ public class GridClusterStateProcessor extends GridProcessorAdapter { - /** Global status. */ - private volatile ClusterState globalState; - - /** Action context. */ - private volatile ChangeGlobalStateContext lastCgsCtx; + /** */ + private volatile DiscoveryDataClusterState globalState; /** Local action future. */ - private final AtomicReference cgsLocFut = new AtomicReference<>(); + private final AtomicReference stateChangeFut = new AtomicReference<>(); + + /** Future initialized if node joins when cluster state change is in progress. */ + private TransitionOnJoinWaitFuture joinFut; /** Process. */ @GridToStringExclude @@ -109,12 +87,6 @@ public class GridClusterStateProcessor extends GridProcessorAdapter { @GridToStringExclude private GridCacheSharedContext sharedCtx; - /** */ - private final ConcurrentHashMap cacheData = new ConcurrentHashMap<>(); - - /** */ - private volatile CacheJoinNodeDiscoveryData localCacheData; - /** Listener. */ private final GridLocalEventListener lsr = new GridLocalEventListener() { @Override public void onEvent(Event evt) { @@ -124,14 +96,15 @@ public class GridClusterStateProcessor extends GridProcessorAdapter { assert e.type() == EVT_NODE_LEFT || e.type() == EVT_NODE_FAILED : this; - final GridChangeGlobalStateFuture f = cgsLocFut.get(); + final GridChangeGlobalStateFuture f = stateChangeFut.get(); - if (f != null) + if (f != null) { f.initFut.listen(new CI1>() { @Override public void apply(IgniteInternalFuture fut) { - f.onDiscoveryEvent(e); + f.onNodeLeft(e); } }); + } } }; @@ -142,531 +115,417 @@ public GridClusterStateProcessor(GridKernalContext ctx) { super(ctx); } - /** {@inheritDoc} */ - @Override public void start() throws IgniteCheckedException { - // Start first node as inactive if persistent enable. - globalState = ctx.config().isPersistentStoreEnabled() ? INACTIVE : - ctx.config().isActiveOnStart() ? ACTIVE : INACTIVE; - - ctx.discovery().setCustomEventListener( - ChangeGlobalStateMessage.class, new CustomEventListener() { - @Override public void onCustomEvent( - AffinityTopologyVersion topVer, ClusterNode snd, ChangeGlobalStateMessage msg) { - assert topVer != null; - assert snd != null; - assert msg != null; - - boolean activate = msg.activate(); - - ChangeGlobalStateContext actx = lastCgsCtx; - - if (actx != null && globalState == TRANSITION) { - GridChangeGlobalStateFuture f = cgsLocFut.get(); - - if (log.isDebugEnabled()) - log.debug("Concurrent " + prettyStr(activate) + " [id=" + - ctx.localNodeId() + " topVer=" + topVer + " actx=" + actx + ", msg=" + msg + "]"); - - if (f != null && f.requestId.equals(msg.requestId())) - f.onDone(new IgniteCheckedException( - "Concurrent change state, now in progress=" + (activate) - + ", initiatingNodeId=" + actx.initiatingNodeId - + ", you try=" + (prettyStr(activate)) + ", locNodeId=" + ctx.localNodeId() - )); - - msg.concurrentChangeState(); - } - else { - if (log.isInfoEnabled()) - log.info("Create " + prettyStr(activate) + " context [id=" + - ctx.localNodeId() + " topVer=" + topVer + ", reqId=" + - msg.requestId() + ", initiatingNodeId=" + msg.initiatorNodeId() + "]"); - - lastCgsCtx = new ChangeGlobalStateContext( - msg.requestId(), - msg.initiatorNodeId(), - msg.getDynamicCacheChangeBatch(), - msg.activate()); - - globalState = TRANSITION; - } - } - }); - - ctx.event().addLocalEventListener(lsr, EVT_NODE_LEFT, EVT_NODE_FAILED); - } - /** - * @param data Joining node discovery data. + * @return Cluster state to be used on public API. */ - public void cacheProcessorStarted(CacheJoinNodeDiscoveryData data) { - assert data != null; + public boolean publicApiActiveState() { + DiscoveryDataClusterState globalState = this.globalState; - localCacheData = data; + assert globalState != null; - cacheProc = ctx.cache(); - sharedCtx = cacheProc.context(); + if (globalState.transition()) { + Boolean transitionRes = globalState.transitionResult(); - sharedCtx.io().addCacheHandler( - 0, GridChangeGlobalStateMessageResponse.class, - new CI2() { - @Override public void apply(UUID nodeId, GridChangeGlobalStateMessageResponse msg) { - processChangeGlobalStateResponse(nodeId, msg); - } - }); + if (transitionRes != null) + return transitionRes; + else + return false; + } + else + return globalState.active(); } /** {@inheritDoc} */ - @Override public void stop(boolean cancel) throws IgniteCheckedException { - super.stop(cancel); - - sharedCtx.io().removeHandler(false, 0, GridChangeGlobalStateMessageResponse.class); - ctx.event().removeLocalEventListener(lsr, EVT_NODE_LEFT, EVT_NODE_FAILED); - - IgniteCheckedException stopErr = new IgniteInterruptedCheckedException( - "Node is stopping: " + ctx.igniteInstanceName()); - - GridChangeGlobalStateFuture f = cgsLocFut.get(); + @Override public void start() throws IgniteCheckedException { + // Start first node as inactive if persistence is enabled. + boolean activeOnStart = !ctx.config().isPersistentStoreEnabled() && ctx.config().isActiveOnStart(); - if (f != null) - f.onDone(stopErr); + globalState = DiscoveryDataClusterState.createState(activeOnStart); - cgsLocFut.set(null); + ctx.event().addLocalEventListener(lsr, EVT_NODE_LEFT, EVT_NODE_FAILED); } /** {@inheritDoc} */ - @Override public void onKernalStart() throws IgniteCheckedException { - super.onKernalStart(); + @Override public void onKernalStop(boolean cancel) { + GridChangeGlobalStateFuture fut = this.stateChangeFut.get(); - if (ctx.isDaemon()) - return; + if (fut != null) + fut.onDone(new IgniteCheckedException("Failed to wait for cluster state change, node is stopping.")); - List nodes = ctx.discovery().serverNodes(AffinityTopologyVersion.NONE); - - assert localCacheData != null; - - // First node started (coordinator). - if (nodes.isEmpty() || nodes.get(0).isLocal()) - cacheData.putAll(localCacheData.caches()); - - if (globalState == INACTIVE) { // Accept inactivate state after join. - if (log != null && log.isInfoEnabled()) - log.info("Got inactivate state from cluster during node join."); - - // Revert start action if get INACTIVE state on join. - sharedCtx.snapshot().onDeActivate(ctx); - - if (sharedCtx.pageStore() != null) - sharedCtx.pageStore().onDeActivate(ctx); - - if (sharedCtx.wal() != null) - sharedCtx.wal().onDeActivate(ctx); - - sharedCtx.database().onDeActivate(ctx); - } - } - - /** {@inheritDoc} */ - @Nullable @Override public DiscoveryDataExchangeType discoveryDataType() { - return DiscoveryDataExchangeType.STATE_PROC; + super.onKernalStop(cancel); } - /** {@inheritDoc} */ - @Override public void collectGridNodeData(DiscoveryDataBag dataBag) { - if (!dataBag.commonDataCollectedFor(STATE_PROC.ordinal())) - dataBag.addGridCommonData(STATE_PROC.ordinal(), globalState); - } + /** + * @param discoCache Discovery data cache. + * @return If transition is in progress returns future which is completed when transition finishes. + */ + @Nullable public IgniteInternalFuture onLocalJoin(DiscoCache discoCache) { + if (globalState.transition()) { + joinFut = new TransitionOnJoinWaitFuture(globalState, discoCache); - /** {@inheritDoc} */ - @Override public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) { - ClusterState state = (ClusterState)data.commonData(); + return joinFut; + } - if (state != null) - globalState = state; + return null; } /** - * + * @param node Failed node. + * @return Message if cluster state changed. */ - public IgniteInternalFuture changeGlobalState(final boolean activate) { - if (ctx.isDaemon()) { - GridFutureAdapter fut = new GridFutureAdapter<>(); + @Nullable public ChangeGlobalStateFinishMessage onNodeLeft(ClusterNode node) { + if (globalState.transition()) { + Set nodes = globalState.transitionNodes(); - sendCompute(activate, fut); + if (nodes.remove(node.id()) && nodes.isEmpty()) { + U.warn(log, "Failed to change cluster state, all participating nodes failed. " + + "Switching to inactive state."); - return fut; - } + ChangeGlobalStateFinishMessage msg = + new ChangeGlobalStateFinishMessage(globalState.transitionRequestId(), false); - if (cacheProc.transactions().tx() != null || sharedCtx.lockedTopologyVersion(null) != null) - throw new IgniteException("Failed to " + prettyStr(activate) + " cluster (must invoke the " + - "method outside of an active transaction)."); + onStateFinishMessage(msg); - if ((globalState == ACTIVE && activate) || (globalState == INACTIVE && !activate)) - return new GridFinishedFuture<>(); + return msg; + } + } - final UUID requestId = UUID.randomUUID(); + return null; + } - final GridChangeGlobalStateFuture cgsFut = new GridChangeGlobalStateFuture(requestId, activate, ctx); + /** + * @param msg Message. + */ + public void onStateFinishMessage(ChangeGlobalStateFinishMessage msg) { + if (msg.requestId().equals(globalState.transitionRequestId())) { + log.info("Received state change finish message: " + msg.clusterActive()); - if (!cgsLocFut.compareAndSet(null, cgsFut)) { - GridChangeGlobalStateFuture locF = cgsLocFut.get(); + globalState = DiscoveryDataClusterState.createState(msg.clusterActive()); - if (locF.activate == activate) - return locF; + ctx.cache().onStateChangeFinish(msg); - return new GridFinishedFuture<>(new IgniteException( - "Failed to " + prettyStr(activate) + ", because another state change operation is currently " + - "in progress: " + prettyStr(locF.activate))); - } + TransitionOnJoinWaitFuture joinFut = this.joinFut; - if (globalState == ACTIVE && !activate && ctx.cache().context().snapshot().snapshotOperationInProgress()){ - return new GridFinishedFuture<>(new IgniteException( - "Failed to " + prettyStr(activate) + ", because snapshot operation in progress.")); + if (joinFut != null) + joinFut.onDone(false); } + else + U.warn(log, "Received state finish message with unexpected ID: " + msg); + } - if (ctx.clientNode()) - sendCompute(activate, cgsFut); - else { - try { - List reqs = new ArrayList<>(); - - DynamicCacheChangeRequest changeGlobalStateReq = new DynamicCacheChangeRequest( - requestId, activate ? ACTIVE : INACTIVE, ctx.localNodeId()); - - reqs.add(changeGlobalStateReq); - - List cacheReqs = activate ? startAllCachesRequests() : stopAllCachesRequests(); + /** + * @param topVer Current topology version. + * @param msg Message. + * @param discoCache Current nodes. + * @return {@code True} if need start state change process. + */ + public boolean onStateChangeMessage(AffinityTopologyVersion topVer, + ChangeGlobalStateMessage msg, + DiscoCache discoCache) { + if (globalState.transition()) { + if (globalState.active() != msg.activate()) { + GridChangeGlobalStateFuture fut = changeStateFuture(msg); + + if (fut != null) + fut.onDone(concurrentStateChangeError(msg.activate())); + } + else { + final GridChangeGlobalStateFuture stateFut = changeStateFuture(msg); - reqs.addAll(cacheReqs); + if (stateFut != null) { + IgniteInternalFuture exchFut = ctx.cache().context().exchange().affinityReadyFuture( + globalState.transitionTopologyVersion()); - printCacheInfo(cacheReqs, activate); + if (exchFut == null) + exchFut = new GridFinishedFuture<>(); - ChangeGlobalStateMessage changeGlobalStateMsg = new ChangeGlobalStateMessage( - requestId, ctx.localNodeId(), activate, new DynamicCacheChangeBatch(reqs)); + exchFut.listen(new CI1>() { + @Override public void apply(IgniteInternalFuture exchFut) { + stateFut.onDone(); + } + }); + } + } + } + else { + if (globalState.active() != msg.activate()) { + ExchangeActions exchangeActions; try { - ctx.discovery().sendCustomEvent(changeGlobalStateMsg); - - if (ctx.isStopping()) - cgsFut.onDone(new IgniteCheckedException("Failed to execute " + prettyStr(activate) + " request, " + - "node is stopping.")); + exchangeActions = ctx.cache().onStateChangeRequest(msg, topVer); } catch (IgniteCheckedException e) { - U.error(log, "Failed to create or send global state change request: " + cgsFut, e); - - cgsFut.onDone(e); - } - } - catch (IgniteCheckedException e) { - cgsFut.onDone(e); - } - } + GridChangeGlobalStateFuture fut = changeStateFuture(msg); - return cgsFut; - } + if (fut != null) + fut.onDone(e); - /** - * - */ - private void sendCompute(boolean activate, final GridFutureAdapter res) { - AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx(); + return false; + } - IgniteCompute comp = ((ClusterGroupAdapter)ctx.cluster().get().forServers()).compute(); + Set nodeIds = U.newHashSet(discoCache.allNodes().size()); - if (log.isInfoEnabled()) - log.info("Sending " + prettyStr(activate) + " request from node [id=" + - ctx.localNodeId() + " topVer=" + topVer + " isClient=" + ctx.isDaemon() + - " isDaemon" + ctx.isDaemon() + "]"); + for (ClusterNode node : discoCache.allNodes()) + nodeIds.add(node.id()); - IgniteFuture fut = comp.runAsync(new ClientChangeGlobalStateComputeRequest(activate)); + GridChangeGlobalStateFuture fut = changeStateFuture(msg); - fut.listen(new CI1() { - @Override public void apply(IgniteFuture fut) { - try { - fut.get(); + if (fut != null) + fut.setRemaining(nodeIds, topVer.nextMinorVersion()); - res.onDone(); - } - catch (Exception e) { - res.onDone(e); - } - } - }); - } - /** - * @param reqs Requests to print. - * @param active Active flag. - */ - private void printCacheInfo(List reqs, boolean active) { - assert reqs != null; + log.info("Start state transition: " + msg.activate()); - StringBuilder sb = new StringBuilder(); + globalState = DiscoveryDataClusterState.createTransitionState(msg.activate(), + msg.requestId(), + topVer, + nodeIds); - sb.append("["); + AffinityTopologyVersion stateChangeTopVer = topVer.nextMinorVersion(); - for (int i = 0; i < reqs.size() - 1; i++) - sb.append(reqs.get(i).cacheName()).append(", "); + StateChangeRequest req = new StateChangeRequest(msg, stateChangeTopVer); - sb.append(reqs.get(reqs.size() - 1).cacheName()); + exchangeActions.stateChangeRequest(req); - sb.append("]"); + msg.exchangeActions(exchangeActions); - sb.append(" ").append(reqs.size()) - .append(" caches will be ") - .append(active ? "started" : "stopped"); + return true; + } + else { + // State already changed. + GridChangeGlobalStateFuture stateFut = changeStateFuture(msg); - if (log.isInfoEnabled()) - log.info(sb.toString()); - } + if (stateFut != null) + stateFut.onDone(); + } + } - /** - * @param req Cache being started. - */ - public void onCacheStart(DynamicCacheChangeRequest req) { - CacheInfo cacheInfo = cacheData.get(req.cacheName()); - - if (cacheInfo == null) - cacheData.put(req.cacheName(), - new CacheInfo( - new StoredCacheData(req.startCacheConfiguration()), - req.cacheType(), req.sql(), - 0L) - ); + return false; } /** - * @param req Cache being stopped. + * @return Current cluster state, should be called only from discovery thread. */ - public void onCacheStop(DynamicCacheChangeRequest req) { - CacheInfo cacheInfo = cacheData.get(req.cacheName()); - - if (cacheInfo != null) - cacheData.remove(req.cacheName()); + public DiscoveryDataClusterState clusterState() { + return globalState; } /** - * @return All caches map. + * @param msg State change message. + * @return Local future for state change process. */ - private Map allCaches() { - Map cfgs = new HashMap<>(); - - for (Map.Entry entry : cacheData.entrySet()) - if (cfgs.get(entry.getKey()) == null) - cfgs.put(entry.getKey(), entry.getValue().cacheData().config()); - - return cfgs; + @Nullable private GridChangeGlobalStateFuture changeStateFuture(ChangeGlobalStateMessage msg) { + return changeStateFuture(msg.initiatorNodeId(), msg.requestId()); } /** - * @return Collection of all caches start requests. - * @throws IgniteCheckedException If failed to create requests. + * @param initiatorNode Node initiated state change process. + * @param reqId State change request ID. + * @return Local future for state change process. */ - private List startAllCachesRequests() throws IgniteCheckedException { - assert !ctx.config().isDaemon(); - - Collection cacheCfgs = allCaches().values(); - - final List reqs = new ArrayList<>(); - - if (sharedCtx.pageStore() != null && sharedCtx.database().persistenceEnabled()) { - Map ccfgs = sharedCtx.pageStore().readCacheConfigurations(); - - for (Map.Entry entry : ccfgs.entrySet()) - reqs.add(createRequest(entry.getValue().config())); + @Nullable private GridChangeGlobalStateFuture changeStateFuture(UUID initiatorNode, UUID reqId) { + assert initiatorNode != null; + assert reqId != null; - for (CacheConfiguration cfg : cacheCfgs) - if (!ccfgs.keySet().contains(cfg.getName())) - reqs.add(createRequest(cfg)); + if (initiatorNode.equals(ctx.localNodeId())) { + GridChangeGlobalStateFuture fut = stateChangeFut.get(); - return reqs; + if (fut != null && fut.requestId.equals(reqId)) + return fut; } - else { - for (CacheConfiguration cfg : cacheCfgs) - reqs.add(createRequest(cfg)); - return reqs; - } + return null; } /** - * @return Collection of requests to stop caches. + * @param activate New state. + * @return State change error. */ - private List stopAllCachesRequests() { - Collection cacheCfgs = allCaches().values(); - - List reqs = new ArrayList<>(cacheCfgs.size()); - - for (CacheConfiguration cfg : cacheCfgs) { - DynamicCacheChangeRequest req = stopRequest(ctx, cfg.getName(), false, false); - - reqs.add(req); - } - - return reqs; + private IgniteCheckedException concurrentStateChangeError(boolean activate) { + return new IgniteCheckedException("Failed to " + prettyStr(activate) + + ", because another state change operation is currently in progress: " + prettyStr(!activate)); } /** - * @param cfg Configuration to create request for. - * @return Dynamic cache change request. + * */ - private DynamicCacheChangeRequest createRequest(CacheConfiguration cfg) { - assert cfg != null; - assert cfg.getName() != null; - - String cacheName = cfg.getName(); + public void cacheProcessorStarted() { + cacheProc = ctx.cache(); + sharedCtx = cacheProc.context(); - DynamicCacheChangeRequest req = new DynamicCacheChangeRequest( - UUID.randomUUID(), cacheName, ctx.localNodeId()); + sharedCtx.io().addCacheHandler( + 0, GridChangeGlobalStateMessageResponse.class, + new CI2() { + @Override public void apply(UUID nodeId, GridChangeGlobalStateMessageResponse msg) { + processChangeGlobalStateResponse(nodeId, msg); + } + }); + } - req.startCacheConfiguration(cfg); - req.template(cfg.getName().endsWith("*")); - req.nearCacheConfiguration(cfg.getNearConfiguration()); - req.deploymentId(IgniteUuid.randomUuid()); - req.schema(new QuerySchema(cfg.getQueryEntities())); - req.cacheType(cacheProc.cacheType(cacheName)); + /** {@inheritDoc} */ + @Override public void stop(boolean cancel) throws IgniteCheckedException { + super.stop(cancel); - return req; - } + if (sharedCtx != null) + sharedCtx.io().removeHandler(false, 0, GridChangeGlobalStateMessageResponse.class); - /** - * - */ - public boolean active() { - ChangeGlobalStateContext actx = lastCgsCtx; + ctx.event().removeLocalEventListener(lsr, EVT_NODE_LEFT, EVT_NODE_FAILED); - if (actx != null && !actx.activate && globalState == TRANSITION) - return true; + IgniteCheckedException stopErr = new IgniteCheckedException( + "Node is stopping: " + ctx.igniteInstanceName()); - if (actx != null && actx.activate && globalState == TRANSITION) - return false; + GridChangeGlobalStateFuture f = stateChangeFut.get(); - return globalState == ACTIVE; + if (f != null) + f.onDone(stopErr); } - /** - * @param cacheName Cache name to check. - * @return Locally configured flag. - */ - public boolean isLocallyConfigured(String cacheName){ - assert localCacheData != null; + /** {@inheritDoc} */ + @Nullable @Override public DiscoveryDataExchangeType discoveryDataType() { + return DiscoveryDataExchangeType.STATE_PROC; + } - return localCacheData.caches().containsKey(cacheName) || localCacheData.templates().containsKey(cacheName); + /** {@inheritDoc} */ + @Override public void collectGridNodeData(DiscoveryDataBag dataBag) { + if (!dataBag.commonDataCollectedFor(STATE_PROC.ordinal())) + dataBag.addGridCommonData(STATE_PROC.ordinal(), globalState); } - /** - * Invoked if cluster is inactive. - * - * @param dataBag Bag to collect data to. - */ - public void collectGridNodeData0(DiscoveryDataBag dataBag) { - if (!dataBag.commonDataCollectedFor(CACHE_PROC.ordinal())) - dataBag.addGridCommonData(CACHE_PROC.ordinal(), cacheData); + /** {@inheritDoc} */ + @Override public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) { + DiscoveryDataClusterState state = (DiscoveryDataClusterState)data.commonData(); + + if (state != null) + globalState = state; } /** - * @param data Joining node discovery data. + * @param activate New cluster state. + * @return State change future. */ - public void onJoiningNodeDataReceived0(JoiningNodeDiscoveryData data) { - if (data.hasJoiningNodeData()) { - if (data.joiningNodeData() instanceof CacheJoinNodeDiscoveryData) { - CacheJoinNodeDiscoveryData data0 = (CacheJoinNodeDiscoveryData)data.joiningNodeData(); + public IgniteInternalFuture changeGlobalState(final boolean activate) { + if (ctx.isDaemon() || ctx.clientNode()) { + GridFutureAdapter fut = new GridFutureAdapter<>(); - cacheData.putAll(data0.caches()); - } - else if (data.joiningNodeData() instanceof CacheClientReconnectDiscoveryData) { - CacheClientReconnectDiscoveryData data0 = (CacheClientReconnectDiscoveryData)data.joiningNodeData(); + sendCompute(activate, fut); - // No-op. - } + return fut; } - } - public void onGridDataReceived0(DiscoveryDataBag.GridDiscoveryData data) { - // Receive data from active cluster. - if (data.commonData() instanceof CacheNodeCommonDiscoveryData) { - CacheNodeCommonDiscoveryData data0 = (CacheNodeCommonDiscoveryData)data.commonData(); + if (cacheProc.transactions().tx() != null || sharedCtx.lockedTopologyVersion(null) != null) { + return new GridFinishedFuture<>(new IgniteCheckedException("Failed to " + prettyStr(activate) + + " cluster (must invoke the method outside of an active transaction).")); + } - Map caches = data0.caches(); + DiscoveryDataClusterState curState = globalState; - Map cacheInfos = new HashMap<>(); + if (!curState.transition() && curState.active() == activate) + return new GridFinishedFuture<>(); - for (Map.Entry entry : caches.entrySet()) { - CacheData val = entry.getValue(); + GridChangeGlobalStateFuture startedFut = null; - CacheInfo info = new CacheInfo( - new StoredCacheData(val.cacheConfiguration()), - val.cacheType(), - val.sql(), - val.flags() - ); + GridChangeGlobalStateFuture fut = stateChangeFut.get(); - cacheInfos.put(entry.getKey(), info); - } + while (fut == null) { + fut = new GridChangeGlobalStateFuture(UUID.randomUUID(), activate, ctx); - cacheData.putAll(cacheInfos); + if (stateChangeFut.compareAndSet(null, fut)) { + startedFut = fut; - } // Receive data from inactive cluster. - else if (data.commonData() instanceof Map) { - Map data0 = (Map)data.commonData(); + break; + } + else + fut = stateChangeFut.get(); + } - cacheData.putAll(data0); + if (startedFut == null) { + if (fut.activate != activate) { + return new GridFinishedFuture<>(new IgniteCheckedException("Failed to " + prettyStr(activate) + + ", because another state change operation is currently in progress: " + prettyStr(fut.activate))); + } + else + return fut; } - cacheData.putAll(localCacheData.caches()); - } + List storedCfgs = null; - /** - * @param exchActions Requests. - * @param topVer Exchange topology version. - */ - public boolean changeGlobalState( - ExchangeActions exchActions, - AffinityTopologyVersion topVer - ) { - assert exchActions != null; - assert topVer != null; + if (activate && sharedCtx.database().persistenceEnabled()) { + try { + Map cfgs = ctx.cache().context().pageStore().readCacheConfigurations(); + + if (!F.isEmpty(cfgs)) + storedCfgs = new ArrayList<>(cfgs.values()); + } + catch (IgniteCheckedException e) { + U.error(log, "Failed to read stored cache configurations: " + e, e); + + startedFut.onDone(e); - if (exchActions.newClusterState() != null) { - ChangeGlobalStateContext cgsCtx = lastCgsCtx; + return startedFut; + } + } - assert cgsCtx != null : topVer; + ChangeGlobalStateMessage msg = new ChangeGlobalStateMessage(startedFut.requestId, + ctx.localNodeId(), + storedCfgs, + activate); - cgsCtx.topologyVersion(topVer); + try { + ctx.discovery().sendCustomEvent(msg); - return true; + if (ctx.isStopping()) + startedFut.onDone(new IgniteCheckedException("Failed to execute " + prettyStr(activate) + " request, " + + "node is stopping.")); } + catch (IgniteCheckedException e) { + U.error(log, "Failed to send global state change request: " + activate, e); - return false; + startedFut.onDone(e); + } + + return startedFut; } /** - * Invoke from exchange future. + * @param activate New cluster state. + * @param resFut State change future. */ - public Exception onChangeGlobalState() { - GridChangeGlobalStateFuture f = cgsLocFut.get(); + private void sendCompute(boolean activate, final GridFutureAdapter resFut) { + AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx(); - ChangeGlobalStateContext cgsCtx = lastCgsCtx; + IgniteCompute comp = ((ClusterGroupAdapter)ctx.cluster().get().forServers()).compute(); - assert cgsCtx != null; + if (log.isInfoEnabled()) { + log.info("Sending " + prettyStr(activate) + " request from node [id=" + ctx.localNodeId() + + ", topVer=" + topVer + + ", client=" + ctx.clientNode() + + ", daemon" + ctx.isDaemon() + "]"); + } - if (f != null) - f.setRemaining(cgsCtx.topVer); + IgniteFuture fut = comp.runAsync(new ClientChangeGlobalStateComputeRequest(activate)); - return cgsCtx.activate ? onActivate(cgsCtx) : onDeActivate(cgsCtx); + fut.listen(new CI1() { + @Override public void apply(IgniteFuture fut) { + try { + fut.get(); + + resFut.onDone(); + } + catch (Exception e) { + resFut.onDone(e); + } + } + }); } /** - * @param exs Exs. + * @param errs Errors. + * @param req State change request. */ - public void onFullResponseMessage(Map exs) { - assert !F.isEmpty(exs); - - ChangeGlobalStateContext actx = lastCgsCtx; - - actx.setFail(); + public void onStateChangeError(Map errs, StateChangeRequest req) { + assert !F.isEmpty(errs); - // Revert change if activation request fail. - if (actx.activate) { + // Revert caches start if activation request fail. + if (req.activate()) { try { cacheProc.onKernalStopCaches(true); @@ -674,22 +533,10 @@ public void onFullResponseMessage(Map exs) { sharedCtx.affinity().removeAllCacheInfo(); - ctx.discovery().cleanCachesAndGroups(); - - if (!ctx.clientNode()) { - sharedCtx.database().onDeActivate(ctx); - - if (sharedCtx.pageStore() != null) - sharedCtx.pageStore().onDeActivate(ctx); - - if (sharedCtx.wal() != null) - sharedCtx.wal().onDeActivate(ctx); - } + if (!ctx.clientNode()) + sharedCtx.deactivate(); } catch (Exception e) { - for (Map.Entry entry : exs.entrySet()) - e.addSuppressed(entry.getValue()); - U.error(log, "Failed to revert activation request changes", e); } } @@ -697,110 +544,33 @@ public void onFullResponseMessage(Map exs) { //todo https://issues.apache.org/jira/browse/IGNITE-5480 } - globalState = actx.activate ? INACTIVE : ACTIVE; - - GridChangeGlobalStateFuture af = cgsLocFut.get(); + GridChangeGlobalStateFuture fut = changeStateFuture(req.initiatorNodeId(), req.requestId()); - if (af != null && af.requestId.equals(actx.requestId)) { + if (fut != null) { IgniteCheckedException e = new IgniteCheckedException( - "Fail " + prettyStr(actx.activate), + "Failed to " + prettyStr(req.activate()) + " cluster", null, false ); - for (Map.Entry entry : exs.entrySet()) + for (Map.Entry entry : errs.entrySet()) e.addSuppressed(entry.getValue()); - af.onDone(e); - } - } - - /** - * - */ - private Exception onActivate(ChangeGlobalStateContext cgsCtx) { - final boolean client = ctx.clientNode(); - - if (log.isInfoEnabled()) - log.info("Start activation process [nodeId=" + ctx.localNodeId() + ", client=" + client + - ", topVer=" + cgsCtx.topVer + "]"); - - try { - if (!client) - sharedCtx.database().lock(); - - IgnitePageStoreManager pageStore = sharedCtx.pageStore(); - - if (pageStore != null) - pageStore.onActivate(ctx); - - if (sharedCtx.wal() != null) - sharedCtx.wal().onActivate(ctx); - - sharedCtx.database().onActivate(ctx); - - sharedCtx.snapshot().onActivate(ctx); - - if (log.isInfoEnabled()) - log.info("Successfully activated persistence managers [nodeId=" - + ctx.localNodeId() + ", client=" + client + ", topVer=" + cgsCtx.topVer + "]"); - - return null; - } - catch (Exception e) { - U.error(log, "Failed to activate persistence managers [nodeId=" + ctx.localNodeId() + ", client=" + client + - ", topVer=" + cgsCtx.topVer + "]", e); - - if (!client) - sharedCtx.database().unLock(); - - return e; - } - } - - /** - * - */ - public Exception onDeActivate(ChangeGlobalStateContext cgsCtx) { - final boolean client = ctx.clientNode(); - - if (log.isInfoEnabled()) - log.info("Starting deactivation [id=" + ctx.localNodeId() + ", client=" + - client + ", topVer=" + cgsCtx.topVer + "]"); - - try { - ctx.dataStructures().onDeActivate(ctx); - - ctx.service().onDeActivate(ctx); - - if (log.isInfoEnabled()) - log.info("Successfully deactivated persistence processors [id=" + ctx.localNodeId() + ", client=" + - client + ", topVer=" + cgsCtx.topVer + "]"); - - return null; - } - catch (Exception e) { - U.error(log, "Failed to execute deactivation callback [nodeId=" + ctx.localNodeId() + ", client=" + client + - ", topVer=" + cgsCtx.topVer + "]", e); - - return e; + fut.onDone(e); } } /** - * + * @param req State change request. */ - private void onFinalActivate(final ChangeGlobalStateContext cgsCtx) { - IgniteInternalFuture asyncActivateFut = ctx.closure().runLocalSafe(new Runnable() { + private void onFinalActivate(final StateChangeRequest req) { + ctx.closure().runLocalSafe(new Runnable() { @Override public void run() { boolean client = ctx.clientNode(); Exception e = null; try { - if (!ctx.config().isDaemon()) - ctx.cacheObjects().onUtilityCacheStarted(); - ctx.service().onUtilityCacheStarted(); ctx.service().onActivate(ctx); @@ -809,146 +579,114 @@ private void onFinalActivate(final ChangeGlobalStateContext cgsCtx) { if (log.isInfoEnabled()) log.info("Successfully performed final activation steps [nodeId=" - + ctx.localNodeId() + ", client=" + client + ", topVer=" + cgsCtx.topVer + "]"); + + ctx.localNodeId() + ", client=" + client + ", topVer=" + req.topologyVersion() + "]"); } catch (Exception ex) { - e = ex; + e = new IgniteCheckedException("Failed to perform final activation steps", ex); U.error(log, "Failed to perform final activation steps [nodeId=" + ctx.localNodeId() + - ", client=" + client + ", topVer=" + lastCgsCtx.topVer + "]", ex); + ", client=" + client + ", topVer=" + req.topologyVersion() + "]", ex); } finally { - globalState = ACTIVE; - - sendChangeGlobalStateResponse(cgsCtx.requestId, cgsCtx.initiatingNodeId, e); + globalState.setTransitionResult(req.requestId(), true); - lastCgsCtx = null; + sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), e); } } }); - - cgsCtx.setAsyncActivateFut(asyncActivateFut); } /** - * + * @param req State change request. */ - public void onFinalDeActivate(ChangeGlobalStateContext cgsCtx) { - final boolean client = ctx.clientNode(); - - if (log.isInfoEnabled()) - log.info("Successfully performed final deactivation steps [nodeId=" - + ctx.localNodeId() + ", client=" + client + ", topVer=" + cgsCtx.topVer + "]"); - - Exception ex = null; - - try { - sharedCtx.snapshot().onDeActivate(ctx); + private void onFinalDeActivate(final StateChangeRequest req) { + globalState.setTransitionResult(req.requestId(), false); - sharedCtx.database().onDeActivate(ctx); - - if (sharedCtx.pageStore() != null) - sharedCtx.pageStore().onDeActivate(ctx); - - if (sharedCtx.wal() != null) - sharedCtx.wal().onDeActivate(ctx); - - sharedCtx.affinity().removeAllCacheInfo(); - } - catch (Exception e) { - ex = e; - } - finally { - globalState = INACTIVE; - } - - sendChangeGlobalStateResponse(cgsCtx.requestId, cgsCtx.initiatingNodeId, ex); - - lastCgsCtx = null; + sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), null); } /** - * + * @param req State change request. */ - public void onExchangeDone() { - ChangeGlobalStateContext cgsCtx = lastCgsCtx; - - assert cgsCtx != null; - - if (!cgsCtx.isFail()) { - if (cgsCtx.activate) - onFinalActivate(cgsCtx); - else - onFinalDeActivate(cgsCtx); - } + public void onStateChangeExchangeDone(StateChangeRequest req) { + if (req.activate()) + onFinalActivate(req); else - lastCgsCtx = null; + onFinalDeActivate(req); } /** + * @param reqId Request ID. * @param initNodeId Initialize node id. * @param ex Exception. */ - private void sendChangeGlobalStateResponse(UUID requestId, UUID initNodeId, Exception ex) { - assert requestId != null; + private void sendChangeGlobalStateResponse(UUID reqId, UUID initNodeId, Exception ex) { + assert reqId != null; assert initNodeId != null; - try { - GridChangeGlobalStateMessageResponse actResp = new GridChangeGlobalStateMessageResponse(requestId, ex); + GridChangeGlobalStateMessageResponse res = new GridChangeGlobalStateMessageResponse(reqId, ex); + try { if (log.isDebugEnabled()) log.debug("Sending global state change response [nodeId=" + ctx.localNodeId() + - ", topVer=" + ctx.discovery().topologyVersionEx() + ", response=" + actResp + "]"); + ", topVer=" + ctx.discovery().topologyVersionEx() + ", res=" + res + "]"); if (ctx.localNodeId().equals(initNodeId)) - processChangeGlobalStateResponse(ctx.localNodeId(), actResp); + processChangeGlobalStateResponse(ctx.localNodeId(), res); else - sharedCtx.io().send(initNodeId, actResp, SYSTEM_POOL); + sharedCtx.io().send(initNodeId, res, SYSTEM_POOL); + } + catch (ClusterTopologyCheckedException e) { + if (log.isDebugEnabled()) { + log.debug("Failed to send change global state response, node left [node=" + initNodeId + + ", res=" + res + ']'); + } } catch (IgniteCheckedException e) { - log.error("Fail send change global state response to " + initNodeId, e); + U.error(log, "Failed to send change global state response [node=" + initNodeId + ", res=" + res + ']', e); } } /** + * @param nodeId Node ID. * @param msg Message. */ private void processChangeGlobalStateResponse(final UUID nodeId, final GridChangeGlobalStateMessageResponse msg) { assert nodeId != null; assert msg != null; - if (log.isDebugEnabled()) + if (log.isDebugEnabled()) { log.debug("Received activation response [requestId=" + msg.getRequestId() + ", nodeId=" + nodeId + "]"); - - ClusterNode node = ctx.discovery().node(nodeId); - - if (node == null) { - U.warn(log, "Received activation response from unknown node (will ignore) [requestId=" + - msg.getRequestId() + ']'); - - return; } UUID requestId = msg.getRequestId(); - final GridChangeGlobalStateFuture fut = cgsLocFut.get(); - - if (fut != null && !fut.isDone() && requestId.equals(fut.requestId)) { - fut.initFut.listen(new CI1>() { - @Override public void apply(IgniteInternalFuture f) { - fut.onResponse(nodeId, msg); - } - }); + final GridChangeGlobalStateFuture fut = stateChangeFut.get(); + + if (fut != null && requestId.equals(fut.requestId)) { + if (fut.initFut.isDone()) + fut.onResponse(nodeId, msg); + else { + fut.initFut.listen(new CI1>() { + @Override public void apply(IgniteInternalFuture f) { + // initFut is completed from discovery thread, process response from other thread. + ctx.getSystemExecutorService().execute(new Runnable() { + @Override public void run() { + fut.onResponse(nodeId, msg); + } + }); + } + }); + } } } - - /** * @param activate Activate. + * @return Activate flag string. */ - private String prettyStr(boolean activate) { + private static String prettyStr(boolean activate) { return activate ? "activate" : "deactivate"; } @@ -993,7 +731,9 @@ private static class GridChangeGlobalStateFuture extends GridFutureAdapter private final IgniteLogger log; /** - * + * @param requestId State change request ID. + * @param activate New cluster state. + * @param ctx Context. */ GridChangeGlobalStateFuture(UUID requestId, boolean activate, GridKernalContext ctx) { this.requestId = requestId; @@ -1006,7 +746,7 @@ private static class GridChangeGlobalStateFuture extends GridFutureAdapter /** * @param event Event. */ - public void onDiscoveryEvent(DiscoveryEvent event) { + void onNodeLeft(DiscoveryEvent event) { assert event != null; if (isDone()) @@ -1024,29 +764,26 @@ public void onDiscoveryEvent(DiscoveryEvent event) { } /** - * + * @param nodesIds Node IDs. + * @param topVer Current topology version. */ - public void setRemaining(AffinityTopologyVersion topVer) { - Collection nodes = ctx.discovery().nodes(topVer); - - List ids = new ArrayList<>(nodes.size()); - - for (ClusterNode n : nodes) - ids.add(n.id()); - - if (log.isDebugEnabled()) - log.debug("Setup remaining node [id=" + ctx.localNodeId() + ", client=" + - ctx.clientNode() + ", topVer=" + ctx.discovery().topologyVersionEx() + - ", nodes=" + Arrays.toString(ids.toArray()) + "]"); + void setRemaining(Set nodesIds, AffinityTopologyVersion topVer) { + if (log.isDebugEnabled()) { + log.debug("Setup remaining node [id=" + ctx.localNodeId() + + ", client=" + ctx.clientNode() + + ", topVer=" + topVer + + ", nodes=" + nodesIds + "]"); + } synchronized (mux) { - remaining.addAll(ids); + remaining.addAll(nodesIds); } initFut.onDone(); } /** + * @param nodeId Sender node ID. * @param msg Activation message response. */ public void onResponse(UUID nodeId, GridChangeGlobalStateMessageResponse msg) { @@ -1072,7 +809,7 @@ public void onResponse(UUID nodeId, GridChangeGlobalStateMessageResponse msg) { * */ private void onAllReceived() { - Throwable e = new Throwable(); + IgniteCheckedException e = new IgniteCheckedException(); boolean fail = false; @@ -1094,9 +831,13 @@ private void onAllReceived() { /** {@inheritDoc} */ @Override public boolean onDone(@Nullable Void res, @Nullable Throwable err) { - ctx.state().cgsLocFut.set(null); + if (super.onDone(res, err)) { + ctx.state().stateChangeFut.compareAndSet(this, null); + + return true; + } - return super.onDone(res, err); + return false; } /** {@inheritDoc} */ @@ -1106,111 +847,66 @@ private void onAllReceived() { } /** - * * */ - private static class ChangeGlobalStateContext { - /** Request id. */ - private final UUID requestId; - - /** Initiating node id. */ - private final UUID initiatingNodeId; - - /** Batch requests. */ - private final DynamicCacheChangeBatch batch; + private static class ClientChangeGlobalStateComputeRequest implements IgniteRunnable { + /** */ + private static final long serialVersionUID = 0L; - /** Activate. */ + /** */ private final boolean activate; - /** Topology version. */ - private AffinityTopologyVersion topVer; - - /** Fail. */ - private boolean fail; - - /** Async activate future. */ - private IgniteInternalFuture asyncActivateFut; + /** Ignite. */ + @IgniteInstanceResource + private Ignite ignite; /** - * + * @param activate New cluster state. */ - ChangeGlobalStateContext( - UUID requestId, - UUID initiatingNodeId, - DynamicCacheChangeBatch batch, - boolean activate - ) { - this.requestId = requestId; - this.batch = batch; + private ClientChangeGlobalStateComputeRequest(boolean activate) { this.activate = activate; - this.initiatingNodeId = initiatingNodeId; - } - - /** - * @param topVer Topology version. - */ - public void topologyVersion(AffinityTopologyVersion topVer) { - this.topVer = topVer; - } - - /** - * - */ - private void setFail() { - fail = true; - } - - /** - * - */ - private boolean isFail() { - return fail; - } - - /** - * - */ - public IgniteInternalFuture getAsyncActivateFut() { - return asyncActivateFut; - } - - /** - * @param asyncActivateFut Async activate future. - */ - public void setAsyncActivateFut(IgniteInternalFuture asyncActivateFut) { - this.asyncActivateFut = asyncActivateFut; } /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(ChangeGlobalStateContext.class, this); + @Override public void run() { + ignite.active(activate); } } /** * */ - private static class ClientChangeGlobalStateComputeRequest implements IgniteRunnable { + class TransitionOnJoinWaitFuture extends GridFutureAdapter { /** */ - private static final long serialVersionUID = 0L; - - /** Activation. */ - private final boolean activation; + private DiscoveryDataClusterState transitionState; - /** Ignite. */ - @IgniteInstanceResource - private Ignite ignite; + /** */ + private final Set transitionNodes; /** - * + * @param state Current state. + * @param discoCache Discovery data cache. */ - private ClientChangeGlobalStateComputeRequest(boolean activation) { - this.activation = activation; + TransitionOnJoinWaitFuture(DiscoveryDataClusterState state, DiscoCache discoCache) { + assert state.transition() : state; + + transitionNodes = U.newHashSet(state.transitionNodes().size()); + + for (UUID nodeId : state.transitionNodes()) { + if (discoCache.node(nodeId) != null) + transitionNodes.add(nodeId); + } } /** {@inheritDoc} */ - @Override public void run() { - ignite.active(activation); + @Override public boolean onDone(@Nullable Boolean res, @Nullable Throwable err) { + if (super.onDone(res, err)) { + joinFut = null; + + return true; + } + + return false; } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IgniteChangeGlobalStateSupport.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IgniteChangeGlobalStateSupport.java index 3dd9911e01617..5d77f57bdedcf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IgniteChangeGlobalStateSupport.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IgniteChangeGlobalStateSupport.java @@ -36,7 +36,6 @@ public interface IgniteChangeGlobalStateSupport { * Called when cluster performing deactivation. * * @param kctx Kernal context. - * @throws IgniteCheckedException If failed. */ - public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException; + public void onDeActivate(GridKernalContext kctx); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java index 52cc9e9eea742..4399fe2f4c95a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java @@ -175,8 +175,8 @@ public DataStructuresProcessor(GridKernalContext ctx) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public void onKernalStart() throws IgniteCheckedException { - if (ctx.config().isDaemon() || !ctx.state().active()) + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { + if (ctx.config().isDaemon() || !active) return; onKernalStart0(); @@ -278,7 +278,7 @@ public void restoreStructuresState(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext ctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext ctx) { if (log.isDebugEnabled()) log.debug("DeActivate data structure processor [nodeId=" + ctx.localNodeId() + " topVer=" + ctx.discovery().topologyVersionEx() + " ]"); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicLongImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicLongImpl.java index c54f801629649..0bc0c63b5e72d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicLongImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicLongImpl.java @@ -368,7 +368,7 @@ private IllegalStateException removedError() { } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext kctx) { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicReferenceImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicReferenceImpl.java index 64b68e305a1f5..42f16f2eaa34a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicReferenceImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicReferenceImpl.java @@ -299,7 +299,7 @@ public T compareAndSetAndGet(final T newVal, final T expVal) { } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext kctx) { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java index 47fa49e0af38d..019de3c464652 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicSequenceImpl.java @@ -451,7 +451,7 @@ private Callable internalUpdate(final long l, final boolean updated) { /** {@inheritDoc} */ @Override public void onDeActivate(GridKernalContext kctx) { - + // No-op. } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicStampedImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicStampedImpl.java index ac171a68ad85d..ed7a225c2bc3c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicStampedImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheAtomicStampedImpl.java @@ -343,7 +343,7 @@ private void checkRemoved() throws IllegalStateException { } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext kctx) { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheCountDownLatchImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheCountDownLatchImpl.java index 585cb20d629ec..7f331c339a3ab 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheCountDownLatchImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheCountDownLatchImpl.java @@ -340,7 +340,7 @@ private void initializeLatch() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext kctx) { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java index 8d3a77068ecbf..b7986703e6411 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheLockImpl.java @@ -1477,8 +1477,8 @@ private void initializeReentrantLock() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { - + @Override public void onDeActivate(GridKernalContext kctx) { + // No-op. } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java index 2f6abb6f2e8fc..c567ac4f08958 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java @@ -37,7 +37,6 @@ import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteQueue; -import org.apache.ignite.cache.affinity.AffinityKeyMapped; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.util.tostring.GridToStringExclude; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java index c76aec44549f4..4abefc970d9ff 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSemaphoreImpl.java @@ -968,7 +968,7 @@ private void initializeSemaphore() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext kctx) { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java index e336474aa37e4..c27770f8dda85 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java @@ -35,7 +35,6 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.IgniteSet; -import org.apache.ignite.cache.affinity.AffinityKeyMapped; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheIteratorConverter; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java index 871275670f6c1..7eb61d10366b3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java @@ -101,8 +101,6 @@ import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_DELETED; import static org.apache.ignite.events.EventType.EVT_IGFS_FILE_OPENED_READ; import static org.apache.ignite.events.EventType.EVT_IGFS_META_UPDATED; -import static org.apache.ignite.igfs.IgfsMode.DUAL_ASYNC; -import static org.apache.ignite.igfs.IgfsMode.DUAL_SYNC; import static org.apache.ignite.igfs.IgfsMode.PRIMARY; import static org.apache.ignite.igfs.IgfsMode.PROXY; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java index 3c2f64d70aa3b..244820ffef478 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java @@ -177,7 +177,7 @@ public IgfsProcessor(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void onKernalStart() throws IgniteCheckedException { + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { if (ctx.config().isDaemon()) return; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index 23ad63d20155a..ce6c9fecb1266 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -512,10 +512,8 @@ private void onSchemaFinishDiscovery(SchemaFinishDiscoveryMessage msg) { cacheData.queryEntities(cacheDesc.schema().entities()); - CacheGroupDescriptor grpDesc = ctx.cache().cacheDescriptors().get(cacheData.config().getName()).groupDescriptor(); - try { - ctx.cache().context().pageStore().storeCacheData(grpDesc, cacheData); + ctx.cache().context().pageStore().storeCacheData(cacheData); } catch (IgniteCheckedException e) { throw new IllegalStateException("Failed to persist cache data: " + cacheData.config().getName(), e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java index 716adf74bd05a..f5281844e2b28 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java @@ -503,7 +503,7 @@ private boolean notStartOnClient() { } /** {@inheritDoc} */ - @Override public void onKernalStart() throws IgniteCheckedException { + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { if (isRestEnabled()) { for (GridRestProtocol proto : protos) proto.onKernalStart(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java index 909b52451f70a..6236026b4e39a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java @@ -64,7 +64,7 @@ public GridChangeStateCommandHandler(GridKernalContext ctx) { try { if (req.command().equals(CLUSTER_CURRENT_STATE)) { - Boolean currentState = ctx.state().active(); + Boolean currentState = ctx.state().publicApiActiveState(); res.setResponse(currentState); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java index 12be63b283bb0..2eeee1bc9bbd9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java @@ -211,8 +211,8 @@ CU.UTILITY_CACHE_NAME, new ServiceEntriesListener(), null, null /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public void onKernalStart() throws IgniteCheckedException { - if (ctx.isDaemon() || !ctx.state().active()) + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { + if (ctx.isDaemon() || !active) return; onKernalStart0(); @@ -363,7 +363,7 @@ private void onKernalStart0() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { + @Override public void onDeActivate(GridKernalContext kctx) { if (log.isDebugEnabled()) log.debug("DeActivate service processor [nodeId=" + ctx.localNodeId() + " topVer=" + ctx.discovery().topologyVersionEx() + " ]"); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java index 7ac7b6405c2dc..d0b88d80b5e24 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java @@ -153,7 +153,7 @@ public GridTaskProcessor(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void onKernalStart() throws IgniteCheckedException { + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { tasksMetaCache = ctx.security().enabled() && !ctx.isDaemon() ? ctx.cache().utilityCache() : null; diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java index d5bacdb1eeb83..5dbfe6e9d4b8d 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java @@ -582,7 +582,8 @@ else if (addrs.isEmpty()) { * @param addr Address. * @return Socket, connect response and client acknowledge support flag. */ - @Nullable private T3 sendJoinRequest(boolean recon, InetSocketAddress addr) { + @Nullable private T3 sendJoinRequest(boolean recon, + InetSocketAddress addr) { assert addr != null; if (log.isDebugEnabled()) @@ -603,6 +604,8 @@ else if (addrs.isEmpty()) { IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(spi, true); + DiscoveryDataPacket discoveryData = null; + while (true) { boolean openSock = false; @@ -645,9 +648,10 @@ else if (addrs.isEmpty()) { marshalCredentials(node); } - msg = new TcpDiscoveryJoinRequestMessage( - node, - spi.collectExchangeData(new DiscoveryDataPacket(getLocalNodeId()))); + if (discoveryData == null) + discoveryData = spi.collectExchangeData(new DiscoveryDataPacket(getLocalNodeId())); + + msg = new TcpDiscoveryJoinRequestMessage(node, discoveryData); } else msg = new TcpDiscoveryClientReconnectMessage(getLocalNodeId(), rmtNodeId, lastMsgId); diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index 03afff5f4177b..c2d9b7e481c51 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -846,8 +846,10 @@ private void joinTopology() throws IgniteSpiException { // Marshal credentials for backward compatibility and security. marshalCredentials(locNode, locCred); + DiscoveryDataPacket discoveryData = spi.collectExchangeData(new DiscoveryDataPacket(getLocalNodeId())); + while (true) { - if (!sendJoinRequestMessage()) { + if (!sendJoinRequestMessage(discoveryData)) { if (log.isDebugEnabled()) log.debug("Join request message has not been sent (local node is the first in the topology)."); @@ -973,13 +975,13 @@ private void localAuthentication(SecurityCredentials locCred){ * Address is provided by {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} and message is * sent to first node connection succeeded to. * + * @param discoveryData Discovery data. * @return {@code true} if send succeeded. * @throws IgniteSpiException If any error occurs. */ @SuppressWarnings({"BusyWait"}) - private boolean sendJoinRequestMessage() throws IgniteSpiException { - TcpDiscoveryAbstractMessage joinReq = new TcpDiscoveryJoinRequestMessage(locNode, - spi.collectExchangeData(new DiscoveryDataPacket(getLocalNodeId()))); + private boolean sendJoinRequestMessage(DiscoveryDataPacket discoveryData) throws IgniteSpiException { + TcpDiscoveryAbstractMessage joinReq = new TcpDiscoveryJoinRequestMessage(locNode, discoveryData); // Time when it has been detected, that addresses from IP finder do not respond. long noResStart = 0; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java index 98d2553a2c282..ab6168739ab47 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java @@ -176,6 +176,16 @@ public void waitForBlocked(Class cls, String nodeName) throws InterruptedExce } } + /** + * @throws InterruptedException If interrupted. + */ + public void waitForBlocked() throws InterruptedException { + synchronized (this) { + while (blockedMsgs.isEmpty()) + wait(); + } + } + /** * @throws InterruptedException If interrupted. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/GridManagerLocalMessageListenerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/GridManagerLocalMessageListenerSelfTest.java index 5e85b62b26b75..b88eef9e417d1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/managers/GridManagerLocalMessageListenerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/GridManagerLocalMessageListenerSelfTest.java @@ -128,7 +128,7 @@ public void testAddLocalMessageListener() throws Exception { mgr.start(); - mgr.onKernalStart(); + mgr.onKernalStart(true); assertTrue(mgr.enabled()); } @@ -143,7 +143,7 @@ public void testRemoveLocalMessageListener() throws Exception { assertTrue(mgr.enabled()); - mgr.onKernalStart(); + mgr.onKernalStart(true); mgr.onKernalStop(false); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteActiveClusterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteActiveClusterTest.java deleted file mode 100644 index cf6876766f013..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteActiveClusterTest.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache; - -import java.util.concurrent.Callable; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; - -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; -import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; - -/** - * - */ -public class IgniteActiveClusterTest extends GridCommonAbstractTest { - /** */ - private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - - /** */ - private boolean client; - - /** */ - private boolean active = true; - - /** */ - private CacheConfiguration ccfg; - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); - - cfg.setClientMode(client); - - cfg.setActiveOnStart(active); - - if (ccfg != null) { - cfg.setCacheConfiguration(ccfg); - - ccfg = null; - } - - return cfg; - } - - /** - * @throws Exception If failed. - */ - public void testActivate() throws Exception { - active = false; - - for (int i = 0; i < 3; i++) { - ccfg = cacheConfiguration(DEFAULT_CACHE_NAME); - - startGrid(i); - } - - ignite(0).active(true); - - startGrid(3); - - for (int i = 0; i < 4; i++) { - IgniteCache cache = ignite(i).cache(DEFAULT_CACHE_NAME); - - for (int j = 0; j < 10; j++) { - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - - Integer key = rnd.nextInt(1000); - - cache.put(key, j); - - assertEquals((Integer)j, cache.get(key)); - } - } - } - - /** - * @throws Exception If failed. - */ - public void testJoinAndActivate() throws Exception { - for (int iter = 0; iter < 3; iter++) { - log.info("Iteration: " + iter); - - active = false; - - for (int i = 0; i < 3; i++) { - ccfg = cacheConfiguration(DEFAULT_CACHE_NAME); - - startGrid(i); - } - - final int START_NODES = 3; - - final CyclicBarrier b = new CyclicBarrier(START_NODES + 1); - - IgniteInternalFuture fut1 = GridTestUtils.runAsync(new Callable() { - @Override public Void call() throws Exception { - b.await(); - - Thread.sleep(ThreadLocalRandom.current().nextLong(100) + 1); - - ignite(0).active(true); - - return null; - } - }); - - final AtomicInteger nodeIdx = new AtomicInteger(3); - - IgniteInternalFuture fut2 = GridTestUtils.runMultiThreadedAsync(new Callable() { - @Override public Void call() throws Exception { - int idx = nodeIdx.getAndIncrement(); - - b.await(); - - startGrid(idx); - - return null; - } - }, START_NODES, "start-node"); - - fut1.get(); - fut2.get(); - - for (int i = 0; i < 6; i++) { - IgniteCache cache = ignite(i).cache(DEFAULT_CACHE_NAME); - - for (int j = 0; j < 10; j++) { - ThreadLocalRandom rnd = ThreadLocalRandom.current(); - - Integer key = rnd.nextInt(1000); - - cache.put(key, j); - - assertEquals((Integer)j, cache.get(key)); - } - } - - stopAllGrids(); - } - } - - /** - * @param name Cache name. - * @return Cache configuration. - */ - private CacheConfiguration cacheConfiguration(String name) { - CacheConfiguration ccfg = new CacheConfiguration(name); - - ccfg.setWriteSynchronizationMode(FULL_SYNC); - ccfg.setAtomicityMode(TRANSACTIONAL); - ccfg.setBackups(3); - - return ccfg; - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java new file mode 100644 index 0000000000000..8a604be6b5c8f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java @@ -0,0 +1,1284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.MemoryConfiguration; +import org.apache.ignite.configuration.PersistentStoreConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteClientReconnectAbstractTest; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgniteKernal; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * + */ +public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + static final String CACHE_NAME_PREFIX = "cache-"; + + /** */ + boolean client; + + /** */ + private boolean active = true; + + /** */ + CacheConfiguration[] ccfgs; + + /** */ + private boolean testSpi; + + /** */ + private boolean testDiscoSpi; + + /** */ + private boolean testReconnectSpi; + + /** */ + private Class[] testSpiRecord; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + if (testReconnectSpi) { + TcpDiscoverySpi spi = new IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi(); + + cfg.setDiscoverySpi(spi); + + spi.setJoinTimeout(2 * 60_000); + } + else if (testDiscoSpi) + cfg.setDiscoverySpi(new TestTcpDiscoverySpi()); + + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER); + + cfg.setConsistentId(igniteInstanceName); + + cfg.setClientMode(client); + + cfg.setActiveOnStart(active); + + if (ccfgs != null) { + cfg.setCacheConfiguration(ccfgs); + + ccfgs = null; + } + + MemoryConfiguration memCfg = new MemoryConfiguration(); + memCfg.setPageSize(1024); + memCfg.setDefaultMemoryPolicySize(10 * 1024 * 1024); + + cfg.setMemoryConfiguration(memCfg); + + if (persistenceEnabled()) { + PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration(); + + pCfg.setWalMode(WALMode.LOG_ONLY); + + cfg.setPersistentStoreConfiguration(pCfg); + } + + if (testSpi) { + TestRecordingCommunicationSpi spi = new TestRecordingCommunicationSpi(); + + if (testSpiRecord != null) + spi.record(testSpiRecord); + + cfg.setCommunicationSpi(spi); + } + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + super.afterTest(); + } + + /** + * @return {@code True} if test with persistence. + */ + protected boolean persistenceEnabled() { + return false; + } + + /** + * @throws Exception If failed. + */ + public void testActivateSimple_SingleNode() throws Exception { + activateSimple(1, 0, 0); + } + + /** + * @throws Exception If failed. + */ + public void testActivateSimple_5_Servers() throws Exception { + activateSimple(5, 0, 0); + } + + /** + * @throws Exception If failed. + */ + public void testActivateSimple_5_Servers2() throws Exception { + activateSimple(5, 0, 4); + } + + /** + * @throws Exception If failed. + */ + public void testActivateSimple_5_Servers_5_Clients() throws Exception { + activateSimple(5, 4, 0); + } + + /** + * @throws Exception If failed. + */ + public void testActivateSimple_5_Servers_5_Clients_FromClient() throws Exception { + activateSimple(5, 4, 6); + } + + /** + * @param srvs Number of servers. + * @param clients Number of clients. + * @param activateFrom Index of node stating activation. + * @throws Exception If failed. + */ + private void activateSimple(int srvs, int clients, int activateFrom) throws Exception { + active = false; + + final int CACHES = 2; + + for (int i = 0; i < srvs + clients; i++) { + client = i >= srvs; + + ccfgs = cacheConfigurations1(); + + startGrid(i); + + checkNoCaches(i); + } + + for (int i = 0; i < srvs + clients; i++) + assertFalse(ignite(i).active()); + + ignite(activateFrom).active(false); // Should be no-op. + + ignite(activateFrom).active(true); + + for (int i = 0; i < srvs + clients; i++) + assertTrue(ignite(i).active()); + + for (int i = 0; i < srvs + clients; i++) { + for (int c = 0; c < 2; c++) + checkCache(ignite(i), CACHE_NAME_PREFIX + c, true); + + checkCache(ignite(i), CU.UTILITY_CACHE_NAME, true); + } + + checkCaches(srvs + clients, CACHES); + + client = false; + + startGrid(srvs + clients); + + for (int c = 0; c < 2; c++) + checkCache(ignite(srvs + clients), CACHE_NAME_PREFIX + c, true); + + checkCaches(srvs + clients + 1, CACHES); + + client = true; + + startGrid(srvs + clients + 1); + + for (int c = 0; c < 2; c++) + checkCache(ignite(srvs + clients + 1), CACHE_NAME_PREFIX + c, false); + + checkCaches(srvs + clients + 2, CACHES); + } + + /** + * @param nodes Number of nodes. + * @param caches Number of caches. + */ + final void checkCaches(int nodes, int caches) { + for (int i = 0; i < nodes; i++) { + for (int c = 0; c < caches; c++) { + IgniteCache cache = ignite(i).cache(CACHE_NAME_PREFIX + c); + + for (int j = 0; j < 10; j++) { + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + Integer key = rnd.nextInt(1000); + + cache.put(key, j); + + assertEquals((Integer)j, cache.get(key)); + } + } + } + } + + /** + * @throws Exception If failed. + */ + public void testJoinWhileActivate1_Server() throws Exception { + joinWhileActivate1(false, false); + } + + /** + * @throws Exception If failed. + */ + public void testJoinWhileActivate1_WithCache_Server() throws Exception { + joinWhileActivate1(false, true); + } + + /** + * @throws Exception If failed. + */ + public void testJoinWhileActivate1_Client() throws Exception { + joinWhileActivate1(true, false); + } + + /** + * @param startClient If {@code true} joins client node, otherwise server. + * @param withNewCache If {@code true} joining node has new cache in configuration. + * @throws Exception If failed. + */ + private void joinWhileActivate1(final boolean startClient, final boolean withNewCache) throws Exception { + IgniteInternalFuture activeFut = startNodesAndBlockStatusChange(2, 0, 0, false); + + IgniteInternalFuture startFut = GridTestUtils.runAsync(new Callable() { + @Override public Void call() throws Exception { + client = startClient; + + ccfgs = withNewCache ? cacheConfigurations2() : cacheConfigurations1(); + + startGrid(2); + + return null; + } + }); + + TestRecordingCommunicationSpi spi1 = TestRecordingCommunicationSpi.spi(ignite(1)); + + spi1.stopBlock(); + + activeFut.get(); + startFut.get(); + + for (int c = 0; c < 2; c++) + checkCache(ignite(2), CACHE_NAME_PREFIX + c, true); + + if (withNewCache) { + for (int i = 0; i < 3; i++) { + for (int c = 0; c < 4; c++) + checkCache(ignite(i), CACHE_NAME_PREFIX + c, true); + } + } + + awaitPartitionMapExchange(); + + checkCaches(3, withNewCache ? 4 : 2); + + client = false; + + startGrid(3); + + checkCaches(4, withNewCache ? 4 : 2); + + client = true; + + startGrid(4); + + checkCaches(5, withNewCache ? 4 : 2); + } + + /** + * @param srvs Number of servers. + * @param clients Number of clients. + * @param stateChangeFrom Index of node initiating changes. + * @param initiallyActive If {@code true} start cluster in active state (otherwise in inactive). + * @param blockMsgNodes Nodes whcis block exchange messages. + * @return State change future. + * @throws Exception If failed. + */ + private IgniteInternalFuture startNodesAndBlockStatusChange(int srvs, + int clients, + final int stateChangeFrom, + final boolean initiallyActive, + int... blockMsgNodes) throws Exception { + active = initiallyActive; + testSpi = true; + + startWithCaches1(srvs, clients); + + if (initiallyActive && persistenceEnabled()) + ignite(0).active(true); + + if (blockMsgNodes.length == 0) + blockMsgNodes = new int[]{1}; + + final AffinityTopologyVersion STATE_CHANGE_TOP_VER = new AffinityTopologyVersion(srvs + clients, 1); + + List spis = new ArrayList<>(); + + for (int idx : blockMsgNodes) { + TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(ignite(idx)); + + spis.add(spi); + + blockExchangeSingleMessage(spi, STATE_CHANGE_TOP_VER); + } + + IgniteInternalFuture stateChangeFut = GridTestUtils.runAsync(new Runnable() { + @Override public void run() { + ignite(stateChangeFrom).active(!initiallyActive); + } + }); + + for (TestRecordingCommunicationSpi spi : spis) + spi.waitForBlocked(); + + U.sleep(500); + + assertFalse(stateChangeFut.isDone()); + + return stateChangeFut; + } + + /** + * @param spi SPI. + * @param topVer Exchange topology version. + */ + private void blockExchangeSingleMessage(TestRecordingCommunicationSpi spi, final AffinityTopologyVersion topVer) { + spi.blockMessages(new IgniteBiPredicate() { + @Override public boolean apply(ClusterNode clusterNode, Message msg) { + if (msg instanceof GridDhtPartitionsSingleMessage) { + GridDhtPartitionsSingleMessage pMsg = (GridDhtPartitionsSingleMessage)msg; + + if (pMsg.exchangeId() != null && pMsg.exchangeId().topologyVersion().equals(topVer)) + return true; + } + + return false; + } + }); + } + + /** + * @throws Exception If failed. + */ + public void testJoinWhileDeactivate1_Server() throws Exception { + joinWhileDeactivate1(false, false); + } + + /** + * @throws Exception If failed. + */ + public void testJoinWhileDeactivate1_WithCache_Server() throws Exception { + joinWhileDeactivate1(false, true); + } + + /** + * @throws Exception If failed. + */ + public void testJoinWhileDeactivate1_Client() throws Exception { + joinWhileDeactivate1(true, false); + } + + /** + * @param startClient If {@code true} joins client node, otherwise server. + * @param withNewCache If {@code true} joining node has new cache in configuration. + * @throws Exception If failed. + */ + private void joinWhileDeactivate1(final boolean startClient, final boolean withNewCache) throws Exception { + IgniteInternalFuture activeFut = startNodesAndBlockStatusChange(2, 0, 0, true); + + IgniteInternalFuture startFut = GridTestUtils.runAsync(new Callable() { + @Override public Void call() throws Exception { + client = startClient; + + ccfgs = withNewCache ? cacheConfigurations2() : cacheConfigurations1(); + + startGrid(2); + + return null; + } + }); + + TestRecordingCommunicationSpi spi1 = TestRecordingCommunicationSpi.spi(ignite(1)); + + spi1.stopBlock(); + + activeFut.get(); + startFut.get(); + + checkNoCaches(3); + + ignite(2).active(true); + + for (int c = 0; c < 2; c++) + checkCache(ignite(2), CACHE_NAME_PREFIX + c, true); + + if (withNewCache) { + for (int i = 0; i < 3; i++) { + for (int c = 0; c < 4; c++) + checkCache(ignite(i), CACHE_NAME_PREFIX + c, true); + } + } + + awaitPartitionMapExchange(); + + checkCaches(3, withNewCache ? 4 : 2); + + client = false; + + startGrid(3); + + checkCaches(4, withNewCache ? 4 : 2); + + client = true; + + startGrid(4); + + checkCaches(5, withNewCache ? 4 : 2); + } + + /** + * @throws Exception If failed. + */ + public void testConcurrentJoinAndActivate() throws Exception { + for (int iter = 0; iter < 3; iter++) { + log.info("Iteration: " + iter); + + active = false; + + for (int i = 0; i < 3; i++) { + ccfgs = cacheConfigurations1(); + + startGrid(i); + } + + final int START_NODES = 3; + + final CyclicBarrier b = new CyclicBarrier(START_NODES + 1); + + IgniteInternalFuture fut1 = GridTestUtils.runAsync(new Callable() { + @Override public Void call() throws Exception { + b.await(); + + Thread.sleep(ThreadLocalRandom.current().nextLong(100) + 1); + + ignite(0).active(true); + + return null; + } + }); + + final AtomicInteger nodeIdx = new AtomicInteger(3); + + IgniteInternalFuture fut2 = GridTestUtils.runMultiThreadedAsync(new Callable() { + @Override public Void call() throws Exception { + int idx = nodeIdx.getAndIncrement(); + + b.await(); + + startGrid(idx); + + return null; + } + }, START_NODES, "start-node"); + + fut1.get(); + fut2.get(); + + checkCaches(6, 2); + + afterTest(); + } + } + + /** + * @throws Exception If failed. + */ + public void testDeactivateSimple_SingleNode() throws Exception { + deactivateSimple(1, 0, 0); + } + + /** + * @throws Exception If failed. + */ + public void testDeactivateSimple_5_Servers() throws Exception { + deactivateSimple(5, 0, 0); + } + + /** + * @throws Exception If failed. + */ + public void testDeactivateSimple_5_Servers2() throws Exception { + deactivateSimple(5, 0, 4); + } + + /** + * @throws Exception If failed. + */ + public void testDeactivateSimple_5_Servers_5_Clients() throws Exception { + deactivateSimple(5, 4, 0); + } + + /** + * @throws Exception If failed. + */ + public void testDeactivateSimple_5_Servers_5_Clients_FromClient() throws Exception { + deactivateSimple(5, 4, 6); + } + + /** + * @param srvs Number of servers. + * @param clients Number of clients. + * @param deactivateFrom Index of node stating deactivation. + * @throws Exception If failed. + */ + private void deactivateSimple(int srvs, int clients, int deactivateFrom) throws Exception { + active = true; + + final int CACHES = 2; + + for (int i = 0; i < srvs + clients; i++) { + client = i >= srvs; + + ccfgs = cacheConfigurations1(); + + startGrid(i); + } + + if (persistenceEnabled()) + ignite(deactivateFrom).active(true); + + ignite(deactivateFrom).active(true); // Should be no-op. + + checkCaches(srvs + clients, CACHES); + + for (int i = 0; i < srvs + clients; i++) + assertTrue(ignite(i).active()); + + ignite(deactivateFrom).active(false); + + for (int i = 0; i < srvs + clients; i++) + assertFalse(ignite(i).active()); + + checkNoCaches(srvs + clients); + + client = false; + + startGrid(srvs + clients); + + checkNoCaches(srvs + clients + 1); + + client = true; + + startGrid(srvs + clients + 1); + + checkNoCaches(srvs + clients + 2); + + for (int i = 0; i < srvs + clients + 2; i++) + assertFalse(ignite(i).active()); + + ignite(deactivateFrom).active(true); + + for (int i = 0; i < srvs + clients + 2; i++) { + assertTrue(ignite(i).active()); + + checkCache(ignite(i), CU.UTILITY_CACHE_NAME, true); + } + + for (int i = 0; i < srvs; i++) { + for (int c = 0; c < 2; c++) + checkCache(ignite(i), CACHE_NAME_PREFIX + c, true); + } + + checkCaches1(srvs + clients + 2); + } + + /** + * @param srvs Number of servers. + * @param clients Number of clients. + * @throws Exception If failed. + */ + private void startWithCaches1(int srvs, int clients) throws Exception { + for (int i = 0; i < srvs + clients; i++) { + ccfgs = cacheConfigurations1(); + + client = i >= srvs; + + startGrid(i); + } + } + + /** + * @throws Exception If failed. + */ + public void testClientReconnectClusterActive() throws Exception { + testReconnectSpi = true; + + ccfgs = cacheConfigurations1(); + + final int SRVS = 3; + final int CLIENTS = 3; + + startWithCaches1(SRVS, CLIENTS); + + if (persistenceEnabled()) + ignite(0).active(true); + + Ignite srv = ignite(0); + Ignite client = ignite(SRVS); + + checkCache(client, CU.UTILITY_CACHE_NAME, true); + + checkCaches1(SRVS + CLIENTS); + + IgniteClientReconnectAbstractTest.reconnectClientNode(log, client, srv, null); + + checkCaches1(SRVS + CLIENTS); + + this.client = false; + + startGrid(SRVS + CLIENTS); + + this.client = true; + + startGrid(SRVS + CLIENTS + 1); + + checkCaches1(SRVS + CLIENTS + 2); + } + + /** + * @throws Exception If failed. + */ + public void testClientReconnectClusterInactive() throws Exception { + testReconnectSpi = true; + + active = false; + + final int SRVS = 3; + final int CLIENTS = 3; + + startWithCaches1(SRVS, CLIENTS); + + Ignite srv = ignite(0); + Ignite client = ignite(SRVS); + + checkNoCaches(SRVS + CLIENTS); + + IgniteClientReconnectAbstractTest.reconnectClientNode(log, client, srv, null); + + checkNoCaches(SRVS + CLIENTS); + + ignite(0).active(true); + + checkCache(client, CU.UTILITY_CACHE_NAME, true); + + checkCaches1(SRVS + CLIENTS); + + this.client = false; + + startGrid(SRVS + CLIENTS); + + this.client = true; + + startGrid(SRVS + CLIENTS + 1); + + checkCaches1(SRVS + CLIENTS); + } + + /** + * @throws Exception If failed. + */ + public void testClientReconnectClusterDeactivated() throws Exception { + clientReconnectClusterDeactivated(false); + } + + /** + * @throws Exception If failed. + */ + public void testClientReconnectClusterDeactivateInProgress() throws Exception { + clientReconnectClusterDeactivated(true); + } + + /** + * @param transition If {@code true} client reconnects while cluster state transition is in progress. + * @throws Exception If failed. + */ + private void clientReconnectClusterDeactivated(final boolean transition) throws Exception { + testReconnectSpi = true; + testSpi = transition; + + final int SRVS = 3; + final int CLIENTS = 3; + + startWithCaches1(SRVS, CLIENTS); + + final Ignite srv = ignite(0); + Ignite client = ignite(SRVS); + + if (persistenceEnabled()) + ignite(0).active(true); + + checkCache(client, CU.UTILITY_CACHE_NAME, true); + + checkCaches1(SRVS + CLIENTS); + + final AffinityTopologyVersion STATE_CHANGE_TOP_VER = new AffinityTopologyVersion(SRVS + CLIENTS + 1, 1); + + final TestRecordingCommunicationSpi spi1 = transition ? TestRecordingCommunicationSpi.spi(ignite(1)) : null; + + final AtomicReference stateFut = new AtomicReference<>(); + + IgniteClientReconnectAbstractTest.reconnectClientNode(log, client, srv, new Runnable() { + @Override public void run() { + if (transition) { + blockExchangeSingleMessage(spi1, STATE_CHANGE_TOP_VER); + + stateFut.set(GridTestUtils.runAsync(new Runnable() { + @Override public void run() { + srv.active(false); + } + }, "deactivate")); + + try { + U.sleep(500); + } + catch (Exception e) { + e.printStackTrace(); + } + } + else + srv.active(false); + } + }); + + checkCache(client, CACHE_NAME_PREFIX + 0, false); + + if (transition) { + assertFalse(stateFut.get().isDone()); + + assertFalse(client.active()); + + spi1.waitForBlocked(); + + spi1.stopBlock(); + + stateFut.get().get(); + } + + checkNoCaches(SRVS + CLIENTS); + + ignite(0).active(true); + + checkCache(client, CU.UTILITY_CACHE_NAME, true); + + assertTrue(client.active()); + + checkCaches1(SRVS + CLIENTS); + + checkCache(client, CACHE_NAME_PREFIX + 0, true); + + this.client = false; + + startGrid(SRVS + CLIENTS); + + this.client = true; + + startGrid(SRVS + CLIENTS + 1); + + checkCaches1(SRVS + CLIENTS + 2); + } + + /** + * @throws Exception If failed. + */ + public void testClientReconnectClusterActivated() throws Exception { + clientReconnectClusterActivated(false); + } + + /** + * @throws Exception If failed. + */ + public void testClientReconnectClusterActivateInProgress() throws Exception { + clientReconnectClusterActivated(true); + } + + /** + * @param transition If {@code true} client reconnects while cluster state transition is in progress. + * @throws Exception If failed. + */ + private void clientReconnectClusterActivated(final boolean transition) throws Exception { + testReconnectSpi = true; + testSpi = transition; + + active = false; + + final int SRVS = 3; + final int CLIENTS = 3; + + startWithCaches1(SRVS, CLIENTS); + + final Ignite srv = ignite(0); + Ignite client = ignite(SRVS); + + checkNoCaches(SRVS + CLIENTS); + + final AffinityTopologyVersion STATE_CHANGE_TOP_VER = new AffinityTopologyVersion(SRVS + CLIENTS + 1, 1); + + final TestRecordingCommunicationSpi spi1 = transition ? TestRecordingCommunicationSpi.spi(ignite(1)) : null; + + final AtomicReference stateFut = new AtomicReference<>(); + + IgniteClientReconnectAbstractTest.reconnectClientNode(log, client, srv, new Runnable() { + @Override public void run() { + if (transition) { + blockExchangeSingleMessage(spi1, STATE_CHANGE_TOP_VER); + + stateFut.set(GridTestUtils.runAsync(new Runnable() { + @Override public void run() { + srv.active(true); + } + }, "activate")); + + try { + U.sleep(500); + } + catch (Exception e) { + e.printStackTrace(); + } + } + else + srv.active(true); + } + }); + + checkCache(client, CACHE_NAME_PREFIX + 0, !transition); + + if (transition) { + assertFalse(stateFut.get().isDone()); + + assertFalse(client.active()); + + spi1.waitForBlocked(); + + spi1.stopBlock(); + + stateFut.get().get(); + } + + checkCache(client, CU.UTILITY_CACHE_NAME, true); + + checkCaches1(SRVS + CLIENTS); + + checkCache(client, CACHE_NAME_PREFIX + 0, true); + + this.client = false; + + startGrid(SRVS + CLIENTS); + + this.client = true; + + startGrid(SRVS + CLIENTS + 1); + + checkCaches1(SRVS + CLIENTS + 2); + } + + /** + * @throws Exception If failed. + */ + public void testInactiveTopologyChanges() throws Exception { + testSpi = true; + + testSpiRecord = new Class[]{GridDhtPartitionsSingleMessage.class, GridDhtPartitionsFullMessage.class}; + + active = false; + + final int SRVS = 4; + final int CLIENTS = 4; + + startWithCaches1(SRVS, CLIENTS); + + checkRecordedMessages(false); + + for (int i = 0; i < 2; i++) { + stopGrid(i); + + client = false; + + startGrid(i); + } + + checkRecordedMessages(false); + + for (int i = 0; i < 2; i++) { + stopGrid(SRVS + i); + + client = true; + + startGrid(SRVS + i); + } + + checkRecordedMessages(false); + + ignite(0).active(true); + + checkCaches1(SRVS + CLIENTS); + + checkRecordedMessages(true); + + client = false; + + startGrid(SRVS + CLIENTS); + + client = true; + + startGrid(SRVS + CLIENTS + 1); + + checkRecordedMessages(true); + + checkCaches1(SRVS + CLIENTS + 2); + } + + /** + * @throws Exception If failed. + */ + public void testActivateFailover1() throws Exception { + stateChangeFailover1(true); + } + + /** + * @throws Exception If failed. + */ + public void testDeactivateFailover1() throws Exception { + stateChangeFailover1(false); + } + + /** + * @param activate If {@code true} tests activation, otherwise deactivation. + * @throws Exception If failed. + */ + private void stateChangeFailover1(boolean activate) throws Exception { + // Nodes 1 and 4 do not reply to coordinator. + IgniteInternalFuture fut = startNodesAndBlockStatusChange(4, 4, 3, !activate, 1, 4); + + client = false; + + // Start one more node while transition is in progress. + IgniteInternalFuture startFut = GridTestUtils.runAsync(new Callable() { + @Override public Object call() throws Exception { + startGrid(8); + + return null; + } + }, "start-node"); + + U.sleep(500); + + stopGrid(getTestIgniteInstanceName(1), true, false); + stopGrid(getTestIgniteInstanceName(4), true, false); + + fut.get(); + + startFut.get(); + + client = false; + + startGrid(1); + + client = true; + + startGrid(4); + + if (!activate) { + checkNoCaches(9); + + ignite(0).active(true); + } + + checkCaches1(9); + } + + /** + * @throws Exception If failed. + */ + public void testActivateFailover2() throws Exception { + stateChangeFailover2(true); + } + + /** + * @throws Exception If failed. + */ + public void testDeactivateFailover2() throws Exception { + stateChangeFailover2(false); + } + + /** + * @param activate If {@code true} tests activation, otherwise deactivation. + * @throws Exception If failed. + */ + private void stateChangeFailover2(boolean activate) throws Exception { + // Nodes 1 and 4 do not reply to coordinator. + IgniteInternalFuture fut = startNodesAndBlockStatusChange(4, 4, 3, !activate, 1, 4); + + client = false; + + // Start one more nodes while transition is in progress. + IgniteInternalFuture startFut1 = GridTestUtils.runAsync(new Callable() { + @Override public Object call() throws Exception { + startGrid(8); + + return null; + } + }, "start-node1"); + IgniteInternalFuture startFut2 = GridTestUtils.runAsync(new Callable() { + @Override public Object call() throws Exception { + startGrid(9); + + return null; + } + }, "start-node2"); + + U.sleep(500); + + // Stop coordinator. + stopGrid(0); + + stopGrid(getTestIgniteInstanceName(1), true, false); + stopGrid(getTestIgniteInstanceName(4), true, false); + + fut.get(); + + startFut1.get(); + startFut2.get(); + + client = false; + + startGrid(0); + startGrid(1); + + client = true; + + startGrid(4); + + if (!activate) { + checkNoCaches(10); + + ignite(0).active(true); + } + + checkCaches1(10); + } + + /** + * @throws Exception If failed. + */ + public void testActivateFailover3() throws Exception { + stateChangeFailover3(true); + } + + /** + * @throws Exception If failed. + */ + public void testDeactivateFailover3() throws Exception { + stateChangeFailover3(false); + } + + /** + * @param activate If {@code true} tests activation, otherwise deactivation. + * @throws Exception If failed. + */ + private void stateChangeFailover3(boolean activate) throws Exception { + testDiscoSpi = true; + + startNodesAndBlockStatusChange(4, 0, 0, !activate); + + client = false; + + IgniteInternalFuture startFut1 = GridTestUtils.runAsync(new Callable() { + @Override public Object call() throws Exception { + startGrid(4); + + return null; + } + }, "start-node1"); + + IgniteInternalFuture startFut2 = GridTestUtils.runAsync(new Callable() { + @Override public Object call() throws Exception { + startGrid(5); + + return null; + } + }, "start-node2"); + + U.sleep(1000); + + // Stop all nodes participating in state change and not allow last node to finish exchange. + for (int i = 0; i < 4; i++) + ((TestTcpDiscoverySpi)ignite(i).configuration().getDiscoverySpi()).simulateNodeFailure(); + + for (int i = 0; i < 4; i++) + stopGrid(getTestIgniteInstanceName(i), true, false); + + startFut1.get(); + startFut2.get(); + + assertFalse(ignite(4).active()); + assertFalse(ignite(5).active()); + + ignite(4).active(true); + + for (int i = 0; i < 4; i++) + startGrid(i); + + checkCaches1(6); + } + + /** + * @param exp If {@code true} there should be recorded messages. + */ + private void checkRecordedMessages(boolean exp) { + for (Ignite node : G.allGrids()) { + List recorded = + TestRecordingCommunicationSpi.spi(node).recordedMessages(false); + + if (exp) + assertFalse(F.isEmpty(recorded)); + else + assertTrue(F.isEmpty(recorded)); + } + } + + /** + * @param nodes Expected nodes number. + */ + private void checkCaches1(int nodes) { + checkCaches(nodes, 2); + } + + /** + * @return Cache configurations. + */ + final CacheConfiguration[] cacheConfigurations1() { + CacheConfiguration[] ccfgs = new CacheConfiguration[2]; + + ccfgs[0] = cacheConfiguration(CACHE_NAME_PREFIX + 0, ATOMIC); + ccfgs[1] = cacheConfiguration(CACHE_NAME_PREFIX + 1, TRANSACTIONAL); + + return ccfgs; + } + + /** + * @return Cache configurations. + */ + final CacheConfiguration[] cacheConfigurations2() { + CacheConfiguration[] ccfgs = new CacheConfiguration[4]; + + ccfgs[0] = cacheConfiguration(CACHE_NAME_PREFIX + 0, ATOMIC); + ccfgs[1] = cacheConfiguration(CACHE_NAME_PREFIX + 1, TRANSACTIONAL); + ccfgs[2] = cacheConfiguration(CACHE_NAME_PREFIX + 2, ATOMIC); + ccfgs[3] = cacheConfiguration(CACHE_NAME_PREFIX + 3, TRANSACTIONAL); + + return ccfgs; + } + + /** + * @param name Cache name. + * @param atomicityMode Atomicity mode. + * @return Cache configuration. + */ + protected final CacheConfiguration cacheConfiguration(String name, CacheAtomicityMode atomicityMode) { + CacheConfiguration ccfg = new CacheConfiguration(name); + + ccfg.setWriteSynchronizationMode(FULL_SYNC); + ccfg.setAtomicityMode(atomicityMode); + ccfg.setBackups(1); + + return ccfg; + } + + /** + * @param cacheName Cache name. + * @param node Node. + * @param exp {@code True} if expect that cache is started on node. + */ + void checkCache(Ignite node, String cacheName, boolean exp) { + GridCacheAdapter cache = ((IgniteKernal)node).context().cache().internalCache(cacheName); + + if (exp) + assertNotNull("Cache not found [cache=" + cacheName + ", node=" + node.name() + ']', cache); + else + assertNull("Unexpected cache found [cache=" + cacheName + ", node=" + node.name() + ']', cache); + } + + /** + * @param nodes Number of nodes. + */ + final void checkNoCaches(int nodes) { + for (int i = 0; i < nodes; i++) { + GridCacheProcessor cache = ((IgniteKernal)ignite(i)).context().cache(); + + assertTrue(cache.caches().isEmpty()); + assertTrue(cache.internalCaches().isEmpty()); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java new file mode 100644 index 0000000000000..4a19aa8a94da1 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteException; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.testframework.GridTestUtils; + +/** + * + */ +public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteClusterActivateDeactivateTest { + /** {@inheritDoc} */ + @Override protected boolean persistenceEnabled() { + return true; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + GridTestUtils.deleteDbFiles(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + GridTestUtils.deleteDbFiles(); + } + + /** + * @throws Exception If failed. + */ + public void testActivateCachesRestore_SingleNode() throws Exception { + activateCachesRestore(1, false); + } + + /** + * @throws Exception If failed. + */ + public void testActivateCachesRestore_SingleNode_WithNewCaches() throws Exception { + activateCachesRestore(1, true); + } + + /** + * @throws Exception If failed. + */ + public void testActivateCachesRestore_5_Servers() throws Exception { + activateCachesRestore(5, false); + } + + /** + * @throws Exception If failed. + */ + public void testActivateCachesRestore_5_Servers_WithNewCaches() throws Exception { + activateCachesRestore(5, false); + } + + /** + * @param srvs Number of server nodes. + * @param withNewCaches If {@code true} then after restart has new caches in configuration. + * @throws Exception If failed. + */ + private void activateCachesRestore(int srvs, boolean withNewCaches) throws Exception { + Ignite srv = startGrids(srvs); + + srv.active(true); + + srv.createCaches(Arrays.asList(cacheConfigurations1())); + + Map cacheData = new LinkedHashMap<>(); + + for (int i = 1; i <= 100; i++) { + for (CacheConfiguration ccfg : cacheConfigurations1()) { + srv.cache(ccfg.getName()).put(-i, i); + + cacheData.put(-i, i); + } + } + + stopAllGrids(); + + for (int i = 0; i < srvs; i++) { + if (withNewCaches) + ccfgs = cacheConfigurations2(); + + startGrid(i); + } + + srv = ignite(0); + + checkNoCaches(srvs); + + srv.active(true); + + final int CACHES = withNewCaches ? 4 : 2; + + for (int i = 0; i < srvs; i++) { + for (int c = 0; c < CACHES; c++) + checkCache(ignite(i), CACHE_NAME_PREFIX + c, true); + } + + for (CacheConfiguration ccfg : cacheConfigurations1()) + checkCacheData(cacheData, ccfg.getName()); + + checkCaches(srvs, CACHES); + + int nodes = srvs; + + client = false; + + startGrid(nodes++); + + for (int i = 0; i < nodes; i++) { + for (int c = 0; c < CACHES; c++) + checkCache(ignite(i), CACHE_NAME_PREFIX + c, true); + } + + checkCaches(nodes, CACHES); + + client = true; + + startGrid(nodes++); + + for (int c = 0; c < CACHES; c++) + checkCache(ignite(nodes - 1), CACHE_NAME_PREFIX + c, false); + + checkCaches(nodes, CACHES); + + for (int i = 0; i < nodes; i++) { + for (int c = 0; c < CACHES; c++) + checkCache(ignite(i), CACHE_NAME_PREFIX + c, true); + } + + for (CacheConfiguration ccfg : cacheConfigurations1()) + checkCacheData(cacheData, ccfg.getName()); + } + + /** + * @throws Exception If failed. + */ + public void testActivateCacheRestoreConfigurationConflict() throws Exception { + final int SRVS = 3; + + Ignite srv = startGrids(SRVS); + + srv.active(true); + + CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); + + srv.createCache(ccfg); + + stopAllGrids(); + + ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME + 1); + + ccfg.setGroupName(DEFAULT_CACHE_NAME); + + ccfgs = new CacheConfiguration[]{ccfg}; + + startGrids(SRVS); + + try { + ignite(0).active(true); + + fail(); + } + catch (IgniteException e) { + // Expected error. + } + + for (int i = 0; i < SRVS; i++) + assertFalse(ignite(i).active()); + + checkNoCaches(SRVS); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java index fdf53503495ad..566860de75e01 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java @@ -22,9 +22,6 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.IgniteKernal; -import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.lang.IgniteCallable; import org.apache.ignite.resources.IgniteInstanceResource; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -107,13 +104,6 @@ public void marshalOnDaemonNode(boolean startFirst) throws Exception { awaitPartitionMapExchange(); - // Workaround for IGNITE-1365. - IgniteInternalFuture fut = ((IgniteKernal) daemonNode).context().cache().context().exchange(). - affinityReadyFuture(new AffinityTopologyVersion(2, 0)); - - if (fut != null) - fut.get(); - TestClass1 res1 = daemonNode.compute(daemonNode.cluster().forRemotes()).call(new TestCallable1()); assertNotNull(res1); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java index 4dfe69b03c6cb..665bb566598ad 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java @@ -151,11 +151,6 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { // No-op. } - /** {@inheritDoc} */ - @Override public void onKernalStart(boolean reconnect) throws IgniteCheckedException { - // No-op. - } - /** {@inheritDoc} */ @Override public void onKernalStop(boolean cancel) { // No-op. @@ -177,8 +172,7 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { } /** {@inheritDoc} */ - @Override public void storeCacheData(CacheGroupDescriptor grpDesc, - StoredCacheData cacheData) throws IgniteCheckedException { + @Override public void storeCacheData(StoredCacheData cacheData) throws IgniteCheckedException { // No-op. } @@ -189,11 +183,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { /** {@inheritDoc} */ @Override public void onActivate(GridKernalContext kctx) { - + // No-op. } /** {@inheritDoc} */ @Override public void onDeActivate(GridKernalContext kctx) { - + // No-op. } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java index 0ef593f7d604b..72450b861f08c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java @@ -43,7 +43,7 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager { /** {@inheritDoc} */ @Override public void resumeLogging(WALPointer ptr) throws IgniteCheckedException { - + // No-op. } /** {@inheritDoc} */ @@ -83,42 +83,37 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager { /** {@inheritDoc} */ @Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException { - + // No-op. } /** {@inheritDoc} */ @Override public void stop(boolean cancel) { - - } - - /** {@inheritDoc} */ - @Override public void onKernalStart(boolean reconnect) throws IgniteCheckedException { - + // No-op. } /** {@inheritDoc} */ @Override public void onKernalStop(boolean cancel) { - + // No-op. } /** {@inheritDoc} */ @Override public void onDisconnected(IgniteFuture reconnectFut) { - + // No-op. } /** {@inheritDoc} */ @Override public void printMemoryStats() { - + // No-op. } /** {@inheritDoc} */ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException { - + // No-op. } /** {@inheritDoc} */ - @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException { - + @Override public void onDeActivate(GridKernalContext kctx) { + // No-op. } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java index 9fa6f7c41b694..675aca5e3253a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java @@ -53,123 +53,160 @@ public abstract class AbstractNodeJoinTemplate extends GridCommonAbstractTest { /** Cache 2. */ protected static final String cache2 = "cache2"; - //Todo Cache with node filter. + /** */ protected static final String cache3 = "cache3"; + /** */ protected static final String cache4 = "cache4"; - protected static final String cache5 = "cache5"; + /** */ + private static final String cache5 = "cache5"; /** Caches info. */ - public static final String CACHES_INFO = "cachesInfo"; + private static final String CACHES_INFO = "cachesInfo"; /** Registered caches. */ - public static final String REGISTERED_CACHES = "registeredCaches"; + private static final String REGISTERED_CACHES = "registeredCaches"; /** Caches. */ public static final String CACHES = "caches"; /** - * @param ig Ig. + * @param ig Node. + * @return Cache descriptors. */ protected static Map cacheDescriptors(IgniteEx ig) { return field((Object)field(ig.context().cache(), CACHES_INFO), REGISTERED_CACHES); } /** - * @param ig Ig. + * @param ig Node. + * @return Node caches. */ protected static Map caches(IgniteEx ig){ return field(ig.context().cache(), CACHES); } /** - * + * @return Test builder. + * @throws Exception If failed. */ public abstract JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception; /** - * + * @return Test builder. + * @throws Exception If failed. */ public abstract JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception; /** - * + * @return Test builder. + * @throws Exception If failed. */ public abstract JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception; /** - * + * @return Test builder. + * @throws Exception If failed. */ public abstract JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception; /** - * + * @return Test builder. + * @throws Exception If failed. */ public abstract JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception; // Client node join. + /** + * @return Test builder. + * @throws Exception If failed. + */ public abstract JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception; + /** + * @return Test builder. + * @throws Exception If failed. + */ public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception; + /** + * @return Test builder. + * @throws Exception If failed. + */ public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception; + /** + * @return Test builder. + * @throws Exception If failed. + */ public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception; + /** + * @return Test builder. + * @throws Exception If failed. + */ public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception; /** - * + * @throws Exception If failed. */ public abstract void testJoinWithOutConfiguration() throws Exception; /** - * + * @throws Exception If failed. */ public abstract void testStaticCacheConfigurationOnJoin() throws Exception; /** + * @throws Exception If failed. * */ public abstract void testStaticCacheConfigurationInCluster() throws Exception; /** + * @throws Exception If failed. * */ public abstract void testStaticCacheConfigurationSameOnBoth() throws Exception; /** + * @throws Exception If failed. * */ public abstract void testStaticCacheConfigurationDifferentOnBoth() throws Exception; /** + * @throws Exception If failed. * */ public abstract void testJoinClientWithOutConfiguration() throws Exception; /** + * @throws Exception If failed. * */ public abstract void testJoinClientStaticCacheConfigurationOnJoin() throws Exception; /** + * @throws Exception If failed. * */ public abstract void testJoinClientStaticCacheConfigurationInCluster() throws Exception; /** + * @throws Exception If failed. * */ public abstract void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception; /** - * + * @throws Exception If failed. */ public abstract void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); @@ -178,6 +215,7 @@ protected static Map caches(IgniteEx ig){ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); } + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { super.afterTest(); @@ -188,6 +226,7 @@ protected static Map caches(IgniteEx ig){ /** * @param idx Index. + * @return Ignite instance name. */ protected String name(int idx) { return getTestIgniteInstanceName(idx); @@ -195,6 +234,8 @@ protected String name(int idx) { /** * @param name Name. + * @return Igntie configuration. + * @throws Exception If failed. */ protected IgniteConfiguration cfg(String name) throws Exception { try { @@ -206,7 +247,7 @@ protected IgniteConfiguration cfg(String name) throws Exception { } /** - * + * @return Test builder. */ protected JoinNodeTestPlanBuilder builder() { return JoinNodeTestPlanBuilder.builder(); @@ -214,13 +255,14 @@ protected JoinNodeTestPlanBuilder builder() { /** * @param cfgs Cfgs. + * @return Configurations. */ - protected static T[] buildConfiguration(T... cfgs) { + private static T[] buildConfiguration(T... cfgs) { return cfgs; } /** - * + * @return Cache configuration. */ protected CacheConfiguration atomicCfg() { return new CacheConfiguration(cache1) @@ -228,6 +270,7 @@ protected CacheConfiguration atomicCfg() { } /** + * @return Cache configuration. * */ protected CacheConfiguration transactionCfg() { @@ -236,7 +279,7 @@ protected CacheConfiguration transactionCfg() { } /** - * + * @return Cache configurations. */ protected CacheConfiguration[] allCacheConfigurations() { return buildConfiguration(atomicCfg(), transactionCfg()); @@ -309,6 +352,7 @@ public static class JoinNodeTestPlanBuilder extends GridCommonAbstractTest { /** After de activate. */ private Runnable afterDeActivate = Noop; + /** */ private IgniteCallable> dynamicCacheStart = new IgniteCallable>() { @Override public List call() throws Exception { @@ -316,6 +360,7 @@ public static class JoinNodeTestPlanBuilder extends GridCommonAbstractTest { } }; + /** */ private IgniteCallable> dynamicCacheStop = new IgniteCallable>() { @Override public List call() throws Exception { @@ -323,15 +368,19 @@ public static class JoinNodeTestPlanBuilder extends GridCommonAbstractTest { } }; + /** */ private Runnable afterDynamicCacheStarted = Noop; + /** */ private Runnable afterDynamicCacheStopped = Noop; /** End. */ private Runnable end = Noop; /** - * + * @param cfgs Configurations. + * @return Test builder. + * @throws Exception If failed. */ public JoinNodeTestPlanBuilder clusterConfiguration(IgniteConfiguration... cfgs) throws Exception { clusterCfg = cfgs; @@ -359,7 +408,8 @@ public JoinNodeTestPlanBuilder clusterConfiguration(IgniteConfiguration... cfgs) } /** - * + * @param cfg Configuration. + * @return Test builder. */ public JoinNodeTestPlanBuilder nodeConfiguration(IgniteConfiguration cfg) { nodeCfg = cfg; @@ -382,6 +432,7 @@ public JoinNodeTestPlanBuilder nodeConfiguration(IgniteConfiguration cfg) { /** * @param func Func. + * @return Test builder. */ public JoinNodeTestPlanBuilder nodeConfiguration( IgniteClosure func @@ -393,7 +444,8 @@ public JoinNodeTestPlanBuilder nodeConfiguration( } /** - * + * @param r Cluster start callback. + * @return Test builder. */ public JoinNodeTestPlanBuilder afterClusterStarted(Runnable r) { strPlanBuilder.append("Check after cluster start\n"); @@ -404,7 +456,8 @@ public JoinNodeTestPlanBuilder afterClusterStarted(Runnable r) { } /** - * + * @param r Node join callback. + * @return Test builder. */ public JoinNodeTestPlanBuilder afterNodeJoin(Runnable r) { strPlanBuilder.append("Check after node join") @@ -416,7 +469,8 @@ public JoinNodeTestPlanBuilder afterNodeJoin(Runnable r) { } /** - * + * @param state State after join. + * @return Test builder. */ public JoinNodeTestPlanBuilder stateAfterJoin(boolean state) { strPlanBuilder.append("Check state on all nodes after join, must be ") @@ -429,7 +483,8 @@ public JoinNodeTestPlanBuilder stateAfterJoin(boolean state) { } /** - * + * @param r Activate callback. + * @return Test builder. */ public JoinNodeTestPlanBuilder afterActivate(Runnable r) { strPlanBuilder.append("Check after activate") @@ -441,7 +496,8 @@ public JoinNodeTestPlanBuilder afterActivate(Runnable r) { } /** - * + * @param r Deactivate callback. + * @return Test builder. */ public JoinNodeTestPlanBuilder afterDeActivate(Runnable r) { strPlanBuilder.append("Check after deActivate") @@ -452,6 +508,10 @@ public JoinNodeTestPlanBuilder afterDeActivate(Runnable r) { return this; } + /** + * @param caches Callback. + * @return Test builder. + */ public JoinNodeTestPlanBuilder dynamicCacheStart(IgniteCallable> caches){ strPlanBuilder.append("Dynamic caches start") .append("\n"); @@ -461,6 +521,10 @@ public JoinNodeTestPlanBuilder dynamicCacheStart(IgniteCallable> caches){ strPlanBuilder.append("Dynamic caches stop") .append("\n"); @@ -479,6 +547,10 @@ public JoinNodeTestPlanBuilder dynamicCacheStop(IgniteCallable> cac return this; } + /** + * @param r Callback. + * @return Test builder. + */ public JoinNodeTestPlanBuilder afterDynamicCacheStopped(Runnable r){ strPlanBuilder.append("Check after dynamic caches stop") .append("\n"); @@ -490,6 +562,7 @@ public JoinNodeTestPlanBuilder afterDynamicCacheStopped(Runnable r){ /** * @param end End. + * @return Test builder. */ public JoinNodeTestPlanBuilder setEnd(Runnable end) { strPlanBuilder.append("Check before stop") @@ -501,7 +574,7 @@ public JoinNodeTestPlanBuilder setEnd(Runnable end) { } /** - * + * @throws Exception If failed. */ public void execute() throws Exception { try { @@ -611,12 +684,19 @@ public void execute() throws Exception { } } + /** + * @param ig Node. + * @return Next minor version. + */ private AffinityTopologyVersion nextMinorVersion(IgniteEx ig){ AffinityTopologyVersion cur = ig.context().discovery().topologyVersionEx(); - return new AffinityTopologyVersion(cur.topologyVersion(), cur.minorTopologyVersion() + 1); + return cur.nextMinorVersion(); } + /** + * @param ver Version. + */ private void awaitTopologyVersion(final AffinityTopologyVersion ver){ onAllNode(new CI1() { @Override public void apply(IgniteEx ig) { @@ -643,7 +723,7 @@ private void awaitTopologyVersion(final AffinityTopologyVersion ver){ } /** - * + * @return Started nodes. */ protected List grids() { List res = new ArrayList<>(); @@ -655,14 +735,14 @@ protected List grids() { } /** - * + * @return Test builder. */ public static JoinNodeTestPlanBuilder builder() { return new JoinNodeTestPlanBuilder(); } /** - * + * @return Callback. */ public Runnable checkCacheOnlySystem() { return onAllNode(new IgniteInClosure() { @@ -686,15 +766,11 @@ public Runnable checkCacheOnlySystem() { } /** - * + * @return Callback. */ public Runnable checkCacheEmpty() { return onAllNode(new IgniteInClosure() { @Override public void apply(IgniteEx ig) { - Map desc = cacheDescriptors(ig); - - Assert.assertTrue(desc.isEmpty()); - Assert.assertNull(ig.context().cache().cache(cache1)); Assert.assertNull(ig.context().cache().cache(cache2)); @@ -706,7 +782,7 @@ public Runnable checkCacheEmpty() { } /** - * + * @return Callback. */ public Runnable checkCacheNotEmpty() { return onAllNode(new IgniteInClosure() { @@ -735,6 +811,7 @@ public Runnable checkCacheNotEmpty() { /** * @param cls Closure. + * @return Callback. */ private Runnable onAllNode(final IgniteInClosure cls) { return new Runnable() { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java index a06e0ceac1572..4e575cc42be46 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java @@ -47,13 +47,10 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst private static final String clientSuffix = "-client"; /** Primary ip finder. */ - protected final TcpDiscoveryIpFinder primaryIpFinder = new TcpDiscoveryVmIpFinder(true); + private final TcpDiscoveryIpFinder primaryIpFinder = new TcpDiscoveryVmIpFinder(true); /** Back up ip finder. */ - protected final TcpDiscoveryIpFinder backUpIpFinder = new TcpDiscoveryVmIpFinder(true); - - /** Consistent id count. */ - private int consistentIdCnt; + private final TcpDiscoveryIpFinder backUpIpFinder = new TcpDiscoveryVmIpFinder(true); /** Nodes. */ protected Map nodes = new ConcurrentHashMap<>(); @@ -91,28 +88,28 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst } /** - * + * @return Number of server nodes in primary cluster. */ protected int primaryNodes() { return 3; } /** - * + * @return Number of client nodes in primary cluster. */ protected int primaryClientNodes() { return 3; } /** - * + * @return Number of server nodes in backup cluster. */ protected int backUpNodes() { return 3; } /** - * + * @return Number of client nodes in backup cluster. */ protected int backUpClientNodes() { return 3; @@ -120,6 +117,7 @@ protected int backUpClientNodes() { /** * @param idx idx. + * @return Primary cluster node. */ protected Ignite primary(int idx) { return nodes.get("node" + idx + primarySuffix); @@ -127,29 +125,33 @@ protected Ignite primary(int idx) { /** * @param idx idx. + * @return Primary cluster client node. */ - protected Ignite primaryClient(int idx) { + Ignite primaryClient(int idx) { return nodes.get("node" + idx + primarySuffix + clientSuffix); } /** * @param idx idx. + * @return Backup cluster node. */ - protected Ignite backUp(int idx) { + Ignite backUp(int idx) { return nodes.get("node" + idx + backUpSuffix); } /** * @param idx idx. + * @return Backup cluster client node. */ - protected Ignite backUpClient(int idx) { + Ignite backUpClient(int idx) { return nodes.get("node" + idx + backUpSuffix + clientSuffix); } /** * @param cnt Count. + * @throws Exception If failed. */ - protected void startPrimaryNodes(int cnt) throws Exception { + private void startPrimaryNodes(int cnt) throws Exception { for (int i = 0; i < cnt; i++) startPrimary(i); @@ -159,8 +161,9 @@ protected void startPrimaryNodes(int cnt) throws Exception { /** * @param idx Index. + * @throws Exception If failed. */ - protected void startPrimary(int idx) throws Exception { + private void startPrimary(int idx) throws Exception { String node = "node" + idx; String name = node + primarySuffix; @@ -176,22 +179,26 @@ protected void startPrimary(int idx) throws Exception { /** * @param cnt Count. + * @throws Exception If failed. */ - protected void startBackUpNodes(int cnt) throws Exception { + private void startBackUpNodes(int cnt) throws Exception { for (int i = 0; i < cnt; i++) startBackUp(i); } /** * @param idx Index. + * @throws Exception If failed. */ - protected void startBackUp(int idx) throws Exception { + void startBackUp(int idx) throws Exception { String node = "node" + idx; String name = node + backUpSuffix; IgniteConfiguration cfg = getConfiguration(name); + cfg.setConsistentId(node); + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(backUpIpFinder); Ignite ig = startGrid(name, cfg); @@ -201,16 +208,19 @@ protected void startBackUp(int idx) throws Exception { /** * @param cnt Count. + * @throws Exception If failed. */ - protected void startPrimaryClientNodes(int cnt) throws Exception { + void startPrimaryClientNodes(int cnt) throws Exception { for (int i = 0; i < cnt; i++) { String node = "node" + i; String name = node + primarySuffix + clientSuffix; IgniteConfiguration cfg = getConfiguration(name); + cfg.setConsistentId(node); cfg.setClientMode(true); + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(primaryIpFinder); Ignite ig = startGrid(name, cfg); @@ -221,8 +231,9 @@ protected void startPrimaryClientNodes(int cnt) throws Exception { /** * @param cnt Count. + * @throws Exception If failed. */ - protected void startBackUpClientNodes(int cnt) throws Exception { + private void startBackUpClientNodes(int cnt) throws Exception { for (int i = 0; i < cnt; i++) { String node = "node" + i; @@ -241,9 +252,9 @@ protected void startBackUpClientNodes(int cnt) throws Exception { } /** - * + * @return All nodes from backup cluster. */ - protected Iterable allBackUpNodes() { + Iterable allBackUpNodes() { List r = new ArrayList<>(); for (String name : this.nodes.keySet()) @@ -254,11 +265,10 @@ protected Iterable allBackUpNodes() { } /** - * + * @param includeClient If {@code true} then allow to return client. + * @return Random node from backup topology. */ - protected Ignite randomBackUp(boolean includeClient) { - int nodes = 0; - + Ignite randomBackUp(boolean includeClient) { List igs = new ArrayList<>(); for (String name : this.nodes.keySet()) @@ -281,7 +291,7 @@ protected Ignite randomBackUp(boolean includeClient) { /** * @param i Idx. */ - protected void stopPrimary(int i) { + void stopPrimary(int i) { String name = "node" + i + primarySuffix; nodes.get(name).close(); @@ -292,7 +302,7 @@ protected void stopPrimary(int i) { /** * */ - protected void stopAllPrimary() { + void stopAllPrimary() { stopAll(primarySuffix); } @@ -357,10 +367,9 @@ private void stopAll(String suffix) { } /** - * + * @return Test class name. */ protected String testName() { return getClass().getSimpleName(); } - } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateCacheTest.java index 2f2385df9f3a9..938b3c81b1650 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateCacheTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateCacheTest.java @@ -62,7 +62,7 @@ public void testCheckValueAfterActivation(){ } /** - * + * @throws Exception If failed. */ public void testMoreKeyValueAfterActivate() throws Exception { String cacheName = "my-cache"; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java index d3e82980d2da1..16be316154b21 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java @@ -36,10 +36,9 @@ public class IgniteChangeGlobalStateDataStreamerTest extends IgniteChangeGlobalS } /** - * + * @throws Exception If failed. */ - public void testDeActivateAndActivateDataStreamer() throws InterruptedException { - + public void testDeActivateAndActivateDataStreamer() throws Exception { Ignite ig1 = primary(0); Ignite ig2 = primary(1); Ignite ig3 = primary(2); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStructureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStructureTest.java index 901322664ade6..8902a36ea5622 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStructureTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStructureTest.java @@ -35,9 +35,9 @@ */ public class IgniteChangeGlobalStateDataStructureTest extends IgniteChangeGlobalStateAbstractTest { /** - * + * @throws Exception If failed. */ - public void testDeActivateAndActivateAtomicLong() throws Exception{ + public void testDeActivateAndActivateAtomicLong() throws Exception { String lName = "myLong"; Ignite ig1 = primary(0); @@ -106,7 +106,7 @@ public void testDeActivateAndActivateAtomicLong() throws Exception{ } /** - * + * @throws Exception If failed. */ public void testDeActivateAndActivateCountDownLatch() throws Exception { final AtomicInteger cnt = new AtomicInteger(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java index d0b2d5668d93c..92d1f2198b3b8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java @@ -25,6 +25,7 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.typedef.internal.U; import static java.lang.Thread.sleep; import static org.apache.ignite.testframework.GridTestUtils.runAsync; @@ -54,7 +55,7 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState } /** - * + * @throws Exception If failed. */ public void testActivateDeActivateOnFixTopology() throws Exception { final Ignite igB1 = backUp(0); @@ -140,7 +141,7 @@ public void testActivateDeActivateOnFixTopology() throws Exception { } /** - * + * @throws Exception If failed. */ public void testActivateDeActivateOnJoiningNode() throws Exception { fail("https://issues.apache.org/jira/browse/IGNITE-5520"); @@ -168,9 +169,9 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { final IgniteInternalFuture af = runAsync(new Callable() { @Override public Void call() throws Exception { while (!stop.get()) { - Ignite ig = randomBackUp(false); - if (canAct.get()) { + Ignite ig = randomBackUp(false); + long start = System.currentTimeMillis(); ig.active(true); @@ -184,6 +185,8 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { canAct.set(false); } + else + U.sleep(100); } return null; @@ -193,9 +196,9 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { final IgniteInternalFuture df = runAsync(new Callable() { @Override public Void call() throws Exception { while (!stop.get()) { - Ignite ig = randomBackUp(false); - if (!canAct.get()) { + Ignite ig = randomBackUp(false); + long start = System.currentTimeMillis(); ig.active(false); @@ -209,7 +212,8 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { canAct.set(true); } - + else + U.sleep(100); } return null; } @@ -243,16 +247,16 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { jf2.get(); } finally { - log.info("total started nodes: " + (seqIdx.get() - backUpNodes())); + log.info("Total started nodes: " + (seqIdx.get() - backUpNodes())); - log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime: " + log.info("Total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime: " + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get())) ); } } /** - * + * @throws Exception If failed. */ public void testActivateDeActivateOnFixTopologyWithPutValues() throws Exception { final Ignite igB1 = backUp(0); @@ -348,7 +352,7 @@ public void testActivateDeActivateOnFixTopologyWithPutValues() throws Exception df.get(); } finally { - log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:" + log.info("Total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:" + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get()) + " nodes: " + backUpNodes())); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java index 9151c24c823a4..80bf1fb399dc3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java @@ -128,74 +128,65 @@ public void testConcurrentActivateFromClientNodeAndServerNode() throws Exception stopAllPrimary(); - final AtomicInteger exCnt = new AtomicInteger(); - final CyclicBarrier barrier = new CyclicBarrier(backUpNodes() + backUpClientNodes()); IgniteInternalFuture f1 = runAsync(new Callable() { @Override public Void call() throws Exception { - try { - barrier.await(); - ig1B.active(true); - }catch (Exception e){ - exCnt.incrementAndGet(); - } + barrier.await(); + + ig1B.active(true); + return null; } }); - IgniteInternalFuture f2 = runAsync(new Runnable() { - @Override public void run() { - try { - barrier.await(); - ig2B.active(true); - }catch (Exception e){ - exCnt.incrementAndGet(); - } + IgniteInternalFuture f2 = runAsync(new Callable() { + @Override public Void call() throws Exception { + barrier.await(); + + ig2B.active(true); + + return null; } }); - IgniteInternalFuture f3 = runAsync(new Runnable() { - @Override public void run() { - try { - barrier.await(); - ig3B.active(true); - }catch (Exception e){ - exCnt.incrementAndGet(); - } + IgniteInternalFuture f3 = runAsync(new Callable() { + @Override public Void call() throws Exception { + barrier.await(); + + ig3B.active(true); + + return null; } }); - IgniteInternalFuture f4 = runAsync(new Runnable() { - @Override public void run() { - try { - barrier.await(); - ig1C.active(true); - }catch (Exception e){ - exCnt.incrementAndGet(); - } + IgniteInternalFuture f4 = runAsync(new Callable() { + @Override public Void call() throws Exception { + barrier.await(); + + ig1C.active(true); + + return null; } }); - IgniteInternalFuture f5 = runAsync(new Runnable() { - @Override public void run() { - try { - barrier.await(); - ig2C.active(true); - }catch (Exception e){ - exCnt.incrementAndGet(); - } + IgniteInternalFuture f5 = runAsync(new Callable() { + @Override public Void call() throws Exception { + barrier.await(); + + ig2C.active(true); + + return null; } }); - IgniteInternalFuture f6 = runAsync(new Runnable() { - @Override public void run() { - try { - barrier.await(); - ig3C.active(true); - }catch (Exception e){ - exCnt.incrementAndGet(); - } + IgniteInternalFuture f6 = runAsync(new Callable() { + @Override public Void call() throws Exception { + barrier.await(); + + ig3C.active(true); + + return null; } }); @@ -207,8 +198,6 @@ public void testConcurrentActivateFromClientNodeAndServerNode() throws Exception f5.get(); f6.get(); - assertTrue(exCnt.get() > 0); - assertTrue(ig1B.active()); assertTrue(ig2B.active()); assertTrue(ig3B.active()); @@ -232,40 +221,35 @@ public void testConcurrentActivateFromServerNode() throws Exception { stopAllPrimary(); - final AtomicInteger exCnt = new AtomicInteger(); - final CyclicBarrier barrier = new CyclicBarrier(3); - IgniteInternalFuture act1 = runAsync(new Runnable() { - @Override public void run() { - try { - barrier.await(); - ig1B.active(true); - }catch (Exception e){ - exCnt.incrementAndGet(); - } + IgniteInternalFuture act1 = runAsync(new Callable() { + @Override public Void call() throws Exception { + barrier.await(); + + ig1B.active(true); + + return null; } }); - IgniteInternalFuture act2 = runAsync(new Runnable() { - @Override public void run() { - try { - barrier.await(); - ig2B.active(true); - }catch (Exception e){ - exCnt.incrementAndGet(); - } + IgniteInternalFuture act2 = runAsync(new Callable() { + @Override public Void call() throws Exception { + barrier.await(); + + ig2B.active(true); + + return null; } }); - IgniteInternalFuture act3 = runAsync(new Runnable() { - @Override public void run() { - try { - barrier.await(); - ig3B.active(true); - }catch (Exception e){ - exCnt.incrementAndGet(); - } + IgniteInternalFuture act3 = runAsync(new Callable() { + @Override public Void call() throws Exception { + barrier.await(); + + ig3B.active(true); + + return null; } }); @@ -273,15 +257,13 @@ public void testConcurrentActivateFromServerNode() throws Exception { act2.get(); act3.get(); - assertEquals(2, exCnt.get()); - assertTrue(ig1B.active()); assertTrue(ig2B.active()); assertTrue(ig3B.active()); } /** - * + * @throws Exception If failed. */ public void testActiveAndInActiveAtTheSameTimeCluster() throws Exception { Ignite ig1P = primary(0); @@ -318,7 +300,7 @@ public void testActiveAndInActiveAtTheSameTimeCluster() throws Exception { } /** - * + * @throws Exception If failed. */ public void testActivateOnAlreadyActivatedCluster() throws Exception { Ignite ig1P = primary(0); @@ -375,7 +357,7 @@ public void testActivateOnAlreadyActivatedCluster() throws Exception { } /** - * + * @throws Exception If failed. */ public void testTryUseCacheInActiveCluster() throws Exception { Ignite ig1B = backUp(0); @@ -410,13 +392,14 @@ private void checkExceptionTryUseCache(final Ignite ig) { assertThrows(log, new Callable() { @Override public Void call() throws Exception { IgniteCache cache = ig.cache("cache"); + return null; } }, IgniteException.class, "Can not perform the operation because the cluster is inactive."); } /** - * + * @throws Exception If failed. */ public void testTryUseServiceInActiveCluster() throws Exception { Ignite ig1B = backUp(0); @@ -445,7 +428,7 @@ public void testTryUseServiceInActiveCluster() throws Exception { } /** - * + * @param ig Node to check. */ private void checkExceptionTryUseService(final Ignite ig) { assertThrows(log, new Callable() { @@ -458,7 +441,7 @@ private void checkExceptionTryUseService(final Ignite ig) { } /** - * + * @throws Exception If failed. */ public void testTryUseDataStructureInActiveCluster() throws Exception { Ignite ig1B = backUp(0); @@ -487,19 +470,20 @@ public void testTryUseDataStructureInActiveCluster() throws Exception { } /** - * + * @param ig Node. */ private void checkExceptionTryUseDataStructure(final Ignite ig){ assertThrows(log, new Callable() { @Override public Void call() throws Exception { IgniteAtomicSequence seq = ig.atomicSequence("seq", 0, true); + return null; } }, IgniteException.class, "Can not perform the operation because the cluster is inactive."); } /** - * + * @throws Exception If failed. */ public void testFailGetLock() throws Exception { Ignite ig1P = primary(0); @@ -556,7 +540,7 @@ public void testFailGetLock() throws Exception { } /** - * + * @throws Exception If failed. */ public void testActivateAfterFailGetLock() throws Exception { Ignite ig1P = primary(0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java index c3bdcdaf73409..2678e51ad49f8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java @@ -60,7 +60,7 @@ public class IgniteStandByClusterTest extends GridCommonAbstractTest { /** * @throws Exception if fail. */ - public void testNotStartDynamicCachesOnClientAfterActivation() throws Exception { + public void testStartDynamicCachesAfterActivation() throws Exception { final String cacheName0 = "cache0"; final String cacheName = "cache"; @@ -114,13 +114,10 @@ public void testNotStartDynamicCachesOnClientAfterActivation() throws Exception Map> caches = U.field(ig3.context().cache(), "caches"); // Only system cache and cache0 - assertTrue(caches.size() == 2); + assertEquals("Unexpected caches: " + caches.keySet(), 3, caches.size()); assertTrue(caches.containsKey(CU.UTILITY_CACHE_NAME)); assertTrue(caches.containsKey(cacheName0)); - - assertNull(caches.get(cacheName)); - - assertNotNull(caches.get(cacheName0)); + assertTrue(caches.containsKey(cacheName)); assertNotNull(ig3.cache(cacheName)); } @@ -156,12 +153,6 @@ public void testStaticCacheStartAfterActivationWithCacheFilter() throws Exceptio assertTrue(!ig2.active()); assertTrue(!ig3.active()); - for (IgniteEx ig : Arrays.asList(ig1, ig2, ig3)) { - Map desc = U.field(U.field(ig.context().cache(), "cachesInfo"), "registeredCaches"); - - assertEquals(0, desc.size()); - } - ig3.active(true); assertTrue(ig1.active()); @@ -286,7 +277,7 @@ private static class NodeFilterIgnoreByName implements IgnitePredicate spi = newCommunicationSpi(); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index a9a870ee8227d..d6d241ccbdac9 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -2099,13 +2099,13 @@ private void awaitTopologyChange() throws IgniteInterruptedCheckedException { for (Ignite g : G.allGrids()) { final GridKernalContext ctx = ((IgniteKernal)g).context(); - if (ctx.isStopping()) + if (ctx.isStopping() || !g.active()) continue; AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx(); AffinityTopologyVersion exchVer = ctx.cache().context().exchange().readyAffinityVersion(); - if (! topVer.equals(exchVer)) { + if (!topVer.equals(exchVer)) { info("Topology version mismatch [node=" + g.name() + ", exchVer=" + exchVer + ", topVer=" + topVer + ']'); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index 3f87c766232b4..dc7e89d6815cb 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -596,6 +596,9 @@ protected void awaitPartitionMapExchange( else startTime = g0.context().discovery().gridStartTime(); + if (g.cluster().localNode().isDaemon()) + continue; + IgniteInternalFuture exchFut = g0.context().cache().context().exchange().affinityReadyFuture(waitTopVer); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java index 6fa158db548f5..f016e390ee76c 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java @@ -18,6 +18,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTest; import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateCacheTest; import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStreamerTest; import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStructureTest; @@ -41,11 +42,13 @@ */ public class IgniteStandByClusterSuite extends TestSuite { /** - * + * @return Test suite. */ public static TestSuite suite() { TestSuite suite = new TestSuite("Ignite Activate/DeActivate Cluster Test Suit"); + suite.addTestSuite(IgniteClusterActivateDeactivateTest.class); + suite.addTestSuite(IgniteStandByClusterTest.class); suite.addTestSuite(IgniteStandByClientReconnectTest.class); suite.addTestSuite(IgniteStandByClientReconnectToNewClusterTest.class); diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java index 329d67fed653f..231fc22b125ec 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java @@ -95,8 +95,8 @@ public HadoopProcessor(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void onKernalStart() throws IgniteCheckedException { - super.onKernalStart(); + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { + super.onKernalStart(active); if (hctx == null) return; From f9f13cf083b481f004531710ec3835afdf5b7cef Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 5 Jul 2017 12:59:43 +0300 Subject: [PATCH 059/155] IGNITE-5605 .NET: Inject resources into remote event filters --- .../Apache.Ignite.Core.Tests/EventsTest.cs | 19 ++++++++++++++----- .../Impl/Events/RemoteListenEventFilter.cs | 3 +++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs index 75784755bbc2c..c05511c0b1810 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs @@ -17,6 +17,8 @@ // ReSharper disable MemberCanBePrivate.Global // ReSharper disable UnusedParameter.Global +// ReSharper disable UnusedAutoPropertyAccessor.Local +// ReSharper disable UnusedAutoPropertyAccessor.Global #pragma warning disable 618 namespace Apache.Ignite.Core.Tests { @@ -33,6 +35,7 @@ namespace Apache.Ignite.Core.Tests using Apache.Ignite.Core.Events; using Apache.Ignite.Core.Impl; using Apache.Ignite.Core.Impl.Events; + using Apache.Ignite.Core.Resource; using Apache.Ignite.Core.Tests.Compute; using NUnit.Framework; @@ -360,14 +363,14 @@ public void TestWaitForLocal() if (i > 3) { // Filter - waitTask = getWaitTask(new EventFilter(e => e.Type == EventType.TaskReduced), new int[0]); + waitTask = getWaitTask(new LocalEventFilter(e => e.Type == EventType.TaskReduced), new int[0]); Assert.IsTrue(waitTask.Wait(timeout)); Assert.IsInstanceOf(typeof(TaskEvent), waitTask.Result); Assert.AreEqual(EventType.TaskReduced, waitTask.Result.Type); // Filter & types - waitTask = getWaitTask(new EventFilter(e => e.Type == EventType.TaskReduced), + waitTask = getWaitTask(new LocalEventFilter(e => e.Type == EventType.TaskReduced), new[] {EventType.TaskReduced}); Assert.IsTrue(waitTask.Wait(timeout)); @@ -868,7 +871,7 @@ public static void VerifyReceive(int count, Type eventObjectType, ICollectionNew instance of event listener. public static IEventListener GetListener() { - return new EventFilter(Listen); + return new LocalEventFilter(Listen); } /// @@ -917,7 +920,7 @@ private static bool Listen(IEvent evt) /// Test event filter. /// [Serializable] - public class EventFilter : IEventFilter, IEventListener where T : IEvent + public class LocalEventFilter : IEventFilter, IEventListener where T : IEvent { /** */ private readonly Func _invoke; @@ -926,7 +929,7 @@ public class EventFilter : IEventFilter, IEventListener where T : IEven /// Initializes a new instance of the class. /// /// The invoke delegate. - public EventFilter(Func invoke) + public LocalEventFilter(Func invoke) { _invoke = invoke; } @@ -960,6 +963,10 @@ public class RemoteEventFilter : IEventFilter /** */ private readonly int _type; + /** */ + [InstanceResource] + public IIgnite Ignite { get; set; } + public RemoteEventFilter(int type) { _type = type; @@ -968,6 +975,8 @@ public RemoteEventFilter(int type) /** */ public bool Invoke(IEvent evt) { + Assert.IsNotNull(Ignite); + return evt.Type == _type; } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/RemoteListenEventFilter.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/RemoteListenEventFilter.cs index 31bfff1364b0a..2e0b66cb9d7e6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/RemoteListenEventFilter.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/RemoteListenEventFilter.cs @@ -22,6 +22,7 @@ namespace Apache.Ignite.Core.Impl.Events using Apache.Ignite.Core.Events; using Apache.Ignite.Core.Impl.Binary.IO; using Apache.Ignite.Core.Impl.Common; + using Apache.Ignite.Core.Impl.Resource; /// /// Event filter/listener holder for RemoteListen. @@ -75,6 +76,8 @@ public static RemoteListenEventFilter CreateInstance(long memPtr, Ignite grid) var pred = reader.ReadObject(); + ResourceProcessor.Inject(pred, grid); + var func = DelegateTypeDescriptor.GetEventFilter(pred.GetType()); return new RemoteListenEventFilter(grid, evt => func(pred, evt)); From 301f310b6348b3f6fbe54b16d065569755e71bce Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Tue, 4 Jul 2017 19:42:33 +0300 Subject: [PATCH 060/155] IGNITE-5663: ODBC: Closing cursor do not reset prepared statement anymore (cherry picked from commit 64c156e) --- .../cpp/odbc-test/src/queries_test.cpp | 61 +++++++++++++++++++ modules/platforms/cpp/odbc/src/odbc.cpp | 2 +- modules/platforms/cpp/odbc/src/statement.cpp | 3 - 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp index 41bc1fca4ee77..263993c5239ff 100644 --- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp +++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp @@ -1934,5 +1934,66 @@ BOOST_AUTO_TEST_CASE(TestParamsNum) CheckParamsNum("INSERT INTO TestType(_key, strField) VALUES(?, ?)", 2); } +BOOST_AUTO_TEST_CASE(TestExecuteAfterCursorClose) +{ + TestType in(1, 2, 3, 4, "5", 6.0f, 7.0, true, Guid(8, 9), MakeDateGmt(1987, 6, 5), + MakeTimeGmt(12, 48, 12), MakeTimestampGmt(1998, 12, 27, 1, 2, 3, 456)); + + cache1.Put(1, in); + + Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache"); + + int64_t key = 0; + char strField[1024] = { 0 }; + SQLLEN strFieldLen = 0; + + // Binding columns. + SQLRETURN ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &key, 0, 0); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + // Binding columns. + ret = SQLBindCol(stmt, 2, SQL_C_CHAR, &strField, sizeof(strField), &strFieldLen); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + // Just selecting everything to make sure everything is OK + SQLCHAR selectReq[] = "SELECT _key, strField FROM TestType"; + + ret = SQLPrepare(stmt, selectReq, sizeof(selectReq)); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + ret = SQLExecute(stmt); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + ret = SQLFreeStmt(stmt, SQL_CLOSE); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + ret = SQLExecute(stmt); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + ret = SQLFetch(stmt); + + if (!SQL_SUCCEEDED(ret)) + BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt)); + + BOOST_CHECK_EQUAL(key, 1); + + BOOST_CHECK_EQUAL(std::string(strField, strFieldLen), "5"); + + ret = SQLFetch(stmt); + + BOOST_CHECK_EQUAL(ret, SQL_NO_DATA); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/modules/platforms/cpp/odbc/src/odbc.cpp b/modules/platforms/cpp/odbc/src/odbc.cpp index a738598a132cb..b450903517fd1 100644 --- a/modules/platforms/cpp/odbc/src/odbc.cpp +++ b/modules/platforms/cpp/odbc/src/odbc.cpp @@ -212,7 +212,7 @@ namespace ignite { using odbc::Statement; - LOG_MSG("SQLFreeStmt called"); + LOG_MSG("SQLFreeStmt called [option=" << option << ']'); Statement *statement = reinterpret_cast(stmt); diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp index 697f5b4ce1725..adc7d6b964c9f 100644 --- a/modules/platforms/cpp/odbc/src/statement.cpp +++ b/modules/platforms/cpp/odbc/src/statement.cpp @@ -745,9 +745,6 @@ namespace ignite SqlResult::Type result = currentQuery->Close(); - if (result == SqlResult::AI_SUCCESS) - currentQuery.reset(); - return result; } From 58a937e5310c62ccb557d8da959bc4ffa8bcca62 Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 5 Jul 2017 14:24:51 +0300 Subject: [PATCH 061/155] Diagnostic info for GridDhtTxFinishFuture. --- .../internal/IgniteDiagnosticMessage.java | 4 ++- .../dht/GridDhtTxFinishFuture.java | 33 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticMessage.java index 075b0fe617b53..6e6bac076685a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticMessage.java @@ -413,7 +413,9 @@ public final static class TxInfoClosure extends DiagnosticBaseClosure { for (IgniteInternalTx tx : ctx.cache().context().tm().activeTransactions()) { if (dhtVer.equals(tx.xidVersion()) || nearVer.equals(tx.nearXidVersion())) { sb.append(U.nl()) - .append(" [ver=").append(tx.xidVersion()) + .append(" ") + .append(tx.getClass().getSimpleName()) + .append(" [ver=").append(tx.xidVersion()) .append(", nearVer=").append(tx.nearXidVersion()) .append(", topVer=").append(tx.topologyVersion()) .append(", state=").append(tx.state()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java index 8a31bace2e71b..d8180b452f289 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java @@ -26,6 +26,8 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.IgniteDiagnosticAware; +import org.apache.ignite.internal.IgniteDiagnosticPrepareContext; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.NodeStoppingException; import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; @@ -35,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.tostring.GridToStringInclude; @@ -53,7 +56,7 @@ * */ public final class GridDhtTxFinishFuture extends GridCacheCompoundIdentityFuture - implements GridCacheFuture { + implements GridCacheFuture, IgniteDiagnosticAware { /** */ private static final long serialVersionUID = 0L; @@ -553,6 +556,34 @@ private boolean finish(boolean commit, return res; } + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override public void addDiagnosticRequest(IgniteDiagnosticPrepareContext ctx) { + if (!isDone()) { + for (IgniteInternalFuture fut : futures()) { + if (!fut.isDone()) { + MiniFuture f = (MiniFuture)fut; + + if (!f.node().isLocal()) { + GridCacheVersion dhtVer = tx.xidVersion(); + GridCacheVersion nearVer = tx.nearXidVersion(); + + ctx.remoteTxInfo(f.node().id(), dhtVer, nearVer, "GridDhtTxFinishFuture " + + "waiting for response [node=" + f.node().id() + + ", topVer=" + tx.topologyVersion() + + ", dhtVer=" + dhtVer + + ", nearVer=" + nearVer + + ", futId=" + futId + + ", miniId=" + f.futId + + ", tx=" + tx + ']'); + + return; + } + } + } + } + } + /** {@inheritDoc} */ @Override public String toString() { Collection futs = F.viewReadOnly(futures(), new C1, String>() { From 15da654a3e5c2bbf14947f0d3dd08270a9ba6766 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Wed, 5 Jul 2017 15:15:35 +0300 Subject: [PATCH 062/155] IGNITE-5187: Improved DynamicIndexAbstractConcurrentSelfTest reliability. This closes #2219. --- ...ynamicIndexAbstractConcurrentSelfTest.java | 81 ++++++++++--------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractConcurrentSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractConcurrentSelfTest.java index 3fb8a30482f1c..913d724ceb5c7 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractConcurrentSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractConcurrentSelfTest.java @@ -17,6 +17,17 @@ package org.apache.ignite.internal.processors.cache.index; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import javax.cache.Cache; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; @@ -37,21 +48,9 @@ import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor; import org.apache.ignite.internal.processors.query.schema.SchemaOperationException; -import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.util.typedef.T3; import org.jetbrains.annotations.NotNull; -import javax.cache.Cache; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - import static org.apache.ignite.internal.IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi; /** @@ -66,7 +65,7 @@ public abstract class DynamicIndexAbstractConcurrentSelfTest extends DynamicInde private static final int LARGE_CACHE_SIZE = 100_000; /** Latches to block certain index operations. */ - private static final ConcurrentHashMap> BLOCKS = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap> BLOCKS = new ConcurrentHashMap<>(); /** Cache mode. */ private final CacheMode cacheMode; @@ -80,7 +79,7 @@ public abstract class DynamicIndexAbstractConcurrentSelfTest extends DynamicInde * @param cacheMode Cache mode. * @param atomicityMode Atomicity mode. */ - protected DynamicIndexAbstractConcurrentSelfTest(CacheMode cacheMode, CacheAtomicityMode atomicityMode) { + DynamicIndexAbstractConcurrentSelfTest(CacheMode cacheMode, CacheAtomicityMode atomicityMode) { this.cacheMode = cacheMode; this.atomicityMode = atomicityMode; } @@ -93,10 +92,11 @@ protected DynamicIndexAbstractConcurrentSelfTest(CacheMode cacheMode, CacheAtomi } /** {@inheritDoc} */ + @SuppressWarnings("ConstantConditions") @Override protected void afterTest() throws Exception { GridQueryProcessor.idxCls = null; - for (T2 block : BLOCKS.values()) + for (T3 block : BLOCKS.values()) block.get1().countDown(); BLOCKS.clear(); @@ -146,14 +146,14 @@ public void testCoordinatorChange() throws Exception { put(srv1, 0, KEY_AFTER); // Test migration between normal servers. - blockIndexing(srv1Id); + CountDownLatch idxLatch = blockIndexing(srv1Id); QueryIndex idx1 = index(IDX_NAME_1, field(FIELD_NAME_1)); IgniteInternalFuture idxFut1 = queryProcessor(cli).dynamicIndexCreate(CACHE_NAME, CACHE_NAME, TBL_NAME, idx1, false); - Thread.sleep(100); + idxLatch.countDown(); //srv1.close(); Ignition.stop(srv1.name(), true); @@ -167,14 +167,14 @@ public void testCoordinatorChange() throws Exception { assertSqlSimpleData(SQL_SIMPLE_FIELD_1, KEY_AFTER - SQL_ARG_1); // Test migration from normal server to non-affinity server. - blockIndexing(srv2Id); + idxLatch = blockIndexing(srv2Id); QueryIndex idx2 = index(IDX_NAME_2, field(aliasUnescaped(FIELD_NAME_2))); IgniteInternalFuture idxFut2 = queryProcessor(cli).dynamicIndexCreate(CACHE_NAME, CACHE_NAME, TBL_NAME, idx2, false); - Thread.sleep(100); + idxLatch.countDown(); //srv2.close(); Ignition.stop(srv2.name(), true); @@ -202,7 +202,7 @@ public void testOperationChaining() throws Exception { createSqlCache(srv1); - blockIndexing(srv1); + CountDownLatch idxLatch = blockIndexing(srv1); QueryIndex idx1 = index(IDX_NAME_1, field(FIELD_NAME_1)); QueryIndex idx2 = index(IDX_NAME_2, field(aliasUnescaped(FIELD_NAME_2))); @@ -229,7 +229,7 @@ public void testOperationChaining() throws Exception { assertIndex(CACHE_NAME, TBL_NAME, IDX_NAME_1, field(FIELD_NAME_1)); assertIndex(CACHE_NAME, TBL_NAME, IDX_NAME_2, field(aliasUnescaped(FIELD_NAME_2))); - Thread.sleep(100); + idxLatch.countDown(); put(srv1, 0, KEY_AFTER); @@ -250,7 +250,7 @@ public void testNodeJoinOnPendingOperation() throws Exception { createSqlCache(srv1); - blockIndexing(srv1); + CountDownLatch idxLatch = blockIndexing(srv1); QueryIndex idx = index(IDX_NAME_1, field(FIELD_NAME_1)); @@ -267,7 +267,7 @@ public void testNodeJoinOnPendingOperation() throws Exception { idxFut.get(); - Thread.sleep(100L); + idxLatch.countDown(); assertIndex(CACHE_NAME, TBL_NAME, IDX_NAME_1, field(FIELD_NAME_1)); @@ -391,15 +391,16 @@ public void testConcurrentRebalance() throws Exception { put(srv1, 0, LARGE_CACHE_SIZE); // Start index operation in blocked state. - blockIndexing(srv1); - blockIndexing(srv2); + CountDownLatch idxLatch1 = blockIndexing(srv1); + CountDownLatch idxLatch2 = blockIndexing(srv2); QueryIndex idx = index(IDX_NAME_1, field(FIELD_NAME_1)); final IgniteInternalFuture idxFut = queryProcessor(srv1).dynamicIndexCreate(CACHE_NAME, CACHE_NAME, TBL_NAME, idx, false); - Thread.sleep(100); + idxLatch1.countDown(); + idxLatch2.countDown(); // Start two more nodes and unblock index operation in the middle. Ignition.start(serverConfiguration(3)); @@ -435,19 +436,19 @@ public void testConcurrentCacheDestroy() throws Exception { Ignite cli = Ignition.start(clientConfiguration(4)); // Start cache and populate it with data. - IgniteCache cache = createSqlCache(cli); + createSqlCache(cli); put(cli, KEY_AFTER); // Start index operation and block it on coordinator. - blockIndexing(srv1); + CountDownLatch idxLatch = blockIndexing(srv1); QueryIndex idx = index(IDX_NAME_1, field(FIELD_NAME_1)); final IgniteInternalFuture idxFut = queryProcessor(srv1).dynamicIndexCreate(CACHE_NAME, CACHE_NAME, TBL_NAME, idx, false); - Thread.sleep(100); + idxLatch.countDown(); // Destroy cache (drop table). destroySqlCache(cli); @@ -967,10 +968,10 @@ public void testConcurrentOperationsAndCacheStartStopMultithreaded() throws Exce * @param node Node. */ @SuppressWarnings("SuspiciousMethodCalls") - private static void blockIndexing(Ignite node) { + private static CountDownLatch blockIndexing(Ignite node) { UUID nodeId = ((IgniteEx)node).localNode().id(); - blockIndexing(nodeId); + return blockIndexing(nodeId); } /** @@ -979,10 +980,14 @@ private static void blockIndexing(Ignite node) { * @param nodeId Node. */ @SuppressWarnings("SuspiciousMethodCalls") - private static void blockIndexing(UUID nodeId) { + private static CountDownLatch blockIndexing(UUID nodeId) { assertFalse(BLOCKS.contains(nodeId)); - BLOCKS.put(nodeId, new T2<>(new CountDownLatch(1), new AtomicBoolean())); + CountDownLatch idxLatch = new CountDownLatch(1); + + BLOCKS.put(nodeId, new T3<>(new CountDownLatch(1), new AtomicBoolean(), idxLatch)); + + return idxLatch; } /** @@ -1001,8 +1006,9 @@ private static void unblockIndexing(Ignite node) { * * @param nodeId Node ID. */ + @SuppressWarnings("ConstantConditions") private static void unblockIndexing(UUID nodeId) { - T2 blocker = BLOCKS.remove(nodeId); + T3 blocker = BLOCKS.remove(nodeId); assertNotNull(blocker); @@ -1014,12 +1020,15 @@ private static void unblockIndexing(UUID nodeId) { * * @param nodeId Node ID. */ + @SuppressWarnings("ConstantConditions") private static void awaitIndexing(UUID nodeId) { - T2 blocker = BLOCKS.get(nodeId); + T3 blocker = BLOCKS.get(nodeId); if (blocker != null) { assertTrue(blocker.get2().compareAndSet(false, true)); + blocker.get3().countDown(); + while (true) { try { blocker.get1().await(); From 0357c51c68540fc14de83ea36f0cfdc7ec987b80 Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 5 Jul 2017 16:59:23 +0300 Subject: [PATCH 063/155] ignite-2.1 Do not use 'compress' flag for GridDhtPartitionsFullMessage send in discovery message. --- .../cache/GridCachePartitionExchangeManager.java | 7 +++++-- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 8 +++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 22345d27dc4c6..ac062959aa971 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -955,7 +955,7 @@ private void refreshPartitions() { * @param nodes Nodes. */ private void sendAllPartitions(Collection nodes) { - GridDhtPartitionsFullMessage m = createPartitionsFullMessage(null, null, null, null); + GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, null, null, null, null); if (log.isDebugEnabled()) log.debug("Sending all partitions [nodeIds=" + U.nodeIds(nodes) + ", msg=" + m + ']'); @@ -978,11 +978,14 @@ private void sendAllPartitions(Collection nodes) { } /** + * @param compress {@code True} if possible to compress message (properly work only if prepareMarshall/ + * finishUnmarshall methods are called). * @param exchId Non-null exchange ID if message is created for exchange. * @param lastVer Last version. * @return Message. */ public GridDhtPartitionsFullMessage createPartitionsFullMessage( + boolean compress, @Nullable final GridDhtPartitionExchangeId exchId, @Nullable GridCacheVersion lastVer, @Nullable IgniteDhtPartitionHistorySuppliersMap partHistSuppliers, @@ -995,7 +998,7 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( partsToReload ); - m.compress(true); + m.compress(compress); final Map> dupData = new HashMap<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index cea758a2cd68b..215110190db27 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -1259,12 +1259,14 @@ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException } /** + * @param compress Message compress flag. * @return Message. */ - private GridDhtPartitionsFullMessage createPartitionsMessage() { + private GridDhtPartitionsFullMessage createPartitionsMessage(boolean compress) { GridCacheVersion last = lastVer.get(); GridDhtPartitionsFullMessage m = cctx.exchange().createPartitionsFullMessage( + compress, exchangeId(), last != null ? last : cctx.versions().last(), partHistSuppliers, @@ -1281,7 +1283,7 @@ private GridDhtPartitionsFullMessage createPartitionsMessage() { * @throws IgniteCheckedException If failed. */ private void sendAllPartitions(Collection nodes) throws IgniteCheckedException { - GridDhtPartitionsFullMessage m = createPartitionsMessage(); + GridDhtPartitionsFullMessage m = createPartitionsMessage(true); assert !nodes.contains(cctx.localNode()); @@ -1613,7 +1615,7 @@ private void onAffinityInitialized(IgniteInternalFuture>> assignmentChange = fut.get(); - GridDhtPartitionsFullMessage m = createPartitionsMessage(); + GridDhtPartitionsFullMessage m = createPartitionsMessage(false); CacheAffinityChangeMessage msg = new CacheAffinityChangeMessage(exchId, m, assignmentChange); From 7504b38a603e593fbb190e9e1e9da262cbb8f855 Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 5 Jul 2017 17:07:54 +0300 Subject: [PATCH 064/155] ignite-2.1 Do not use 'compress' flag for GridDhtPartitionsFullMessage send in discovery message. --- .../processors/cache/GridCachePartitionExchangeManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index ac062959aa971..b3cbd174bac4e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -1018,7 +1018,7 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( if (locMap != null) { addFullPartitionsMap(m, dupData, - true, + compress, grp.groupId(), locMap, affCache.similarAffinityKey()); @@ -1036,7 +1036,7 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( if (map != null) { addFullPartitionsMap(m, dupData, - true, + compress, top.groupId(), map, top.similarAffinityKey()); From ad42f6205b3956dca0ee54e85ce385e6591ec7a9 Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 5 Jul 2017 17:41:30 +0300 Subject: [PATCH 065/155] ignite-2.1 --- .../yardstick/IgniteBenchmarkArguments.java | 46 ++++ .../cache/IgniteStreamerBenchmark.java | 234 ++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteStreamerBenchmark.java diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java index d3b860c11907f..5ec6c54c53551 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java @@ -18,6 +18,7 @@ package org.apache.ignite.yardstick; import com.beust.jcommander.Parameter; +import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.configuration.MemoryConfiguration; import org.apache.ignite.configuration.PersistentStoreConfiguration; @@ -28,6 +29,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.ignite.yardstick.cache.IgniteStreamerBenchmark; import org.jetbrains.annotations.Nullable; /** @@ -222,6 +224,22 @@ public class IgniteBenchmarkArguments { @Parameter(names = {"-pds", "--persistentStore"}, description = "Persistent store flag") private boolean persistentStoreEnabled; + /** */ + @Parameter(names = {"-stcp", "--streamerCachesPrefix"}, description = "Cache name prefix for streamer benchmark") + private String streamerCachesPrefix = "streamer"; + + /** */ + @Parameter(names = {"-stci", "--streamerCachesIndex"}, description = "First cache index for streamer benchmark") + private int streamerCacheIndex; + + /** */ + @Parameter(names = {"-stcc", "--streamerConcCaches"}, description = "Number of concurrently loaded caches for streamer benchmark") + private int streamerConcurrentCaches = 1; + + /** */ + @Parameter(names = {"-stbs", "--streamerBufSize"}, description = "Data streamer buffer size") + private int streamerBufSize = IgniteDataStreamer.DFLT_PER_NODE_BUFFER_SIZE; + /** * @return {@code True} if need set {@link PersistentStoreConfiguration}. */ @@ -552,6 +570,34 @@ public String description() { "-txc=" + txConcurrency + "-rd=" + restartDelay + "-rs=" + restartSleep; } + /** + * @return Cache name prefix for caches to be used in {@link IgniteStreamerBenchmark}. + */ + public String streamerCachesPrefix() { + return streamerCachesPrefix; + } + + /** + * @return First cache index for {@link IgniteStreamerBenchmark}. + */ + public int streamerCacheIndex() { + return streamerCacheIndex; + } + + /** + * @return Number of concurrently loaded caches for {@link IgniteStreamerBenchmark}. + */ + public int streamerConcurrentCaches() { + return streamerConcurrentCaches; + } + + /** + * @return Streamer buffer size {@link IgniteStreamerBenchmark} (see {@link IgniteDataStreamer#perNodeBufferSize()}. + */ + public int streamerBufferSize() { + return streamerBufSize; + } + /** {@inheritDoc} */ @Override public String toString() { return GridToStringBuilder.toString(IgniteBenchmarkArguments.class, this); diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteStreamerBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteStreamerBenchmark.java new file mode 100644 index 0000000000000..9e253e1e343f1 --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteStreamerBenchmark.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.yardstick.IgniteAbstractBenchmark; +import org.apache.ignite.yardstick.cache.model.SampleValue; +import org.yardstickframework.BenchmarkConfiguration; +import org.yardstickframework.BenchmarkUtils; + +/** + * + */ +public class IgniteStreamerBenchmark extends IgniteAbstractBenchmark { + /** */ + private List cacheNames; + + /** */ + private ExecutorService executor; + + /** */ + private int entries; + + /** {@inheritDoc} */ + @Override public void setUp(BenchmarkConfiguration cfg) throws Exception { + super.setUp(cfg); + + entries = args.range(); + + if (entries <= 0) + throw new IllegalArgumentException("Invalid number of entries: " + entries); + + if (cfg.threads() != 1) + throw new IllegalArgumentException("IgniteStreamerBenchmark should be run with single thread. " + + "Internally it starts multiple threads."); + + String cacheNamePrefix = args.streamerCachesPrefix(); + + if (cacheNamePrefix == null || cacheNamePrefix.isEmpty()) + throw new IllegalArgumentException("Streamer caches prefix not set."); + + List caches = new ArrayList<>(); + + for (String cacheName : ignite().cacheNames()) { + if (cacheName.startsWith(cacheNamePrefix)) + caches.add(cacheName); + } + + if (caches.isEmpty()) + throw new IllegalArgumentException("Failed to find for IgniteStreamerBenchmark caches " + + "starting with '" + cacheNamePrefix + "'"); + + BenchmarkUtils.println("Found " + caches.size() + " caches for IgniteStreamerBenchmark: " + caches); + + if (args.streamerCacheIndex() >= caches.size()) { + throw new IllegalArgumentException("Invalid streamer cache index: " + args.streamerCacheIndex() + + ", there are only " + caches.size() + " caches."); + } + + if (args.streamerCacheIndex() + args.streamerConcurrentCaches() > caches.size()) { + throw new IllegalArgumentException("There are no enough caches [cacheIndex=" + args.streamerCacheIndex() + + ", concurrentCaches=" + args.streamerConcurrentCaches() + + ", totalCaches=" + caches.size() + ']'); + } + + Collections.sort(caches); + + cacheNames = new ArrayList<>(caches.subList(args.streamerCacheIndex(), + args.streamerCacheIndex() + args.streamerConcurrentCaches())); + + executor = Executors.newFixedThreadPool(args.streamerConcurrentCaches()); + + BenchmarkUtils.println("IgniteStreamerBenchmark start [cacheIndex=" + args.streamerCacheIndex() + + ", concurrentCaches=" + args.streamerConcurrentCaches() + + ", entries=" + entries + + ", bufferSize=" + args.streamerBufferSize() + + ", cachesToUse=" + cacheNames + ']'); + + if (cfg.warmup() > 0) { + BenchmarkUtils.println("IgniteStreamerBenchmark start warmup [warmupTimeMillis=" + cfg.warmup() + ']'); + + final long warmupEnd = System.currentTimeMillis() + cfg.warmup(); + + final AtomicBoolean stop = new AtomicBoolean(); + + try { + List> futs = new ArrayList<>(); + + for (final String cacheName : cacheNames) { + futs.add(executor.submit(new Callable() { + @Override public Void call() throws Exception { + Thread.currentThread().setName("streamer-" + cacheName); + + BenchmarkUtils.println("IgniteStreamerBenchmark start warmup for cache " + + "[name=" + cacheName + ']'); + + final int KEYS = Math.min(100_000, entries); + + int key = 1; + + try (IgniteDataStreamer streamer = ignite().dataStreamer(cacheName)) { + streamer.perNodeBufferSize(args.streamerBufferSize()); + + while (System.currentTimeMillis() < warmupEnd && !stop.get()) { + for (int i = 0; i < 10; i++) { + streamer.addData(-key++, new SampleValue(key)); + + if (key >= KEYS) + key = 1; + } + + streamer.flush(); + } + } + + BenchmarkUtils.println("IgniteStreamerBenchmark finished warmup for cache " + + "[name=" + cacheName + ']'); + + return null; + } + })); + } + + for (Future fut : futs) + fut.get(); + } + finally { + stop.set(true); + } + } + } + + /** {@inheritDoc} */ + @Override public boolean test(Map map) throws Exception { + BenchmarkUtils.println("IgniteStreamerBenchmark start test."); + + long start = System.currentTimeMillis(); + + final AtomicBoolean stop = new AtomicBoolean(); + + try { + List> futs = new ArrayList<>(); + + for (final String cacheName : cacheNames) { + futs.add(executor.submit(new Callable() { + @Override public Void call() throws Exception { + Thread.currentThread().setName("streamer-" + cacheName); + + long start = System.currentTimeMillis(); + + BenchmarkUtils.println("IgniteStreamerBenchmark start load cache [name=" + cacheName + ']'); + + try (IgniteDataStreamer streamer = ignite().dataStreamer(cacheName)) { + for (int i = 0; i < entries; i++) { + streamer.addData(i, new SampleValue(i)); + + if (i > 0 && i % 1000 == 0) { + if (stop.get()) + break; + + if (i % 100_000 == 0) { + BenchmarkUtils.println("IgniteStreamerBenchmark cache load progress [name=" + cacheName + + ", entries=" + i + + ", timeMillis=" + (System.currentTimeMillis() - start) + ']'); + } + } + } + } + + long time = System.currentTimeMillis() - start; + + BenchmarkUtils.println("IgniteStreamerBenchmark finished load cache [name=" + cacheName + + ", entries=" + entries + + ", bufferSize=" + args.streamerBufferSize() + + ", totalTimeMillis=" + time + ']'); + + return null; + } + })); + } + + for (Future fut : futs) + fut.get(); + } + finally { + stop.set(true); + } + + long time = System.currentTimeMillis() - start; + + BenchmarkUtils.println("IgniteStreamerBenchmark finished [totalTimeMillis=" + time + + ", entries=" + entries + + ", bufferSize=" + args.streamerBufferSize() + ']'); + + for (String cacheName : cacheNames) { + BenchmarkUtils.println("Cache size [cacheName=" + cacheName + + ", size=" + ignite().cache(cacheName).size() + ']'); + } + + return false; + } + + /** {@inheritDoc} */ + @Override public void tearDown() throws Exception { + if (executor != null) + executor.shutdown(); + + super.tearDown(); + } +} From 29d532e8be971ccac40ece00fc84a6a6bffdad0f Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Wed, 5 Jul 2017 18:51:27 +0300 Subject: [PATCH 066/155] IGNITE-5576: Added Compute::Run() for C++ (cherry picked from commit 80c95ff79f344daf1fca3f094733a24bac2a218d) --- .../core-test/config/cache-query-default.xml | 18 ++ .../cpp/core-test/src/compute_test.cpp | 176 ++++++++++++++++++ .../cpp/core/include/ignite/compute/compute.h | 35 +++- .../ignite/impl/compute/compute_impl.h | 42 +++++ .../ignite/impl/compute/compute_job_holder.h | 73 ++++++++ .../ignite/impl/compute/compute_job_result.h | 112 +++++++++++ .../ignite/impl/compute/compute_task_holder.h | 85 +++++++++ 7 files changed, 539 insertions(+), 2 deletions(-) diff --git a/modules/platforms/cpp/core-test/config/cache-query-default.xml b/modules/platforms/cpp/core-test/config/cache-query-default.xml index 38636e5a02499..16f601d06a5b6 100644 --- a/modules/platforms/cpp/core-test/config/cache-query-default.xml +++ b/modules/platforms/cpp/core-test/config/cache-query-default.xml @@ -94,6 +94,12 @@ + + + + + + @@ -115,6 +121,12 @@ + + + + + + @@ -132,6 +144,12 @@ + + + + + + diff --git a/modules/platforms/cpp/core-test/src/compute_test.cpp b/modules/platforms/cpp/core-test/src/compute_test.cpp index d3b1183a8781b..8c57ef17404f9 100644 --- a/modules/platforms/cpp/core-test/src/compute_test.cpp +++ b/modules/platforms/cpp/core-test/src/compute_test.cpp @@ -146,6 +146,49 @@ struct Func2 : ComputeFunc IgniteError err; }; +struct Func3 : ComputeFunc +{ + Func3() : + a(), b(), err() + { + // No-op. + } + + Func3(int32_t a, int32_t b) : + a(a), b(b), err() + { + // No-op. + } + + Func3(IgniteError err) : + a(), b(), err(err) + { + // No-op. + } + + virtual void Call() + { + boost::this_thread::sleep_for(boost::chrono::milliseconds(200)); + + if (err.GetCode() != IgniteError::IGNITE_SUCCESS) + throw err; + + std::stringstream tmp; + + tmp << a << '.' << b; + + res = tmp.str(); + } + + int32_t a; + int32_t b; + IgniteError err; + + static std::string res; +}; + +std::string Func3::res; + namespace ignite { namespace binary @@ -235,6 +278,49 @@ namespace ignite dst.err = reader.ReadObject("err"); } }; + + template<> + struct BinaryType + { + static int32_t GetTypeId() + { + return GetBinaryStringHashCode("Func3"); + } + + static void GetTypeName(std::string& dst) + { + dst = "Func3"; + } + + static int32_t GetFieldId(const char* name) + { + return GetBinaryStringHashCode(name); + } + + static bool IsNull(const Func3& obj) + { + return false; + } + + static void GetNull(Func3& dst) + { + dst = Func3(0, 0); + } + + static void Write(BinaryWriter& writer, const Func3& obj) + { + writer.WriteInt32("a", obj.a); + writer.WriteInt32("b", obj.b); + writer.WriteObject("err", obj.err); + } + + static void Read(BinaryReader& reader, Func3& dst) + { + dst.a = reader.ReadInt32("a"); + dst.b = reader.ReadInt32("b"); + dst.err = reader.ReadObject("err"); + } + }; } } @@ -244,6 +330,7 @@ IGNITE_EXPORTED_CALL void IgniteModuleInit1(IgniteBindingContext& context) binding.RegisterComputeFunc(); binding.RegisterComputeFunc(); + binding.RegisterComputeFunc(); } BOOST_FIXTURE_TEST_SUITE(ComputeTestSuite, ComputeTestSuiteFixture) @@ -334,4 +421,93 @@ BOOST_AUTO_TEST_CASE(IgniteCallTestRemoteError) BOOST_CHECK_EXCEPTION(res.GetValue(), IgniteError, IsTestError); } +BOOST_AUTO_TEST_CASE(IgniteRunSyncLocal) +{ + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Running"); + compute.Run(Func3(8, 5)); + + BOOST_CHECK_EQUAL(Func3::res, "8.5"); +} + +BOOST_AUTO_TEST_CASE(IgniteRunAsyncLocal) +{ + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Running"); + Future res = compute.RunAsync(Func3(312, 245)); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECKPOINT("Waiting with timeout"); + res.WaitFor(100); + + BOOST_CHECK(!res.IsReady()); + + res.GetValue(); + + BOOST_CHECK_EQUAL(Func3::res, "312.245"); +} + +BOOST_AUTO_TEST_CASE(IgniteRunSyncLocalError) +{ + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Running"); + + BOOST_CHECK_EXCEPTION(compute.Run(Func3(MakeTestError())), IgniteError, IsTestError); +} + +BOOST_AUTO_TEST_CASE(IgniteRunAsyncLocalError) +{ + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Running"); + Future res = compute.RunAsync(Func3(MakeTestError())); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECKPOINT("Waiting with timeout"); + res.WaitFor(100); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECK_EXCEPTION(res.GetValue(), IgniteError, IsTestError); +} + +BOOST_AUTO_TEST_CASE(IgniteRunTestRemote) +{ + Ignite node2 = MakeNode("ComputeNode2"); + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Running"); + compute.CallAsync(Func2(8, 5)); + + compute.Run(Func3(42, 24)); + + BOOST_CHECK_EQUAL(Func3::res, "42.24"); +} + +BOOST_AUTO_TEST_CASE(IgniteRunTestRemoteError) +{ + Ignite node2 = MakeNode("ComputeNode2"); + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Running"); + compute.CallAsync(Func2(8, 5)); + + Future res = compute.RunAsync(Func3(MakeTestError())); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECKPOINT("Waiting with timeout"); + res.WaitFor(100); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECK_EXCEPTION(res.GetValue(), IgniteError, IsTestError); +} + + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/modules/platforms/cpp/core/include/ignite/compute/compute.h b/modules/platforms/cpp/core/include/ignite/compute/compute.h index b07956977d7d2..75c8c85145e1e 100644 --- a/modules/platforms/cpp/core/include/ignite/compute/compute.h +++ b/modules/platforms/cpp/core/include/ignite/compute/compute.h @@ -94,7 +94,7 @@ namespace ignite * @tparam R Call return type. BinaryType should be specialized for * the type if it is not primitive. Should not be void. For * non-returning methods see Compute::Run(). - * @tparam F Compute function type. Should implement ComputeFunc + * @tparam F Compute function type. Should implement ComputeFunc * class. * @param func Compute function to call. * @return Computation result. @@ -113,7 +113,7 @@ namespace ignite * @tparam R Call return type. BinaryType should be specialized for * the type if it is not primitive. Should not be void. For * non-returning methods see Compute::Run(). - * @tparam F Compute function type. Should implement ComputeFunc + * @tparam F Compute function type. Should implement ComputeFunc * class. * @param func Compute function to call. * @return Future that can be used to access computation result once @@ -126,6 +126,37 @@ namespace ignite return impl.Get()->CallAsync(func); } + /** + * Runs provided ComputeFunc on a node within the underlying cluster + * group. + * + * @tparam F Compute function type. Should implement ComputeFunc + * class. + * @param action Compute function to call. + * @throw IgniteError in case of error. + */ + template + void Run(const F& action) + { + return impl.Get()->RunAsync(action).GetValue(); + } + + /** + * Asyncronuously runs provided ComputeFunc on a node within the + * underlying cluster group. + * + * @tparam F Compute function type. Should implement ComputeFunc + * class. + * @param action Compute function to call. + * @return Future that can be used to wait for action to complete. + * @throw IgniteError in case of error. + */ + template + Future RunAsync(const F& action) + { + return impl.Get()->RunAsync(action); + } + private: /** Implementation. */ common::concurrent::SharedPointer impl; diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h index 389c571815d17..63f9a46199955 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h @@ -108,6 +108,48 @@ namespace ignite return promise.GetFuture(); } + /** + * Asyncronuously runs provided ComputeFunc on a node within + * the underlying cluster group. + * + * @tparam F Compute action type. Should implement ComputeAction + * class. + * @param action Compute action to call. + * @return Future that can be used to wait for action to complete. + * @throw IgniteError in case of error. + */ + template + Future RunAsync(const F& action) + { + common::concurrent::SharedPointer mem = GetEnvironment().AllocateMemory(); + interop::InteropOutputStream out(mem.Get()); + binary::BinaryWriterImpl writer(&out, GetEnvironment().GetTypeManager()); + + common::concurrent::SharedPointer job(new ComputeJobHolderImpl(action)); + + int64_t jobHandle = GetEnvironment().GetHandleRegistry().Allocate(job); + + ComputeTaskHolderImpl* taskPtr = new ComputeTaskHolderImpl(jobHandle); + common::concurrent::SharedPointer task(taskPtr); + + int64_t taskHandle = GetEnvironment().GetHandleRegistry().Allocate(task); + + writer.WriteInt64(taskHandle); + writer.WriteInt32(1); + writer.WriteInt64(jobHandle); + writer.WriteObject(action); + + out.Synchronize(); + + jobject target = InStreamOutObject(Operation::Unicast, *mem.Get()); + std::auto_ptr cancelable(new CancelableImpl(GetEnvironmentPointer(), target)); + + common::Promise& promise = taskPtr->GetPromise(); + promise.SetCancelTarget(cancelable); + + return promise.GetFuture(); + } + private: IGNITE_NO_COPY_ASSIGNMENT(ComputeImpl); }; diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_holder.h b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_holder.h index e218e36fccc26..9f35a111994a4 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_holder.h +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_holder.h @@ -132,6 +132,79 @@ namespace ignite /** Job. */ JobType job; }; + + /** + * Compute job holder. Internal class. + * Specialisation for void return type + * + * @tparam F Actual job type. + */ + template + class ComputeJobHolderImpl : public ComputeJobHolder + { + public: + typedef F JobType; + + /** + * Constructor. + * + * @param job Job. + */ + ComputeJobHolderImpl(JobType job) : + job(job) + { + // No-op. + } + + /** + * Destructor. + */ + virtual ~ComputeJobHolderImpl() + { + // No-op. + } + + const ComputeJobResult& GetResult() + { + return res; + } + + virtual void ExecuteLocal() + { + try + { + job.Call(); + res.SetResult(); + } + catch (const IgniteError& err) + { + res.SetError(err); + } + catch (const std::exception& err) + { + res.SetError(IgniteError(IgniteError::IGNITE_ERR_STD, err.what())); + } + catch (...) + { + res.SetError(IgniteError(IgniteError::IGNITE_ERR_UNKNOWN, + "Unknown error occurred during call.")); + } + } + + virtual void ExecuteRemote(binary::BinaryWriterImpl& writer) + { + ExecuteLocal(); + + res.Write(writer); + } + + private: + /** Result. */ + ComputeJobResult res; + + /** Job. */ + JobType job; + }; } } } diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_result.h b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_result.h index 5bcb762bbc3eb..087452259270c 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_result.h +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_result.h @@ -27,6 +27,8 @@ #include #include +#include +#include namespace ignite { @@ -154,6 +156,116 @@ namespace ignite /** Erorr. */ IgniteError err; }; + + /** + * Used to hold compute job result. + */ + template<> + class ComputeJobResult + { + public: + /** + * Default constructor. + */ + ComputeJobResult() : + err() + { + // No-op. + } + + /** + * Mark as complete. + */ + void SetResult() + { + err = IgniteError(); + } + + /** + * Set error. + * + * @param error Error to set. + */ + void SetError(const IgniteError error) + { + err = error; + } + + /** + * Set promise to a state which corresponds to result. + * + * @param promise Promise, which state to set. + */ + void SetPromise(common::Promise& promise) + { + if (err.GetCode() != IgniteError::IGNITE_SUCCESS) + promise.SetError(err); + else + promise.SetValue(); + } + + /** + * Write using writer. + * + * @param writer Writer. + */ + void Write(binary::BinaryWriterImpl& writer) + { + if (err.GetCode() != IgniteError::IGNITE_SUCCESS) + { + // Fail + writer.WriteBool(false); + + // Native Exception + writer.WriteBool(true); + + writer.WriteObject(err); + } + else + { + // Success + writer.WriteBool(true); + + writer.WriteNull(); + } + } + + /** + * Read using reader. + * + * @param reader Reader. + */ + void Read(binary::BinaryReaderImpl& reader) + { + bool success = reader.ReadBool(); + + if (success) + err = IgniteError(); + else + { + bool native = reader.ReadBool(); + + if (native) + err = reader.ReadObject(); + else + { + std::stringstream buf; + + buf << reader.ReadObject() << " : "; + buf << reader.ReadObject() << ", "; + buf << reader.ReadObject(); + + std::string msg = buf.str(); + + err = IgniteError(IgniteError::IGNITE_ERR_GENERIC, msg.c_str()); + } + } + } + + private: + /** Erorr. */ + IgniteError err; + }; } } } diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_task_holder.h b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_task_holder.h index bdd7513465cc8..f627f2792b334 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_task_holder.h +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_task_holder.h @@ -206,6 +206,91 @@ namespace ignite /** Task result promise. */ common::Promise promise; }; + + /** + * Compute task holder type-specific implementation. + */ + template + class ComputeTaskHolderImpl : public ComputeTaskHolder + { + public: + typedef F JobType; + + /** + * Constructor. + * + * @param handle Job handle. + */ + ComputeTaskHolderImpl(int64_t handle) : + ComputeTaskHolder(handle) + { + // No-op. + } + + /** + * Destructor. + */ + virtual ~ComputeTaskHolderImpl() + { + // No-op. + } + + /** + * Process local job result. + * + * @param job Job. + * @return Policy. + */ + virtual int32_t JobResultLocal(ComputeJobHolder& job) + { + typedef ComputeJobHolderImpl ActualComputeJobHolder; + + ActualComputeJobHolder& job0 = static_cast(job); + + res = job0.GetResult(); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Process remote job result. + * + * @param job Job. + * @param reader Reader for stream with result. + * @return Policy. + */ + virtual int32_t JobResultRemote(ComputeJobHolder& job, binary::BinaryReaderImpl& reader) + { + res.Read(reader); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Reduce results of related jobs. + */ + virtual void Reduce() + { + res.SetPromise(promise); + } + + /** + * Get result promise. + * + * @return Reference to result promise. + */ + common::Promise& GetPromise() + { + return promise; + } + + private: + /** Result. */ + ComputeJobResult res; + + /** Task result promise. */ + common::Promise promise; + }; } } } From 905e34d4eee40f9c31288eb9563812f6e9ab888d Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Wed, 5 Jul 2017 19:16:01 +0300 Subject: [PATCH 067/155] Added onActivate/onDeactivate callbacks for plugins --- .../cache/GridCacheSharedContext.java | 5 + .../org.apache.ignite.plugin.PluginProvider | 3 +- .../IgniteStandByClusterTest.java | 164 ++++++++++++++++++ 3 files changed, 171 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java index 9adca8d284942..40b263f599113 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java @@ -66,6 +66,7 @@ import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.marshaller.Marshaller; +import org.apache.ignite.plugin.PluginProvider; import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; @@ -223,6 +224,10 @@ public GridCacheSharedContext( stateAwareMgrs.add(dbMgr); stateAwareMgrs.add(snpMgr); + + for (PluginProvider prv : kernalCtx.plugins().allProviders()) + if (prv instanceof IgniteChangeGlobalStateSupport) + stateAwareMgrs.add(((IgniteChangeGlobalStateSupport)prv)); } /** diff --git a/modules/core/src/test/java/META-INF/services/org.apache.ignite.plugin.PluginProvider b/modules/core/src/test/java/META-INF/services/org.apache.ignite.plugin.PluginProvider index f03038669f034..5b6ed7d5cd603 100644 --- a/modules/core/src/test/java/META-INF/services/org.apache.ignite.plugin.PluginProvider +++ b/modules/core/src/test/java/META-INF/services/org.apache.ignite.plugin.PluginProvider @@ -1,2 +1,3 @@ org.apache.ignite.platform.plugin.PlatformTestPluginProvider -org.apache.ignite.spi.discovery.tcp.TestReconnectPluginProvider \ No newline at end of file +org.apache.ignite.spi.discovery.tcp.TestReconnectPluginProvider +org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteStandByClusterTest$StanByClusterTestProvider \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java index 2678e51ad49f8..30fff0843a18e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java @@ -17,24 +17,39 @@ package org.apache.ignite.internal.processors.cache.persistence.standbycluster; +import java.io.Serializable; import java.util.Arrays; import java.util.Collection; import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.PersistentStoreConfiguration; +import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; +import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.plugin.CachePluginContext; +import org.apache.ignite.plugin.CachePluginProvider; +import org.apache.ignite.plugin.ExtensionRegistry; +import org.apache.ignite.plugin.IgnitePlugin; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.plugin.PluginProvider; +import org.apache.ignite.plugin.PluginValidationException; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.Nullable; import org.junit.Assert; /** @@ -269,6 +284,58 @@ public void testRestartCluster() throws Exception { assertEquals(String.valueOf(i), cache2.get(i)); } + /** + * @throws Exception if fail. + */ + public void testActivateDeActivateCallbackForPluginProviders() throws Exception { + IgniteEx ig1 = startGrid(getConfiguration("node1")); + IgniteEx ig2 = startGrid(getConfiguration("node2")); + IgniteEx ig3 = startGrid(getConfiguration("node3")); + + assertTrue(!ig1.active()); + assertTrue(!ig2.active()); + assertTrue(!ig3.active()); + + ig1.active(true); + + checkPlugin(ig1,1,0); + checkPlugin(ig2,1,0); + checkPlugin(ig3,1,0); + + ig2.active(false); + + ig3.active(true); + + checkPlugin(ig1,2,1); + checkPlugin(ig2,2,1); + checkPlugin(ig3,2,1); + + ig1.active(false); + + ig2.active(true); + + checkPlugin(ig1,3,2); + checkPlugin(ig2,3,2); + checkPlugin(ig3,3,2); + + } + + /** + * @param ig ignite. + * @param act Expected activation counter. + * @param deAct Expected deActivation counter. + */ + private void checkPlugin(Ignite ig, int act, int deAct) { + IgnitePlugin pl = ig.plugin(StanByClusterTestProvider.NAME); + + assertNotNull(pl); + + StanByClusterTestProvider plugin = (StanByClusterTestProvider)pl; + + assertEquals(act, plugin.actCnt.get()); + assertEquals(deAct, plugin.deActCnt.get()); + } + /** * */ @@ -289,6 +356,103 @@ private NodeFilterIgnoreByName(String name) { } } + /** + * + */ + public static class StanByClusterTestProvider implements PluginProvider, IgnitePlugin, IgniteChangeGlobalStateSupport { + /** */ + static final String NAME = "StanByClusterTestProvider"; + + /** */ + final AtomicInteger actCnt = new AtomicInteger(); + + /** */ + final AtomicInteger deActCnt = new AtomicInteger(); + + /** {@inheritDoc} */ + @Override public String name() { + return NAME; + } + + /** {@inheritDoc} */ + @Override public String version() { + return "1.0"; + } + + /** {@inheritDoc} */ + @Override public String copyright() { + return null; + } + + /** {@inheritDoc} */ + @Override public void initExtensions( + PluginContext ctx, + ExtensionRegistry registry + ) throws IgniteCheckedException { + + } + + /** {@inheritDoc} */ + @Override public CachePluginProvider createCacheProvider(CachePluginContext ctx) { + return null; + } + + /** {@inheritDoc} */ + @Override public void start(PluginContext ctx) throws IgniteCheckedException { + + } + + /** {@inheritDoc} */ + @Override public void stop(boolean cancel) throws IgniteCheckedException { + + } + + /** {@inheritDoc} */ + @Override public void onIgniteStart() throws IgniteCheckedException { + + } + + /** {@inheritDoc} */ + @Override public void onIgniteStop(boolean cancel) { + + } + + /** {@inheritDoc} */ + @Nullable @Override public Serializable provideDiscoveryData(UUID nodeId) { + return null; + } + + /** {@inheritDoc} */ + @Override public void receiveDiscoveryData(UUID nodeId, Serializable data) { + + } + + /** {@inheritDoc} */ + @Override public void validateNewNode(ClusterNode node) throws PluginValidationException { + + } + + /** {@inheritDoc} */ + @Nullable @Override public Object createComponent(PluginContext ctx, Class cls) { + return null; + } + + /** {@inheritDoc} */ + @Override public IgnitePlugin plugin() { + return this; + } + + /** {@inheritDoc} */ + @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException { + actCnt.incrementAndGet(); + } + + /** {@inheritDoc} */ + @Override public void onDeActivate(GridKernalContext kctx) { + deActCnt.incrementAndGet(); + } + } + /** * */ From 69357c5d8be431aa51fc3add9e345807fe984fee Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Wed, 5 Jul 2017 19:24:47 +0300 Subject: [PATCH 068/155] IGNITE-5604 - Expand WAL iterator buffer if record size is greater than current buffer size - Fixes #2244. Signed-off-by: Alexey Goncharuk --- .../RendezvousAffinityFunction.java | 4 - .../wal/AbstractWalRecordsIterator.java | 14 ++- .../persistence/wal/ByteBufferExpander.java | 47 ++++++++ .../cache/persistence/wal/FileInput.java | 20 +++- .../wal/FileWriteAheadLogManager.java | 2 +- .../db/wal/IgniteWalRecoveryTest.java | 100 +++++++++++++----- 6 files changed, 146 insertions(+), 41 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferExpander.java diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java index 1bd0587c3b8a2..0fb20eefb102f 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java @@ -17,10 +17,6 @@ package org.apache.ignite.cache.affinity.rendezvous; -import java.io.Externalizable; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java index 7dc0a280e5f54..f4bace171ecca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -22,7 +22,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import org.apache.ignite.IgniteCheckedException; @@ -41,8 +40,8 @@ * Iterator over WAL segments. This abstract class provides most functionality for reading records in log. * Subclasses are to override segment switching functionality */ -public abstract class AbstractWalRecordsIterator extends GridCloseableIteratorAdapter> - implements WALIterator { +public abstract class AbstractWalRecordsIterator + extends GridCloseableIteratorAdapter> implements WALIterator { /** */ private static final long serialVersionUID = 0L; @@ -73,7 +72,7 @@ public abstract class AbstractWalRecordsIterator extends GridCloseableIteratorAd @NotNull private final RecordSerializer serializer; /** Utility buffer for reading records */ - private final ByteBuffer buf; + private final ByteBufferExpander buf; /** * @param log Logger @@ -85,15 +84,14 @@ protected AbstractWalRecordsIterator( @NotNull final IgniteLogger log, @NotNull final GridCacheSharedContext sharedCtx, @NotNull final RecordSerializer serializer, - final int bufSize) { + final int bufSize + ) { this.log = log; this.sharedCtx = sharedCtx; this.serializer = serializer; // Do not allocate direct buffer for iterator. - buf = ByteBuffer.allocate(bufSize); - buf.order(ByteOrder.nativeOrder()); - + buf = new ByteBufferExpander(bufSize, ByteOrder.nativeOrder()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferExpander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferExpander.java new file mode 100644 index 0000000000000..75d3a98c366d7 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferExpander.java @@ -0,0 +1,47 @@ +package org.apache.ignite.internal.processors.cache.persistence.wal; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * ByteBuffer wrapper for dynamically expand buffer size. + */ +public class ByteBufferExpander { + /** Byte buffer */ + private ByteBuffer buf; + + public ByteBufferExpander(int initSize, ByteOrder order) { + ByteBuffer buffer = ByteBuffer.allocate(initSize); + buffer.order(order); + + this.buf = buffer; + } + + /** + * Current byte buffer. + * + * @return Current byteBuffer. + */ + public ByteBuffer buffer() { + return buf; + } + + /** + * Expands current byte buffer to the requested size. + * + * @return ByteBuffer with requested size. + */ + public ByteBuffer expand(int size) { + ByteBuffer newBuf = ByteBuffer.allocate(size); + + newBuf.order(buf.order()); + + newBuf.put(buf); + + newBuf.flip(); + + buf = newBuf; + + return newBuf; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java index e2d7cbaab5568..00c7c02b6f95c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java @@ -42,6 +42,9 @@ public final class FileInput implements ByteBufferBackedDataInput { /** */ private long pos; + /** */ + private ByteBufferExpander expBuf; + /** * @param ch Channel to read from * @param buf Buffer for reading blocks of data into @@ -57,6 +60,16 @@ public FileInput(FileChannel ch, ByteBuffer buf) throws IOException { clearBuffer(); } + /** + * @param ch Channel to read from + * @param expBuf ByteBufferWrapper with ability expand buffer dynamically. + */ + public FileInput(FileChannel ch, ByteBufferExpander expBuf) throws IOException { + this(ch, expBuf.buffer()); + + this.expBuf = expBuf; + } + /** * Clear buffer. */ @@ -96,8 +109,11 @@ public void seek(long pos) throws IOException { if (available >= requested) return; - if (buf.capacity() < requested) - throw new IOException("Requested size is greater than buffer: " + requested); + if (buf.capacity() < requested) { + buf = expBuf.expand(requested); + + assert available == buf.remaining(); + } buf.compact(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 8993112b23735..162f43d674a7e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -2327,7 +2327,7 @@ private RecordsIterator( super(log, cctx, serializer, - Math.min(16 * tlbSize, psCfg.getWalRecordIteratorBufferSize())); + psCfg.getWalRecordIteratorBufferSize()); this.walWorkDir = walWorkDir; this.walArchiveDir = walArchiveDir; this.psCfg = psCfg; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java index 6b4907c872278..843fb5bf8c8b0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java @@ -136,6 +136,8 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest { PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration(); + pCfg.setWalRecordIteratorBufferSize(1024 * 1024); + if (logOnly) pCfg.setWalMode(WALMode.LOG_ONLY); @@ -180,46 +182,79 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest { * @throws Exception if failed. */ public void testWalBig() throws Exception { - try { - IgniteEx ignite = startGrid(1); + IgniteEx ignite = startGrid(1); - ignite.active(true); + ignite.active(true); - IgniteCache cache = ignite.cache("partitioned"); + IgniteCache cache = ignite.cache("partitioned"); - Random rnd = new Random(); + Random rnd = new Random(); - Map map = new HashMap<>(); + Map map = new HashMap<>(); - for (int i = 0; i < 10_000; i++) { - if (i % 1000 == 0) - X.println(" >> " + i); + for (int i = 0; i < 10_000; i++) { + if (i % 1000 == 0) + X.println(" >> " + i); - int k = rnd.nextInt(300_000); - IndexedObject v = new IndexedObject(rnd.nextInt(10_000)); + int k = rnd.nextInt(300_000); + IndexedObject v = new IndexedObject(rnd.nextInt(10_000)); - cache.put(k, v); - map.put(k, v); - } + cache.put(k, v); + map.put(k, v); + } - // Check. - for (Integer k : map.keySet()) - assertEquals(map.get(k), cache.get(k)); + // Check. + for (Integer k : map.keySet()) + assertEquals(map.get(k), cache.get(k)); - stopGrid(1); + stopGrid(1); - ignite = startGrid(1); + ignite = startGrid(1); - ignite.active(true); + ignite.active(true); - cache = ignite.cache("partitioned"); + cache = ignite.cache("partitioned"); - // Check. - for (Integer k : map.keySet()) - assertEquals(map.get(k), cache.get(k)); + // Check. + for (Integer k : map.keySet()) + assertEquals(map.get(k), cache.get(k)); + } + + /** + * @throws Exception if failed. + */ + public void testWalBigObjectNodeCancel() throws Exception { + final int MAX_SIZE_POWER = 21; + + IgniteEx ignite = startGrid(1); + + ignite.active(true); + + IgniteCache cache = ignite.cache("partitioned"); + + for (int i = 0; i < MAX_SIZE_POWER; ++i) { + int size = 1 << i; + + cache.put("key_" + i, createTestData(size)); } - finally { - stopAllGrids(); + + stopGrid(1, true); + + ignite = startGrid(1); + + ignite.active(true); + + cache = ignite.cache("partitioned"); + + // Check. + for (int i = 0; i < MAX_SIZE_POWER; ++i) { + int size = 1 << i; + + int[] data = createTestData(size); + + int[] val = (int[])cache.get("key_" + i); + + assertTrue("Invalid data. [key=key_" + i + ']', Arrays.equals(data, val)); } } @@ -976,6 +1011,19 @@ else if (rec instanceof PageDeltaRecord) { } } + /** + * @param size Size of data. + * @return Test data. + */ + private int[] createTestData(int size) { + int[] data = new int[size]; + + for (int d = 0; d < size; ++d) + data[d] = d; + + return data; + } + /** * */ From d82b2b81a487e244584a7101c6d0e3bc0ad38151 Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Wed, 5 Jul 2017 20:05:52 +0300 Subject: [PATCH 069/155] Code style --- ...FinishSnapshotOperationAckDiscoveryMessage.java | 8 +++----- .../cache/persistence/DbCheckpointListener.java | 14 +++++++++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java index 83c512a31479c..f6758e08d4ad6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.pagemem.snapshot; import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; +import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; @@ -77,10 +78,7 @@ public boolean success() { /** {@inheritDoc} */ @Override public String toString() { - return "FinishSnapshotOperationAckDiscoveryMessage{" + - "id=" + id + - ", opId=" + opId + - ", success=" + success + - '}'; + return S.toString(FinishSnapshotOperationAckDiscoveryMessage.class, this, + "id", id, "opId", opId, "success", success); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java index daaccff5fc78a..0b28b6ab4d546 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java @@ -25,16 +25,28 @@ * */ public interface DbCheckpointListener { + /** + * Context with information about current snapshots. + */ public interface Context { + /** + * + */ public boolean nextSnapshot(); + /** + * + */ public Map, T2> partitionStatMap(); + /** + * @param cacheOrGrpName Cache or group name. + */ public boolean needToSnapshot(String cacheOrGrpName); } /** * @throws IgniteCheckedException If failed. */ - public void onCheckpointBegin(Context context) throws IgniteCheckedException; + public void onCheckpointBegin(Context ctx) throws IgniteCheckedException; } From dd30e584afd841f0fa0923cefbe49bee2fa1ede1 Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 5 Jul 2017 22:07:04 +0300 Subject: [PATCH 070/155] Fixed missing SUID. --- .../internal/pagemem/snapshot/SnapshotCheckParameters.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java index e95e79d138148..58cb2406a7240 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.ignite.internal.pagemem.snapshot; import java.io.File; @@ -25,6 +26,9 @@ * Tuple for passing optional parameters of {@link SnapshotOperationType#CHECK}. */ public class SnapshotCheckParameters implements Serializable { + /** */ + private static final long serialVersionUID = 0L; + /** Optional paths. */ private final Collection optionalPaths; From 907d4a8e04ee96b78d4e53db69acf13d00798c91 Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 6 Jul 2017 11:11:38 +0300 Subject: [PATCH 071/155] 2.1 Removed extra add in pendingMsgs --- .../ignite/internal/processors/cache/GridCacheIoManager.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java index f9d1114fd8515..1d2be1302e124 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java @@ -244,11 +244,6 @@ else if (desc.receivedFromStartVersion() != null) final int stripe = curThread instanceof IgniteThread ? ((IgniteThread)curThread).stripe() : -1; - synchronized (pendingMsgs) { - if (pendingMsgs.size() < 100) - pendingMsgs.add(cacheMsg); - } - fut.listen(new CI1>() { @Override public void apply(IgniteInternalFuture t) { Runnable c = new Runnable() { From 740b0b2bdb37154857363c1d94ec88d867bd8b65 Mon Sep 17 00:00:00 2001 From: Sergey Chugunov Date: Thu, 6 Jul 2017 12:20:55 +0300 Subject: [PATCH 072/155] IGNITE-5401 Fix in MarshallerConextImpl. This closes #2205. --- .../JdbcAbstractDmlStatementSelfTest.java | 12 + .../JdbcDynamicIndexAbstractSelfTest.java | 2 - .../JdbcThinDynamicIndexAbstractSelfTest.java | 2 - .../internal/MarshallerContextImpl.java | 10 +- .../GridMarshallerMappingProcessor.java | 16 +- ...eMarshallerCacheClassNameConflictTest.java | 11 +- .../IgniteMarshallerCacheFSRestoreTest.java | 217 ++++++++++++++++++ .../testsuites/IgniteBasicTestSuite.java | 2 + 8 files changed, 248 insertions(+), 24 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java index a001eb3f0628d..f220b475c3d50 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX; @@ -139,6 +140,17 @@ protected String getCfgUrl() { conn.close(); assertTrue(conn.isClosed()); + + cleanUpWorkingDir(); + } + + /** + * Clean up working directory. + */ + private void cleanUpWorkingDir() throws Exception { + String workDir = U.defaultWorkDirectory(); + + deleteRecursively(U.resolveWorkDirectory(workDir, "marshaller", false)); } /** diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDynamicIndexAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDynamicIndexAbstractSelfTest.java index d4da1f3ac9df3..7bbda6f0380d7 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDynamicIndexAbstractSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDynamicIndexAbstractSelfTest.java @@ -242,8 +242,6 @@ public void testDropMissingIndexIfExists() throws SQLException { * Test that changes in cache affect index, and vice versa. */ public void testIndexState() throws SQLException { - fail("https://issues.apache.org/jira/browse/IGNITE-5373"); - IgniteCache cache = cache(); assertSize(3); diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinDynamicIndexAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinDynamicIndexAbstractSelfTest.java index 7404ebd92cdd8..3f762fc60ed81 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinDynamicIndexAbstractSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinDynamicIndexAbstractSelfTest.java @@ -254,8 +254,6 @@ public void testDropMissingIndexIfExists() throws SQLException { * @throws SQLException If failed. */ public void testIndexState() throws SQLException { - fail("https://issues.apache.org/jira/browse/IGNITE-5373"); - IgniteCache cache = cache(); assertSize(3); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java index cad06c322b70b..6f1550792fd0d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java @@ -316,18 +316,14 @@ private IgniteCheckedException duplicateIdException( * * @param item type mapping to propose * @return null if cache doesn't contain any mappings for given (platformId, typeId) pair, - * previous class name otherwise. + * previous {@link MappedName mapped name} otherwise. */ - public String onMappingProposed(MarshallerMappingItem item) { + public MappedName onMappingProposed(MarshallerMappingItem item) { ConcurrentMap cache = getCacheFor(item.platformId()); MappedName newName = new MappedName(item.className(), false); - MappedName oldName; - if ((oldName = cache.putIfAbsent(item.typeId(), newName)) == null) - return null; - else - return oldName.className(); + return cache.putIfAbsent(item.typeId(), newName); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/GridMarshallerMappingProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/GridMarshallerMappingProcessor.java index c23d068ccd95e..df0c7202dcb88 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/GridMarshallerMappingProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/GridMarshallerMappingProcessor.java @@ -113,7 +113,7 @@ public GridMarshallerMappingProcessor(GridKernalContext ctx) { marshallerCtx.onMarshallerProcessorStarted(ctx, transport); - discoMgr.setCustomEventListener(MappingProposedMessage.class, new MarshallerMappingExchangeListener()); + discoMgr.setCustomEventListener(MappingProposedMessage.class, new MappingProposedListener()); discoMgr.setCustomEventListener(MappingAcceptedMessage.class, new MappingAcceptedListener()); @@ -233,7 +233,7 @@ private final class MissingMappingResponseListener implements GridMessageListene /** * */ - private final class MarshallerMappingExchangeListener implements CustomEventListener { + private final class MappingProposedListener implements CustomEventListener { /** {@inheritDoc} */ @Override public void onCustomEvent( AffinityTopologyVersion topVer, @@ -246,13 +246,15 @@ private final class MarshallerMappingExchangeListener implements CustomEventList if (!msg.inConflict()) { MarshallerMappingItem item = msg.mappingItem(); - String conflictingName = marshallerCtx.onMappingProposed(item); + MappedName existingName = marshallerCtx.onMappingProposed(item); - if (conflictingName != null) { - if (conflictingName.equals(item.className())) + if (existingName != null) { + String existingClsName = existingName.className(); + + if (existingClsName.equals(item.className()) && !existingName.accepted()) msg.markDuplicated(); - else - msg.conflictingWithClass(conflictingName); + else if (!existingClsName.equals(item.className())) + msg.conflictingWithClass(existingClsName); } } else { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java index c8a0e76bca04d..80d0fd163edcf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java @@ -54,9 +54,6 @@ * with not-null conflictingClsName field. */ public class IgniteMarshallerCacheClassNameConflictTest extends GridCommonAbstractTest { - /** */ - private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); - /** */ private volatile boolean bbClsRejected; @@ -79,7 +76,7 @@ public class IgniteMarshallerCacheClassNameConflictTest extends GridCommonAbstra IgniteConfiguration cfg = super.getConfiguration(gridName); TcpDiscoverySpi disco = new TestTcpDiscoverySpi(); - disco.setIpFinder(ipFinder); + disco.setIpFinder(LOCAL_IP_FINDER); cfg.setDiscoverySpi(disco); @@ -207,7 +204,7 @@ private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) { DiscoveryCustomMessage customMsg = spiCustomMsg == null ? null : (DiscoveryCustomMessage) U.field(spiCustomMsg, "delegate"); - if (customMsg != null) + if (customMsg != null) { //don't want to make this class public, using equality of class name instead of instanceof operator if ("MappingProposedMessage".equals(customMsg.getClass().getSimpleName())) { String conflClsName = U.field(customMsg, "conflictingClsName"); @@ -219,8 +216,10 @@ else if (conflClsName.contains(BB.class.getSimpleName())) aaClsRejected = true; } } + } - delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg); + if (delegate != null) + delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg); } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java new file mode 100644 index 0000000000000..38fa324e1abdd --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.cache; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.Map; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; +import org.apache.ignite.internal.processors.marshaller.MappingProposedMessage; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage; +import org.apache.ignite.spi.discovery.DiscoverySpiListener; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.Nullable; + +/** + * + */ +public class IgniteMarshallerCacheFSRestoreTest extends GridCommonAbstractTest { + /** */ + private volatile boolean isDuplicateObserved = true; + + /** + * + */ + private static class SimpleValue { + /** */ + private final int iF; + + /** */ + private final String sF; + + /** + * @param iF Int field. + * @param sF String field. + */ + SimpleValue(int iF, String sF) { + this.iF = iF; + this.sF = sF; + } + } + + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + TcpDiscoverySpi discoSpi = new TestTcpDiscoverySpi(); + discoSpi.setIpFinder(LOCAL_IP_FINDER); + + cfg.setDiscoverySpi(discoSpi); + + CacheConfiguration singleCacheConfig = new CacheConfiguration() + .setName(DEFAULT_CACHE_NAME) + .setCacheMode(CacheMode.PARTITIONED) + .setBackups(1) + .setAtomicityMode(CacheAtomicityMode.ATOMIC); + + cfg.setCacheConfiguration(singleCacheConfig); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + cleanUpWorkDir(); + } + + /** + * + */ + private void cleanUpWorkDir() throws Exception { + String workDir = U.defaultWorkDirectory(); + + deleteRecursively(U.resolveWorkDirectory(workDir, "marshaller", false)); + } + + /** + * Test checks a scenario when in multinode cluster one node may read marshaller mapping + * from file storage and add it directly to marshaller context with accepted=true flag, + * when another node sends a proposed request for the same mapping. + * + * In that case the request must not be marked as duplicate and must be processed in a regular way. + * No hangs must take place. + * + * @see IGNITE-5401 Take a look at JIRA ticket for more information about context of this test. + * + * This test must never hang on proposing of MarshallerMapping. + */ + public void testFileMappingReadAndPropose() throws Exception { + prepareMarshallerFileStore(); + + IgniteEx ignite0 = startGrid(0); + IgniteEx ignite1 = startGrid(1); + + BinaryObject obj0 = ignite0.binary().builder(SimpleValue.class.getName()) + .setField("iF", 10) + .setField("sF", "str0") + .build(); + + BinaryObject obj1 = ignite0.binary().builder(SimpleValue.class.getName()) + .setField("iF", 20) + .setField("sF", "str1") + .build(); + + IgniteCache binCache = ignite0.cache(DEFAULT_CACHE_NAME).withKeepBinary(); + + binCache.put(1, obj0); + binCache.put(2, obj1); + + ignite0.cache(DEFAULT_CACHE_NAME).remove(1); + + ignite1.cache(DEFAULT_CACHE_NAME).put(3, new SimpleValue(30, "str2")); + + assertFalse(isDuplicateObserved); + } + + /** + * + */ + private void prepareMarshallerFileStore() throws Exception { + String typeName = SimpleValue.class.getName(); + int typeId = typeName.toLowerCase().hashCode(); + + String fileName = typeId + ".classname0"; + + File marshStoreDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false); + + try(FileOutputStream out = new FileOutputStream(new File(marshStoreDir, fileName))) { + try (Writer writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { + writer.write(typeName); + + writer.flush(); + } + } + } + + /** */ + private class TestTcpDiscoverySpi extends TcpDiscoverySpi { + + /** */ + private class DiscoverySpiListenerWrapper implements DiscoverySpiListener { + /** */ + private DiscoverySpiListener delegate; + + /** + * @param delegate Delegate. + */ + private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) { + this.delegate = delegate; + } + + /** {@inheritDoc} */ + @Override public void onDiscovery( + int type, + long topVer, + ClusterNode node, + Collection topSnapshot, + @Nullable Map> topHist, + @Nullable DiscoverySpiCustomMessage spiCustomMsg + ) { + DiscoveryCustomMessage customMsg = spiCustomMsg == null ? null + : (DiscoveryCustomMessage) U.field(spiCustomMsg, "delegate"); + + if (customMsg != null) { + //don't want to make this class public, using equality of class name instead of instanceof operator + if ("MappingProposedMessage".equals(customMsg.getClass().getSimpleName())) { + try { + isDuplicateObserved = U.invoke(MappingProposedMessage.class, customMsg, "duplicated"); + } + catch (Exception e) { + log().error("Error when examining MappingProposedMessage.", e); + } + } + } + + if (delegate != null) + delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg); + } + + /** {@inheritDoc} */ + @Override public void onLocalNodeInitialized(ClusterNode locNode) { + // No-op. + } + } + + /** {@inheritDoc} */ + @Override public void setListener(@Nullable DiscoverySpiListener lsnr) { + super.setListener(new DiscoverySpiListenerWrapper(lsnr)); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java index de509ab351175..d79e8683d6bdd 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java @@ -43,6 +43,7 @@ import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheClassNameConflictTest; import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheClientRequestsMappingOnMissTest; import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheConcurrentReadWriteTest; +import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheFSRestoreTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteRejectConnectOnNodeStopTest; import org.apache.ignite.internal.processors.closure.GridClosureProcessorSelfTest; import org.apache.ignite.internal.processors.closure.GridClosureSerializationTest; @@ -173,6 +174,7 @@ public static TestSuite suite(@Nullable final Set ignoredTests) throws Ex suite.addTestSuite(FreeListImplSelfTest.class); suite.addTestSuite(MemoryMetricsSelfTest.class); + suite.addTestSuite(IgniteMarshallerCacheFSRestoreTest.class); suite.addTestSuite(IgniteMarshallerCacheClassNameConflictTest.class); suite.addTestSuite(IgniteMarshallerCacheClientRequestsMappingOnMissTest.class); From c84328d46776de62a341c64a463d3d0e429768a4 Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 6 Jul 2017 12:30:52 +0300 Subject: [PATCH 073/155] 2.1 Unused imports --- .../java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index c2d9b7e481c51..d75aae9c0f8a2 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -143,7 +143,6 @@ import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED; import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; -import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_ACTIVE_ON_START; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER; From 6e371ec6a7621e8c764762251f343b1b113c0754 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 6 Jul 2017 12:40:16 +0300 Subject: [PATCH 074/155] IGNITE-5702: Set CacheWriteSynchronizationMode.FULL_SYNC for predefined CREATE TABLE templates. This closes #2246. --- .../ignite/internal/processors/query/GridQueryProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index ce6c9fecb1266..d44be2c1df07f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -43,6 +43,7 @@ import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheKeyConfiguration; import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.QueryIndex; import org.apache.ignite.cache.query.FieldsQueryCursor; @@ -58,7 +59,6 @@ import org.apache.ignite.internal.managers.communication.GridMessageListener; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.CacheObjectContext; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; @@ -1320,6 +1320,8 @@ else if (QueryUtils.TEMPLATE_REPLICÄTED.equalsIgnoreCase(templateName)) ccfg = new CacheConfiguration<>().setCacheMode(CacheMode.REPLICATED); else throw new SchemaOperationException(SchemaOperationException.CODE_CACHE_NOT_FOUND, templateName); + + ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); } if (!F.isEmpty(ccfg.getQueryEntities())) From c3401cbd009d7b9cfc8aed0cc9c3f34fb5f433db Mon Sep 17 00:00:00 2001 From: Alexey Kukushkin Date: Thu, 6 Jul 2017 12:44:27 +0300 Subject: [PATCH 075/155] IGNITE-4901 Decrease logging level for DataStremer retry --- .../datastreamer/DataStreamProcessor.java | 3 +- .../datastreamer/DataStreamerImpl.java | 14 +- .../DataStreamerImplSelfTest.java | 123 +++++++++++++++++- 3 files changed, 132 insertions(+), 8 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java index 789f0d2b582d4..84d536f8c5bfb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; import org.apache.ignite.internal.managers.communication.GridIoManager; import org.apache.ignite.internal.managers.communication.GridMessageListener; import org.apache.ignite.internal.managers.deployment.GridDeployment; @@ -340,7 +341,7 @@ private void localUpdate(final UUID nodeId, AffinityTopologyVersion topVer = fut.topologyVersion(); if (!allowOverwrite && !topVer.equals(req.topologyVersion())) { - Exception err = new IgniteCheckedException( + Exception err = new ClusterTopologyCheckedException( "DataStreamer will retry data transfer at stable topology " + "[reqTop=" + req.topologyVersion() + ", topVer=" + topVer + ", node=remote]"); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java index ae441deec86b4..df51fac168df5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java @@ -1782,10 +1782,16 @@ void onResponse(DataStreamerResponse res, UUID nodeId) { try { GridPeerDeployAware jobPda0 = jobPda; - err = new IgniteCheckedException("DataStreamer request failed [node=" + nodeId + "]", - (Throwable)U.unmarshal(ctx, - errBytes, - U.resolveClassLoader(jobPda0 != null ? jobPda0.classLoader() : null, ctx.config()))); + final Throwable cause = U.unmarshal( + ctx, + errBytes, + U.resolveClassLoader(jobPda0 != null ? jobPda0.classLoader() : null, ctx.config())); + + final String msg = "DataStreamer request failed [node=" + nodeId + "]"; + + err = cause instanceof ClusterTopologyCheckedException ? + new ClusterTopologyCheckedException(msg, cause) : + new IgniteCheckedException(msg, cause); } catch (IgniteCheckedException e) { f.onDone(null, new IgniteCheckedException("Failed to unmarshal response.", e)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java index 6d103124bb460..e72a9b488e1ba 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImplSelfTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.datastreamer; +import java.io.StringWriter; import java.util.Map; import java.util.Random; import java.util.concurrent.Callable; @@ -25,17 +26,29 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.IgniteException; import org.apache.ignite.cache.CacheServerNotFoundException; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.log4j.Appender; +import org.apache.log4j.Logger; +import org.apache.log4j.SimpleLayout; +import org.apache.log4j.WriterAppender; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; @@ -50,12 +63,18 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { /** Number of keys to load via data streamer. */ private static final int KEYS_COUNT = 1000; + /** Next nodes after MAX_CACHE_COUNT start without cache */ + private static final int MAX_CACHE_COUNT = 4; + /** Started grid counter. */ private static int cnt; /** No nodes filter. */ private static volatile boolean noNodesFilter; + /** Indicates whether we need to make the topology stale */ + private static boolean needStaleTop = false; + /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { super.afterTest(); @@ -72,8 +91,9 @@ public class DataStreamerImplSelfTest extends GridCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); - // Forth node goes without cache. - if (cnt < 4) + cfg.setCommunicationSpi(new StaleTopologyCommunicationSpi()); + + if (cnt < MAX_CACHE_COUNT) cfg.setCacheConfiguration(cacheConfiguration()); cnt++; @@ -231,6 +251,44 @@ public void testNoDataNodesOnFlush() throws Exception { } } + /** + * Cluster topology mismatch shall result in DataStreamer retrying cache update with the latest topology and + * no error logged to the console. + * + * @throws Exception if failed + */ + public void testRetryWhenTopologyMismatch() throws Exception { + final int KEY = 1; + final String VAL = "1"; + + cnt = 0; + + StringWriter logWriter = new StringWriter(); + Appender logAppender = new WriterAppender(new SimpleLayout(), logWriter); + + Logger.getRootLogger().addAppender(logAppender); + + startGrids(MAX_CACHE_COUNT - 1); // cache-enabled nodes + + try (Ignite ignite = startGrid(MAX_CACHE_COUNT); + IgniteDataStreamer streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) { + + needStaleTop = true; // simulate stale topology for the next action + + streamer.addData(KEY, VAL); + } finally { + needStaleTop = false; + + logWriter.flush(); + + Logger.getRootLogger().removeAppender(logAppender); + + logAppender.close(); + } + + assertFalse(logWriter.toString().contains("DataStreamer will retry data transfer at stable topology")); + } + /** * Gets cache configuration. * @@ -248,4 +306,63 @@ private CacheConfiguration cacheConfiguration() { return cacheCfg; } -} + + /** + * Simulate stale (not up-to-date) topology + */ + private static class StaleTopologyCommunicationSpi extends TcpCommunicationSpi { + /** {@inheritDoc} */ + @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { + // Send stale topology only in the first request to avoid indefinitely getting failures. + if (needStaleTop) { + if (msg instanceof GridIoMessage) { + GridIoMessage ioMsg = (GridIoMessage)msg; + + Message appMsg = ioMsg.message(); + + if (appMsg != null && appMsg instanceof DataStreamerRequest) { + DataStreamerRequest req = (DataStreamerRequest)appMsg; + + AffinityTopologyVersion validTop = req.topologyVersion(); + + // Simulate situation when a node did not receive the latest "node joined" topology update causing + // topology mismatch + AffinityTopologyVersion staleTop = new AffinityTopologyVersion( + validTop.topologyVersion() - 1, + validTop.minorTopologyVersion()); + + appMsg = new DataStreamerRequest( + req.requestId(), + req.responseTopicBytes(), + req.cacheName(), + req.updaterBytes(), + req.entries(), + req.ignoreDeploymentOwnership(), + req.skipStore(), + req.keepBinary(), + req.deploymentMode(), + req.sampleClassName(), + req.userVersion(), + req.participants(), + req.classLoaderId(), + req.forceLocalDeployment(), + staleTop); + + msg = new GridIoMessage( + GridTestUtils.getFieldValue(ioMsg, "plc"), + GridTestUtils.getFieldValue(ioMsg, "topic"), + GridTestUtils.getFieldValue(ioMsg, "topicOrd"), + appMsg, + GridTestUtils.getFieldValue(ioMsg, "ordered"), + ioMsg.timeout(), + ioMsg.skipOnTimeout()); + + needStaleTop = false; + } + } + } + + super.sendMessage(node, msg, ackC); + } + } +} \ No newline at end of file From 333e32b58fd6e7341fcc14b862140f244b575315 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 6 Jul 2017 13:00:33 +0300 Subject: [PATCH 076/155] IGNITE-5533: Proper index cleanup in H2 database during DROP TALBE. This closes #2167. --- .../processors/query/h2/opt/GridH2Table.java | 49 ++++++++++++++++- .../cache/index/H2DynamicTableSelfTest.java | 53 +++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java index d656cc3194ee4..76d025811bd32 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java @@ -40,7 +40,9 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteBiTuple; import org.h2.command.ddl.CreateTableData; +import org.h2.engine.DbObject; import org.h2.engine.Session; +import org.h2.engine.SysProperties; import org.h2.index.Index; import org.h2.index.IndexType; import org.h2.index.SpatialIndex; @@ -48,6 +50,7 @@ import org.h2.result.Row; import org.h2.result.SearchRow; import org.h2.result.SortOrder; +import org.h2.schema.SchemaObject; import org.h2.table.IndexColumn; import org.h2.table.TableBase; import org.h2.table.TableType; @@ -76,6 +79,9 @@ public class GridH2Table extends TableBase { /** */ private final int pkIndexPos; + /** Total number of system indexes. */ + private final int sysIdxsCnt; + /** */ private final Map tmpIdxs = new HashMap<>(); @@ -184,6 +190,8 @@ public GridH2Table(CreateTableData createTblData, @Nullable GridH2RowDescriptor pkIndexPos = hasHashIndex ? 2 : 1; + sysIdxsCnt = idxs.size(); + final int segments = desc != null ? desc.context().config().getQueryParallelism() : // Get index segments count from PK index. Null desc can be passed from tests. index(pkIndexPos).segmentsCount(); @@ -449,6 +457,45 @@ private Object[] doSnapshotIndexes(int segment, Object[] segmentSnapshot, GridH2 // No-op. } + /** {@inheritDoc} */ + @SuppressWarnings("ThrowableResultOfMethodCallIgnored") + @Override public void removeChildrenAndResources(Session ses) { + lock(true); + + try { + super.removeChildrenAndResources(ses); + + // Clear all user indexes registered in schema. + while (idxs.size() > sysIdxsCnt) { + Index idx = idxs.get(sysIdxsCnt); + + if (idx.getName() != null && idx.getSchema().findIndex(ses, idx.getName()) == idx) { + // This call implicitly removes both idx and its proxy, if any, from idxs. + database.removeSchemaObject(ses, idx); + + // We have to call destroy here if we are who has removed this index from the table. + if (idx instanceof GridH2IndexBase) + ((GridH2IndexBase)idx).destroy(); + } + } + + if (SysProperties.CHECK) { + for (SchemaObject obj : database.getAllSchemaObjects(DbObject.INDEX)) { + Index idx = (Index) obj; + if (idx.getTable() == this) + DbException.throwInternalError("index not dropped: " + idx.getName()); + } + } + + database.removeMeta(ses, getId()); + invalidate(); + + } + finally { + unlock(true); + } + } + /** * Destroy the table. */ @@ -791,7 +838,7 @@ private Index commitUserIndex(Session ses, String idxName) { Index cloneIdx = createDuplicateIndexIfNeeded(idx); ArrayList newIdxs = new ArrayList<>( - idxs.size() + ((cloneIdx == null) ? 1 : 2)); + idxs.size() + ((cloneIdx == null) ? 1 : 2)); newIdxs.addAll(idxs); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java index 5975b4e91a80a..b0e69f15f1ccf 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java @@ -693,6 +693,59 @@ public void testAffinityKeyNotFound() { }, IgniteSQLException.class, "Affinity key column with given name not found: missing"); } + /** + * Tests behavior on sequential create and drop of a table and its index. + */ + public void testTableAndIndexRecreate() { + execute("drop table if exists \"PUBLIC\".t"); + + // First let's check behavior without index name set + execute("create table \"PUBLIC\".t (a int primary key, b varchar(30))"); + + fillRecreatedTable(); + + execute("create index on \"PUBLIC\".t (b desc)"); + execute("drop table \"PUBLIC\".t"); + + assertNull(client().cache("t")); + + execute("create table \"PUBLIC\".t (a int primary key, b varchar(30))"); + + fillRecreatedTable(); + + execute("create index on \"PUBLIC\".t (b desc)"); + execute("drop table \"PUBLIC\".t"); + + assertNull(client().cache("t")); + + // And now let's do the same for the named index + execute("create table \"PUBLIC\".t (a int primary key, b varchar(30))"); + + fillRecreatedTable(); + + execute("create index namedIdx on \"PUBLIC\".t (b desc)"); + execute("drop table \"PUBLIC\".t"); + + assertNull(client().cache("t")); + + execute("create table \"PUBLIC\".t (a int primary key, b varchar(30))"); + + fillRecreatedTable(); + + execute("create index namedIdx on \"PUBLIC\".t (b desc)"); + execute("drop table \"PUBLIC\".t"); + } + + /** + * Fill re-created table with data. + */ + private void fillRecreatedTable() { + for (int j = 1; j < 10; j++) { + String s = Integer.toString(j); + execute("insert into \"PUBLIC\".t (a,b) values (" + s + ", '" + s + "')"); + } + } + /** * Check that dynamic cache created with {@code CREATE TABLE} is correctly configured affinity wise. * @param cacheName Cache name to check. From c396b0bcfa5f959013c94ece47dc1111fa650d97 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 6 Jul 2017 13:03:50 +0300 Subject: [PATCH 077/155] IGNITE-5441 .NET: Propagate CacheStore exception stack traces to the caller node This closes #2247 --- .../Cache/CacheAbstractTest.cs | 26 ++++++++----------- .../Cache/PersistentStoreTest.cs | 1 - .../Cache/Store/CacheStoreTest.cs | 21 ++++++++++++--- .../Common/JavaException.cs | 15 ++++++++++- .../Apache.Ignite.Core/Impl/ExceptionUtils.cs | 5 ++-- 5 files changed, 44 insertions(+), 24 deletions(-) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTest.cs index 351c25c0c15ce..9e44720cfc357 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTest.cs @@ -2091,26 +2091,22 @@ private void TestInvoke(bool async) where T: AddArgCacheEntryProcessor, new() () => cache.Invoke(key, new T { ThrowErrNonSerializable = true }, arg), "ExpectedException"); } + /// + /// Asserts that specified action throws a CacheEntryProcessorException. + /// private static void AssertThrowsCacheEntryProcessorException(Action action, string containsText = null) { - try - { - action(); + var ex = Assert.Throws(() => action()); - Assert.Fail(); + Assert.IsInstanceOf(ex.InnerException); + + if (string.IsNullOrEmpty(containsText)) + { + Assert.AreEqual(AddArgCacheEntryProcessor.ExceptionText, ex.GetBaseException().Message); } - catch (Exception ex) + else { - Assert.IsInstanceOf(ex); - - if (string.IsNullOrEmpty(containsText)) - { - Assert.IsNotNull(ex.InnerException); - Assert.AreEqual(AddArgCacheEntryProcessor.ExceptionText, ex.InnerException.Message); - } - else - Assert.IsTrue(ex.ToString().Contains(containsText), - "Expected: " + containsText + ", actual: " + ex); + Assert.IsTrue(ex.ToString().Contains(containsText), "Expected: " + containsText + ", actual: " + ex); } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs index cc72147e4e013..adb91efc247f9 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs @@ -18,7 +18,6 @@ namespace Apache.Ignite.Core.Tests.Cache { using Apache.Ignite.Core.Common; - using Apache.Ignite.Core.Configuration; using Apache.Ignite.Core.PersistentStore; using NUnit.Framework; diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs index 12c442d382f50..e05f4bded7796 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Store/CacheStoreTest.cs @@ -24,6 +24,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Store using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Cache; using Apache.Ignite.Core.Cache.Store; + using Apache.Ignite.Core.Common; using Apache.Ignite.Core.Impl; using NUnit.Framework; @@ -203,6 +204,9 @@ public void TestPutLoad() Assert.AreEqual(1, cache.GetSize()); } + /// + /// Tests that exceptions from user code are propagated properly. + /// [Test] public void TestExceptions() { @@ -211,7 +215,18 @@ public void TestExceptions() cache.Put(1, "val"); CacheTestStore.ThrowError = true; - CheckCustomStoreError(Assert.Throws(() => cache.Put(-2, "fail")).InnerException); + + var ex = Assert.Throws(() => cache.Put(-2, "fail")); + + Assert.IsTrue(ex.ToString().Contains( + "at Apache.Ignite.Core.Tests.Cache.Store.CacheTestStore.ThrowIfNeeded")); // Check proper stack trace. + + Assert.IsNotNull(ex.InnerException); // RollbackException. + + var javaEx = ex.InnerException.InnerException as JavaException; + Assert.IsNotNull(javaEx); + + CheckCustomStoreError(javaEx.InnerException); // TODO: IGNITE-4535 //cache.LocalEvict(new[] {1}); @@ -599,11 +614,9 @@ private static IDictionary GetStoreMap() /// private static void CheckCustomStoreError(Exception err) { - var customErr = err as CacheTestStore.CustomStoreException ?? - err.InnerException as CacheTestStore.CustomStoreException; + var customErr = err.GetBaseException() as CacheTestStore.CustomStoreException; Assert.IsNotNull(customErr); - Assert.AreEqual(customErr.Message, customErr.Details); } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Common/JavaException.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Common/JavaException.cs index 1988335e99508..e6c6f7b769a4a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Common/JavaException.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Common/JavaException.cs @@ -62,7 +62,20 @@ public JavaException(string message) : base(message) /// Java exception message. /// Java stack trace. public JavaException(string javaClassName, string javaMessage, string stackTrace) - : base(stackTrace ?? javaMessage) + : this(javaClassName, javaMessage, stackTrace, null) + { + // No-op. + } + + /// + /// Initializes a new instance of the class. + /// + /// Java exception class name. + /// Java exception message. + /// Java stack trace. + /// The cause. + public JavaException(string javaClassName, string javaMessage, string stackTrace, Exception cause) + : base(stackTrace ?? javaMessage, cause) { // Send stackTrace to base ctor because it has all information, including class names and messages. // Store ClassName and Message separately for mapping purposes. diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs index dd70f5ade8c1c..ebfcc285868e7 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/ExceptionUtils.cs @@ -121,9 +121,8 @@ static ExceptionUtils() public static Exception GetException(Ignite ignite, string clsName, string msg, string stackTrace, BinaryReader reader = null, Exception innerException = null) { - // Set JavaException as inner only if there is no InnerException. - if (innerException == null) - innerException = new JavaException(clsName, msg, stackTrace); + // Set JavaException as immediate inner. + innerException = new JavaException(clsName, msg, stackTrace, innerException); ExceptionFactory ctor; From 1cf402f379516b01ef7317a50c820bf0dec25d81 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Wed, 21 Jun 2017 19:30:27 +0300 Subject: [PATCH 078/155] IGNITE-5552: ServiceProcessor recalculates all service assignments even if there is a pending topology change. This closes #2182. --- .../service/GridServiceProcessor.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java index 87db72aeda61a..d67f2d16d45c5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java @@ -1530,6 +1530,9 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog * Topology listener. */ private class TopologyListener implements DiscoveryEventListener { + /** */ + private volatile AffinityTopologyVersion currTopVer = null; + /** {@inheritDoc} */ @Override public void onEvent(DiscoveryEvent evt, final DiscoCache discoCache) { GridSpinBusyLock busyLock = GridServiceProcessor.this.busyLock; @@ -1562,6 +1565,8 @@ else if (msg instanceof DynamicCacheChangeBatch) { else topVer = new AffinityTopologyVersion((evt).topologyVersion(), 0); + currTopVer = topVer; + depExe.execute(new DepRunnable() { @Override public void run0() { // In case the cache instance isn't tracked by DiscoveryManager anymore. @@ -1582,6 +1587,19 @@ else if (msg instanceof DynamicCacheChangeBatch) { boolean firstTime = true; while (it.hasNext()) { + // If topology changed again, let next event handle it. + AffinityTopologyVersion currTopVer0 = currTopVer; + + if (currTopVer0 != topVer) { + if (log.isInfoEnabled()) + log.info("Service processor detected a topology change during " + + "assignments calculation (will abort current iteration and " + + "re-calculate on the newer version): " + + "[topVer=" + topVer + ", newTopVer=" + currTopVer + ']'); + + return; + } + Cache.Entry e = it.next(); if (firstTime) { From d8a50e47a51718c9ef202375b324695b78225813 Mon Sep 17 00:00:00 2001 From: Alexander Belyak Date: Thu, 6 Jul 2017 14:28:22 +0300 Subject: [PATCH 079/155] Fixed "IGNITE-5390 But in IgniteCacheTxStoreSessionWriteBehindCoalescingTest" Signed-off-by: nikolay_tikhonov --- .../processors/cache/ClusterCachesInfo.java | 4 + .../processors/cache/GridCacheAttributes.java | 7 ++ .../store/GridCacheStoreManagerAdapter.java | 1 + .../cache/IgniteCacheAbstractTest.java | 17 ++++ ...heStoreSessionWriteBehindAbstractTest.java | 62 +++++++++---- ...StoreSessionWriteBehindCoalescingTest.java | 88 +++++++++++++++++++ ...ientWriteBehindStoreNonCoalescingTest.java | 30 ++++--- .../testsuites/IgniteCacheTestSuite4.java | 2 + 8 files changed, 180 insertions(+), 31 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java index 5452bd20fcb19..5aca8c9811ae1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java @@ -263,6 +263,10 @@ private void checkCache(CacheJoinNodeDiscoveryData.CacheInfo locInfo, CacheData "Write behind batch size", locAttr.writeBehindBatchSize(), rmtAttr.writeBehindBatchSize(), false); + CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "writeBehindCoalescing", + "Write behind coalescing", locAttr.writeBehindCoalescing(), rmtAttr.writeBehindCoalescing(), + false); + CU.checkAttributeMismatch(log, rmtAttr.cacheName(), rmt, "writeBehindEnabled", "Write behind enabled", locAttr.writeBehindEnabled(), rmtAttr.writeBehindEnabled(), false); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java index dca428631f94d..32871ea49c665 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAttributes.java @@ -265,6 +265,13 @@ public int writeBehindBatchSize() { return ccfg.getWriteBehindBatchSize(); } + /** + * @return Write coalescing flag. + */ + public boolean writeBehindCoalescing() { + return ccfg.getWriteBehindCoalescing(); + } + /** * @return Interceptor class name. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java index 8ff2f5aafa0cc..c02e2c73b2710 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java @@ -185,6 +185,7 @@ private CacheStore cacheStoreWrapper(GridKernalContext ctx, store.setFlushThreadCount(cfg.getWriteBehindFlushThreadCount()); store.setFlushFrequency(cfg.getWriteBehindFlushFrequency()); store.setBatchSize(cfg.getWriteBehindBatchSize()); + store.setWriteCoalescing(cfg.getWriteBehindCoalescing()); return store; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractTest.java index c5cb71507e07b..34a811bad66bd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractTest.java @@ -144,6 +144,9 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throw cfg.setReadThrough(true); cfg.setWriteThrough(true); cfg.setLoadPreviousValue(true); + + cfg.setWriteBehindEnabled(writeBehindEnabled()); + cfg.setWriteBehindCoalescing(writeBehindCoalescing()); } if (cacheMode() == PARTITIONED) @@ -161,6 +164,20 @@ protected Factory cacheStoreFactory() { return null; } + /** + * @return write behind enabled flag. + */ + protected boolean writeBehindEnabled() { + return false; + } + + /** + * @return write behind coalescing flag. + */ + protected boolean writeBehindCoalescing() { + return true; + } + /** * @return Cache loader factory. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java index dcbb63f8655f9..7ad240d553d6a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheStoreSessionWriteBehindAbstractTest.java @@ -48,6 +48,9 @@ public abstract class IgniteCacheStoreSessionWriteBehindAbstractTest extends Ign /** */ private static volatile CountDownLatch latch; + /** */ + protected static volatile CountDownLatch entLatch; + /** */ private static volatile ExpectedData expData; @@ -66,36 +69,42 @@ public abstract class IgniteCacheStoreSessionWriteBehindAbstractTest extends Ign return null; } - /** {@inheritDoc} */ + /** + * @param igniteInstanceName Ignite instance name. + * @return Cache configuration. + * @throws Exception In case of error. + */ @SuppressWarnings("unchecked") - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { + CacheConfiguration ccfg0 = super.cacheConfiguration(igniteInstanceName); - assert cfg.getCacheConfiguration().length == 1; - - CacheConfiguration ccfg0 = cfg.getCacheConfiguration()[0]; ccfg0.setReadThrough(true); ccfg0.setWriteThrough(true); ccfg0.setWriteBehindBatchSize(10); ccfg0.setWriteBehindFlushSize(10); - ccfg0.setWriteBehindFlushFrequency(60_000); + ccfg0.setWriteBehindFlushFrequency(600); ccfg0.setWriteBehindEnabled(true); ccfg0.setCacheStoreFactory(singletonFactory(new TestStore())); - CacheConfiguration ccfg1 = cacheConfiguration(igniteInstanceName); + return ccfg0; + } + + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - ccfg1.setReadThrough(true); - ccfg1.setWriteThrough(true); - ccfg1.setWriteBehindBatchSize(10); - ccfg1.setWriteBehindFlushSize(10); - ccfg1.setWriteBehindFlushFrequency(60_000); - ccfg1.setWriteBehindEnabled(true); + assert cfg.getCacheConfiguration().length == 1; - ccfg1.setName(CACHE_NAME1); + CacheConfiguration ccfg0 = cacheConfiguration(igniteInstanceName); + + ccfg0.setName(DEFAULT_CACHE_NAME); + + CacheConfiguration ccfg1 = cacheConfiguration(igniteInstanceName); - ccfg1.setCacheStoreFactory(singletonFactory(new TestStore())); + ccfg1.setName(CACHE_NAME1); cfg.setCacheConfiguration(ccfg0, ccfg1); @@ -120,6 +129,7 @@ private void testCache(String cacheName) throws Exception { try { latch = new CountDownLatch(2); + entLatch = new CountDownLatch(11); expData = new ExpectedData("writeAll", cacheName); @@ -127,13 +137,17 @@ private void testCache(String cacheName) throws Exception { cache.put(i, i); assertTrue(latch.await(10_000, TimeUnit.MILLISECONDS)); + + assertTrue(entLatch.await(10_000,TimeUnit.MILLISECONDS)); } finally { latch = null; + entLatch = null; } try { latch = new CountDownLatch(2); + entLatch = new CountDownLatch(11); expData = new ExpectedData("deleteAll", cacheName); @@ -141,16 +155,20 @@ private void testCache(String cacheName) throws Exception { cache.remove(i); assertTrue(latch.await(10_000, TimeUnit.MILLISECONDS)); + + assertTrue(entLatch.await(10_000,TimeUnit.MILLISECONDS)); } finally { latch = null; + entLatch = null; } } /** * */ - private class TestStore implements CacheStore { + protected class TestStore implements CacheStore { + /** Auto-injected store session. */ @CacheStoreSessionResource private CacheStoreSession ses; @@ -191,10 +209,13 @@ private class TestStore implements CacheStore { /** {@inheritDoc} */ @Override public void writeAll(Collection> entries) throws CacheWriterException { log.info("writeAll: " + entries); - + assertTrue("Unexpected entries: " + entries, entries.size() == 10 || entries.size() == 1); checkSession("writeAll"); + + for (int i = 0; i < entries.size(); i++) + entLatch.countDown(); } /** {@inheritDoc} */ @@ -209,6 +230,9 @@ private class TestStore implements CacheStore { assertTrue("Unexpected keys: " + keys, keys.size() == 10 || keys.size() == 1); checkSession("deleteAll"); + + for (int i = 0; i < keys.size(); i++) + entLatch.countDown(); } /** @@ -221,7 +245,7 @@ private CacheStoreSession session() { /** * @param mtd Called stored method. */ - private void checkSession(String mtd) { + protected void checkSession(String mtd) { assertNotNull(ignite); CacheStoreSession ses = session(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java new file mode 100644 index 0000000000000..58cc380c8bc5a --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/integration/IgniteCacheTxStoreSessionWriteBehindCoalescingTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.integration; + +import java.util.Collection; +import javax.cache.Cache; +import javax.cache.integration.CacheWriterException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.configuration.CacheConfiguration; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; + +/** + * Integration test write behind cache store with {@link CacheConfiguration#getWriteBehindCoalescing()}={@code False} + * parameter. + */ +public class IgniteCacheTxStoreSessionWriteBehindCoalescingTest extends IgniteCacheStoreSessionWriteBehindAbstractTest { + /** {@inheritDoc} */ + @Override protected CacheAtomicityMode atomicityMode() { + return TRANSACTIONAL; + } + + /** + * @param igniteInstanceName Ignite instance name. + * @return Cache configuration. + * @throws Exception In case of error. + */ + @SuppressWarnings("unchecked") + protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception { + CacheConfiguration ccfg = super.cacheConfiguration(igniteInstanceName); + + ccfg.setWriteBehindCoalescing(false); + + ccfg.setCacheStoreFactory(singletonFactory(new TestNonCoalescingStore())); + + return ccfg; + } + + /** + * + */ + private class TestNonCoalescingStore extends TestStore { + + /** {@inheritDoc} */ + @Override public void writeAll(Collection> entries) throws CacheWriterException { + log.info("writeAll: " + entries); + + assertTrue("Unexpected entries: " + entries, entries.size() <= 10); + + checkSession("writeAll"); + + for (int i = 0; i < entries.size(); i++) + entLatch.countDown(); + } + + /** {@inheritDoc} */ + @Override public void delete(Object key) throws CacheWriterException { + fail(); + } + + /** {@inheritDoc} */ + @Override public void deleteAll(Collection keys) throws CacheWriterException { + log.info("deleteAll: " + keys); + + assertTrue("Unexpected keys: " + keys, keys.size() <= 10); + + checkSession("deleteAll"); + + for (int i = 0; i < keys.size(); i++) + entLatch.countDown(); + } + } +} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java index 6a75dbdb8f64b..4ffa973ff90e6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/IgnteCacheClientWriteBehindStoreNonCoalescingTest.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.Map; +import java.util.Random; import java.util.Set; import javax.cache.Cache; import javax.cache.configuration.Factory; @@ -36,6 +37,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest; +import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.lang.IgniteFuture; @@ -70,6 +72,14 @@ public class IgnteCacheClientWriteBehindStoreNonCoalescingTest extends IgniteCac return new TestIncrementStoreFactory(); } + /** {@inheritDoc} */ + @Override protected boolean writeBehindEnabled() { return true;} + + /** {@inheritDoc} */ + @Override protected boolean writeBehindCoalescing() { return false;} + + private static Random rnd = new Random(); + /** * @throws Exception If failed. */ @@ -81,35 +91,30 @@ public void testNonCoalescingIncrementing() throws Exception { assertEquals(cache.getConfiguration(CacheConfiguration.class).getCacheStoreFactory().getClass(), TestIncrementStoreFactory.class); - Set keys = new HashSet<>(); - - for (int i = 0; i < 1000; i++) { - keys.add(i); - + for (int i = 0; i < CacheConfiguration.DFLT_WRITE_BEHIND_FLUSH_SIZE * 2; i++) { cache.put(i, i); } Collection> futs = new ArrayList<>(); - for (int i = 0; i < 100; i++) - futs.add(updateKeys(cache, keys)); + for (int i = 0; i < 1000; i++) + futs.add(updateKey(cache)); for (IgniteFuture fut : futs) fut.get(); } /** - * Update specified keys in async mode. + * Update random key in async mode. * * @param cache Cache to use. - * @param keys Keys to update. * @return IgniteFuture. */ - private IgniteFuture updateKeys(IgniteCache cache, Set keys) { + private IgniteFuture updateKey(IgniteCache cache) { IgniteCache asyncCache = cache.withAsync(); // Using EntryProcessor.invokeAll to increment every value in place. - asyncCache.invokeAll(keys, new EntryProcessor() { + asyncCache.invoke(rnd.nextInt(100), new EntryProcessor() { @Override public Object process(MutableEntry entry, Object... arguments) throws EntryProcessorException { entry.setValue(entry.getValue() + 1); @@ -150,7 +155,8 @@ public static class TestIncrementStore extends CacheStoreAdapter @Override public void write(Cache.Entry entry) { Object oldVal = storeMap.put(entry.getKey(), entry.getValue()); - if (oldVal instanceof Integer && entry.getValue() instanceof Integer) { + + if (oldVal != null) { Integer oldInt = (Integer) oldVal; Integer newInt = (Integer)entry.getValue(); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java index 1b35acbc92c0c..45f575e45a016 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java @@ -134,6 +134,7 @@ import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxNoReadThroughTest; import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxNoWriteThroughTest; import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxStoreSessionTest; +import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxStoreSessionWriteBehindCoalescingTest; import org.apache.ignite.internal.processors.cache.integration.IgniteCacheTxStoreSessionWriteBehindTest; import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryLocalAtomicSwapDisabledSelfTest; import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryLocalTransactionalSelfTest; @@ -172,6 +173,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteCacheTxStoreSessionTest.class); suite.addTestSuite(IgniteCacheAtomicStoreSessionWriteBehindTest.class); suite.addTestSuite(IgniteCacheTxStoreSessionWriteBehindTest.class); + suite.addTestSuite(IgniteCacheTxStoreSessionWriteBehindCoalescingTest.class); suite.addTestSuite(IgniteCacheAtomicNoReadThroughTest.class); suite.addTestSuite(IgniteCacheAtomicNearEnabledNoReadThroughTest.class); From 9675061e3f1d190d46f55d1de6ce962b9da7dc8c Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Thu, 6 Jul 2017 14:33:13 +0300 Subject: [PATCH 080/155] IGNITE-5502 .NET: Persistent Store test --- .../Cache/PersistentStoreTest.cs | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs index adb91efc247f9..96ae47c18dae6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs @@ -17,7 +17,9 @@ namespace Apache.Ignite.Core.Tests.Cache { + using System.IO; using Apache.Ignite.Core.Common; + using Apache.Ignite.Core.Impl; using Apache.Ignite.Core.PersistentStore; using NUnit.Framework; @@ -26,6 +28,78 @@ namespace Apache.Ignite.Core.Tests.Cache /// public class PersistentStoreTest { + /** Temp dir for WAL. */ + private readonly string _tempDir = IgniteUtils.GetTempDirectoryName(); + + /// + /// Tears down the test. + /// + [TearDown] + public void TearDown() + { + Ignition.StopAll(true); + + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, true); + } + } + + /// + /// Tests that cache data survives node restart. + /// + [Test] + public void TestCacheDataSurvivesNodeRestart() + { + var cfg = new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + PersistentStoreConfiguration = new PersistentStoreConfiguration + { + PersistentStorePath = Path.Combine(_tempDir, "Store"), + WalStorePath = Path.Combine(_tempDir, "WalStore"), + WalArchivePath = Path.Combine(_tempDir, "WalArchive") + } + }; + + const string cacheName = "persistentCache"; + + // Start Ignite, put data, stop. + using (var ignite = Ignition.Start(cfg)) + { + ignite.SetActive(true); + + var cache = ignite.CreateCache(cacheName); + + cache[1] = 1; + } + + // Verify directories. + Assert.IsTrue(Directory.Exists(cfg.PersistentStoreConfiguration.PersistentStorePath)); + Assert.IsTrue(Directory.Exists(cfg.PersistentStoreConfiguration.WalStorePath)); + Assert.IsTrue(Directory.Exists(cfg.PersistentStoreConfiguration.WalArchivePath)); + + // Start Ignite, verify data survival. + using (var ignite = Ignition.Start(cfg)) + { + ignite.SetActive(true); + + var cache = ignite.GetCache(cacheName); + + Assert.AreEqual(1, cache[1]); + } + + // Delete store directory. + Directory.Delete(_tempDir, true); + + // Start Ignite, verify data loss. + using (var ignite = Ignition.Start(cfg)) + { + ignite.SetActive(true); + + Assert.IsFalse(ignite.GetCacheNames().Contains(cacheName)); + } + } + /// /// Tests the grid activation with persistence (inactive by default). /// From 4a169dc5625f6f9794c17bfcf5bc1a318c91a996 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Tue, 20 Jun 2017 07:59:09 +0300 Subject: [PATCH 081/155] IGNITE-5528 - IS_EVICT_DISABLED flag is not cleared on cache store exception. This closes #2183. --- .../processors/cache/GridCacheAdapter.java | 62 ++++--- .../processors/cache/GridCacheEntryEx.java | 3 +- .../processors/cache/GridCacheMapEntry.java | 2 +- ...tionEvictionDuringReadThroughSelfTest.java | 160 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite5.java | 3 + 5 files changed, 205 insertions(+), 25 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index 5937a48900421..9213be3cf5e4b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -2086,35 +2086,18 @@ else if (storeEnabled) } }); - if (loaded.size() != loadKeys.size()) { - boolean needTouch = - tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED); - - for (Map.Entry e : loadKeys.entrySet()) { - if (loaded.contains(e.getKey())) - continue; - - if (needTouch || e.getValue().reserved()) { - GridCacheEntryEx entry = peekEx(e.getKey()); - - if (entry != null) { - if (e.getValue().reserved()) - entry.clearReserveForLoad(e.getValue().version()); - - if (needTouch) - ctx.evicts().touch(entry, topVer); - } - } - } - } + clearReservationsIfNeeded(topVer, loadKeys, loaded, tx0); return map; } }), true), new C2, Exception, IgniteInternalFuture>>() { @Override public IgniteInternalFuture> apply(Map map, Exception e) { - if (e != null) + if (e != null) { + clearReservationsIfNeeded(topVer, loadKeys, loaded, tx0); + return new GridFinishedFuture<>(e); + } if (tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED)) { Collection notFound = new HashSet<>(loadKeys.keySet()); @@ -2181,6 +2164,41 @@ else if (storeEnabled) } } + /** + * @param topVer Affinity topology version for which load was performed. + * @param loadKeys Keys to load. + * @param loaded Actually loaded keys. + * @param tx0 Transaction within which the load was run, if any. + */ + private void clearReservationsIfNeeded( + AffinityTopologyVersion topVer, + Map loadKeys, + Collection loaded, + IgniteTxLocalAdapter tx0 + ) { + if (loaded.size() != loadKeys.size()) { + boolean needTouch = + tx0 == null || (!tx0.implicit() && tx0.isolation() == READ_COMMITTED); + + for (Map.Entry e : loadKeys.entrySet()) { + if (loaded.contains(e.getKey())) + continue; + + if (needTouch || e.getValue().reserved()) { + GridCacheEntryEx entry = peekEx(e.getKey()); + + if (entry != null) { + if (e.getValue().reserved()) + entry.clearReserveForLoad(e.getValue().version()); + + if (needTouch) + ctx.evicts().touch(entry, topVer); + } + } + } + } + } + /** {@inheritDoc} */ @Override public final V getAndPut(K key, V val) throws IgniteCheckedException { return getAndPut(key, val, null); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java index 4e52680cf94df..5aabd30875b60 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java @@ -332,9 +332,8 @@ public EntryGetResult innerGetAndReserveForLoad(boolean updateMetrics, /** * @param ver Expected entry version. - * @throws IgniteCheckedException If failed. */ - public void clearReserveForLoad(GridCacheVersion ver) throws IgniteCheckedException; + public void clearReserveForLoad(GridCacheVersion ver); /** * Reloads entry from underlying storage. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index eea8935423763..673945a92a64d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -2699,7 +2699,7 @@ protected long nextPartitionCounter(AffinityTopologyVersion topVer, boolean prim } /** {@inheritDoc} */ - @Override public synchronized void clearReserveForLoad(GridCacheVersion ver) throws IgniteCheckedException { + @Override public synchronized void clearReserveForLoad(GridCacheVersion ver) { if (obsoleteVersionExtras() != null) return; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java new file mode 100644 index 0000000000000..d5351f7e37776 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCachePartitionEvictionDuringReadThroughSelfTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import java.util.LinkedHashSet; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.cache.Cache; +import javax.cache.configuration.Factory; +import javax.cache.expiry.CreatedExpiryPolicy; +import javax.cache.expiry.Duration; +import javax.cache.integration.CacheLoaderException; +import javax.cache.integration.CacheWriterException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy; +import org.apache.ignite.cache.store.CacheStore; +import org.apache.ignite.cache.store.CacheStoreAdapter; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * + */ +public class GridCachePartitionEvictionDuringReadThroughSelfTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + CacheConfiguration ccfg = + new CacheConfiguration() + .setName("config") + .setAtomicityMode(CacheAtomicityMode.ATOMIC) + .setBackups(0) // No need for backup, just load from the store if needed + .setCacheStoreFactory(new CacheStoreFactory()) + .setEvictionPolicy(new LruEvictionPolicy(100)) + .setNearConfiguration(new NearCacheConfiguration() + .setNearEvictionPolicy(new LruEvictionPolicy())); + + ccfg.setExpiryPolicyFactory(CreatedExpiryPolicy.factoryOf(new Duration(TimeUnit.MINUTES, 1))) + .setReadThrough(true) + .setWriteThrough(false); + + cfg.setCacheConfiguration(ccfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + } + + /** + * @throws Exception if failed. + */ + public void testPartitionRent() throws Exception { + startGrid(0); + + final AtomicBoolean done = new AtomicBoolean(); + + IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Callable() { + @Override + public Integer call() throws Exception { + LinkedHashSet set = new LinkedHashSet<>(); + + set.add(1); + set.add(2); + set.add(3); + set.add(4); + set.add(5); + + while (!done.get()) { + try { + grid(0).cache("config").getAll(set); + } + catch (Throwable ignore) { + // No-op. + } + } + + return null; + } + }, 4, "loader"); + + IgniteInternalFuture startFut = GridTestUtils.runAsync(new Callable() { + @Override public Void call() throws Exception { + for (int i = 1; i < 5; i++) { + startGrid(i); + + awaitPartitionMapExchange(); + } + + return null; + } + }); + + startFut.get(); + + done.set(true); + + fut.get(); + } + + /** + * + */ + private static class CacheStoreFactory implements Factory> { + /** {@inheritDoc} */ + @Override public CacheStore create() { + return new HangingCacheStore(); + } + } + + /** + * + */ + private static class HangingCacheStore extends CacheStoreAdapter { + /** */ + private CountDownLatch releaseLatch = new CountDownLatch(1); + + /** {@inheritDoc} */ + @Override public Integer load(Integer key) throws CacheLoaderException { + if (key == 3) + throw new CacheLoaderException(); + + return key; + } + + /** {@inheritDoc} */ + @Override public void write(Cache.Entry entry) throws CacheWriterException { + + } + + /** {@inheritDoc} */ + @Override public void delete(Object key) throws CacheWriterException { + + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java index 1ab516aeada57..1395b954df600 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java @@ -37,6 +37,7 @@ import org.apache.ignite.internal.processors.cache.PartitionsExchangeOnDiscoveryHistoryOverflowTest; import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest; import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest; +import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheGroupsPartitionLossPolicySelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartitionLossPolicySelfTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheTxIteratorSelfTest; @@ -92,6 +93,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(GridCachePartitionExchangeManagerHistSizeTest.class); + suite.addTestSuite(GridCachePartitionEvictionDuringReadThroughSelfTest.class); + return suite; } } From 1130a87a3157ce3a786994835fb927668c10d558 Mon Sep 17 00:00:00 2001 From: nikolay_tikhonov Date: Fri, 30 Jun 2017 14:38:54 +0300 Subject: [PATCH 082/155] IGNITE-5424: GridServiceProxy does not unwraps exception message from InvocationTargetException. This closes #2168. --- .../processors/service/GridServiceProxy.java | 9 ++- .../GridServiceProcessorProxySelfTest.java | 65 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java index d16a4c48dcf86..3a40b907f91b2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java @@ -23,6 +23,7 @@ import java.io.ObjectOutput; import java.io.Serializable; import java.lang.reflect.InvocationHandler; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.util.ArrayList; @@ -413,7 +414,13 @@ private ServiceProxyCallable(String mtdName, String svcName, Class[] argTypes, O if (mtd == null) throw new GridServiceMethodNotFoundException(svcName, mtdName, argTypes); - return mtd.invoke(svcCtx.service(), args); + try { + return mtd.invoke(svcCtx.service(), args); + } + catch (InvocationTargetException e) { + // Get error message. + throw new IgniteCheckedException(e.getCause().getMessage(), e); + } } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java index 9fd2d2c2a6b06..d1c5294d48604 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java @@ -18,9 +18,11 @@ package org.apache.ignite.internal.processors.service; import java.util.Map; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteException; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.util.typedef.PA; import org.apache.ignite.internal.util.typedef.X; @@ -66,6 +68,31 @@ public void testNodeSingletonProxy() throws Exception { } } + /** + * Unwraps error message from InvocationTargetException. + * + * @throws Exception If failed. + */ + @SuppressWarnings("ThrowableNotThrown") + public void testException() throws Exception { + String name = "errorService"; + + Ignite ignite = grid(0); + + ignite.services(ignite.cluster().forRemotes()).deployNodeSingleton(name, new ErrorServiceImpl()); + + final ErrorService svc = ignite.services().serviceProxy(name, ErrorService.class, false); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + svc.go(); + + return null; + } + }, IgniteException.class, "Test exception"); + + } + /** * @throws Exception If failed. */ @@ -371,6 +398,7 @@ protected static class MapServiceImpl implements MapService, Service map.clear(); } + /** {@inheritDoc} */ @Override public int size() { return map.size(); } @@ -390,4 +418,41 @@ protected static class MapServiceImpl implements MapService, Service X.println("Executing cache service: " + ctx.name()); } } + + /** + * + */ + protected interface ErrorService extends Service { + /** + * + */ + void go() throws Exception; + } + + /** + * + */ + protected class ErrorServiceImpl implements ErrorService { + /** {@inheritDoc} */ + @Override public void cancel(ServiceContext ctx) { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void init(ServiceContext ctx) throws Exception { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void execute(ServiceContext ctx) throws Exception { + // No-op. + } + + /** {@inheritDoc} */ + @Override public void go() throws Exception { + throw new Exception("Test exception"); + } + } + + } From 2f1270ee9b3edf8c50562e33275dfbd3807d608a Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 6 Jul 2017 14:53:25 +0300 Subject: [PATCH 083/155] 2.1 Do print stack traces if tx is cancelled on node stop --- .../processors/cache/GridCacheEntryEx.java | 14 +---- .../processors/cache/GridCacheMapEntry.java | 54 +++---------------- .../GridDistributedTxRemoteAdapter.java | 13 ++++- .../dht/GridDhtTransactionalCacheAdapter.java | 2 +- .../near/GridNearTxFinishFuture.java | 2 +- .../cache/transactions/IgniteTxAdapter.java | 28 +++++----- .../cache/transactions/IgniteTxHandler.java | 2 +- .../transactions/IgniteTxLocalAdapter.java | 12 ++++- .../cache/GridCacheTestEntryEx.java | 10 +--- 9 files changed, 48 insertions(+), 89 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java index 4e52680cf94df..b708b4340118e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java @@ -192,23 +192,11 @@ public interface GridCacheEntryEx { /** * Invalidates this entry. * - * @param curVer Current version to match ({@code null} means always match). * @param newVer New version to set. * @return {@code true} if entry is obsolete. * @throws IgniteCheckedException If swap could not be released. */ - public boolean invalidate(@Nullable GridCacheVersion curVer, GridCacheVersion newVer) throws IgniteCheckedException; - - /** - * Invalidates this entry if it passes given filter. - * - * @param filter Optional filter that entry should pass before invalidation. - * @return {@code true} if entry was actually invalidated. - * @throws IgniteCheckedException If swap could not be released. - * @throws GridCacheEntryRemovedException If entry was removed. - */ - public boolean invalidate(@Nullable CacheEntryPredicate[] filter) - throws GridCacheEntryRemovedException, IgniteCheckedException; + public boolean invalidate(GridCacheVersion newVer) throws IgniteCheckedException; /** * @param obsoleteVer Version for eviction. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index eea8935423763..14bd1c55ffca0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -2226,20 +2226,18 @@ protected final boolean markObsolete0(GridCacheVersion ver, boolean clear, GridC } /** {@inheritDoc} */ - @Override public synchronized boolean invalidate(@Nullable GridCacheVersion curVer, GridCacheVersion newVer) + @Override public synchronized boolean invalidate(GridCacheVersion newVer) throws IgniteCheckedException { assert newVer != null; - if (curVer == null || ver.equals(curVer)) { - value(null); + value(null); - ver = newVer; - flags &= ~IS_EVICT_DISABLED; + ver = newVer; + flags &= ~IS_EVICT_DISABLED; - removeValue(); + removeValue(); - onInvalidate(); - } + onInvalidate(); return obsoleteVersionExtras() != null; } @@ -2251,46 +2249,6 @@ protected void onInvalidate() { // No-op. } - /** {@inheritDoc} */ - @Override public boolean invalidate(@Nullable CacheEntryPredicate[] filter) - throws GridCacheEntryRemovedException, IgniteCheckedException { - if (F.isEmptyOrNulls(filter)) { - synchronized (this) { - checkObsolete(); - - invalidate(null, nextVersion()); - - return true; - } - } - else { - // For optimistic checking. - GridCacheVersion startVer; - - synchronized (this) { - checkObsolete(); - - startVer = ver; - } - - if (!cctx.isAll(this, filter)) - return false; - - synchronized (this) { - checkObsolete(); - - if (startVer.equals(ver)) { - invalidate(null, nextVersion()); - - return true; - } - } - - // If version has changed then repeat the process. - return invalidate(filter); - } - } - /** * @param val New value. * @param expireTime Expiration time. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java index 5e3020daf8f25..ea6461db6c17a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java @@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.NodeStoppingException; import org.apache.ignite.internal.pagemem.wal.StorageException; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; @@ -61,6 +62,7 @@ import org.apache.ignite.internal.util.tostring.GridToStringBuilder; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; @@ -716,14 +718,21 @@ else if (op == READ) { } } catch (Throwable ex) { + boolean nodeStopping = X.hasCause(ex, NodeStoppingException.class); + // In case of error, we still make the best effort to commit, // as there is no way to rollback at this point. err = new IgniteTxHeuristicCheckedException("Commit produced a runtime exception " + "(all transaction entries will be invalidated): " + CU.txString(this), ex); - U.error(log, "Commit failed.", err); + if (nodeStopping) { + U.warn(log, "Failed to commit transaction, node is stopping [tx=" + this + + ", err=" + ex + ']'); + } + else + U.error(log, "Commit failed.", err); - uncommit(); + uncommit(nodeStopping); state(UNKNOWN); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java index 5d31581b2b002..73942ffe83243 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java @@ -1700,7 +1700,7 @@ private void invalidateNearEntry(KeyCacheObject key, GridCacheVersion ver) throw GridCacheEntryEx nearEntry = near().peekEx(key); if (nearEntry != null) - nearEntry.invalidate(null, ver); + nearEntry.invalidate(ver); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java index 27cebf8606454..7f6f79337058c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java @@ -349,7 +349,7 @@ else if (err != null) GridCacheEntryEx entry = cacheCtx.cache().peekEx(e.key()); if (entry != null) - entry.invalidate(null, tx.xidVersion()); + entry.invalidate(tx.xidVersion()); } } catch (Throwable t) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java index ee8afb0a785ad..51956ac6cd5bd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java @@ -412,24 +412,28 @@ public void storeEnabled(boolean storeEnabled) { /** * Uncommits transaction by invalidating all of its entries. Courtesy to minimize inconsistency. + * + * @param nodeStopping {@code True} if tx was cancelled during node stop. */ @SuppressWarnings({"CatchGenericClass"}) - protected void uncommit() { + protected void uncommit(boolean nodeStopping) { try { - for (IgniteTxEntry e : writeMap().values()) { - try { - GridCacheEntryEx Entry = e.cached(); + if (!nodeStopping) { + for (IgniteTxEntry e : writeMap().values()) { + try { + GridCacheEntryEx entry = e.cached(); - if (e.op() != NOOP) - Entry.invalidate(null, xidVer); - } - catch (Throwable t) { - U.error(log, "Failed to invalidate transaction entries while reverting a commit.", t); + if (e.op() != NOOP) + entry.invalidate(xidVer); + } + catch (Throwable t) { + U.error(log, "Failed to invalidate transaction entries while reverting a commit.", t); - if (t instanceof Error) - throw (Error)t; + if (t instanceof Error) + throw (Error)t; - break; + break; + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index 7efd6ecbe6c20..c473bfe3f1324 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -1595,7 +1595,7 @@ private void invalidateNearEntry(GridCacheContext cacheCtx, KeyCacheObject key, GridCacheEntryEx nearEntry = near.peekEx(key); if (nearEntry != null) - nearEntry.invalidate(null, ver); + nearEntry.invalidate(ver); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java index e4b850dfcf5f9..49b67da4194b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java @@ -32,6 +32,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.NodeStoppingException; import org.apache.ignite.internal.pagemem.wal.StorageException; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; @@ -828,11 +829,18 @@ assert ownsLock(txEntry.cached()): throw ex; } else { + boolean nodeStopping = X.hasCause(ex, NodeStoppingException.class); + IgniteCheckedException err = new IgniteTxHeuristicCheckedException("Failed to locally write to cache " + "(all transaction entries will be invalidated, however there was a window when " + "entries for this transaction were visible to others): " + this, ex); - U.error(log, "Heuristic transaction failure.", err); + if (nodeStopping) { + U.warn(log, "Failed to commit transaction, node is stopping " + + "[tx=" + this + ", err=" + ex + ']'); + } + else + U.error(log, "Heuristic transaction failure.", err); COMMIT_ERR_UPD.compareAndSet(this, null, err); @@ -840,7 +848,7 @@ assert ownsLock(txEntry.cached()): try { // Courtesy to minimize damage. - uncommit(); + uncommit(nodeStopping); } catch (Throwable ex1) { U.error(log, "Failed to uncommit transaction: " + this, ex1); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java index a6c9cd4d69629..0241165bc9d66 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java @@ -353,21 +353,13 @@ void recheckLock() { } /** @inheritDoc */ - @Override public boolean invalidate(@Nullable GridCacheVersion curVer, GridCacheVersion newVer) + @Override public boolean invalidate(GridCacheVersion newVer) throws IgniteCheckedException { assert false; return false; } - /** @inheritDoc */ - @Override public boolean invalidate(@Nullable CacheEntryPredicate[] filter) - throws GridCacheEntryRemovedException, IgniteCheckedException { - assert false; - - return false; - } - /** @inheritDoc */ @Override public boolean evictInternal(GridCacheVersion obsoleteVer, @Nullable CacheEntryPredicate[] filter, boolean evictOffheap) { From 9c6a65e2723ea8bdb33dde89190a1b12799faf3a Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 6 Jul 2017 15:10:24 +0300 Subject: [PATCH 084/155] 2.1 Added remote node pending messages in diagnostic info --- .../internal/IgniteDiagnosticMessage.java | 8 ++++++ .../IgniteDiagnosticPrepareContext.java | 3 +++ .../processors/cache/GridCacheIoManager.java | 25 ++++++++++++------- .../GridCachePartitionExchangeManager.java | 7 +++++- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticMessage.java index 6e6bac076685a..bd4ec3a315068 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticMessage.java @@ -459,6 +459,14 @@ static void dumpExchangeInfo(StringBuilder sb, GridKernalContext ctx) { .append("Last initialized exchange future: ").append(fut); } + /** + * @param sb String builder. + * @param ctx Context. + */ + static void dumpPendingCacheMessages(StringBuilder sb, GridKernalContext ctx) { + ctx.cache().context().io().dumpPendingMessages(sb); + } + /** * @param ctx Context. * @param nodeId Target node ID. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticPrepareContext.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticPrepareContext.java index b55199a064fc4..416ef29702597 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticPrepareContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticPrepareContext.java @@ -41,6 +41,7 @@ import static org.apache.ignite.internal.IgniteDiagnosticMessage.dumpCommunicationInfo; import static org.apache.ignite.internal.IgniteDiagnosticMessage.dumpExchangeInfo; import static org.apache.ignite.internal.IgniteDiagnosticMessage.dumpNodeBasicInfo; +import static org.apache.ignite.internal.IgniteDiagnosticMessage.dumpPendingCacheMessages; /** * Groups diagnostic closures by node/closure type. @@ -193,6 +194,8 @@ private final static class CompoundInfoClosure implements IgniteClosure pendingMsgs = new ArrayList<>(MAX_STORED_PENDING_MESSAGES); /** - * + * @param sb String builder. */ - public void dumpPendingMessages() { + public void dumpPendingMessages(StringBuilder sb) { synchronized (pendingMsgs) { if (pendingMsgs.isEmpty()) return; - diagnosticLog.info("Pending cache messages waiting for exchange [" + - "readyVer=" + cctx.exchange().readyAffinityVersion() + - ", discoVer=" + cctx.discovery().topologyVersion() + ']'); + sb.append("Pending cache messages waiting for exchange [readyVer="). + append(cctx.exchange().readyAffinityVersion()). + append(", discoVer="). + append(cctx.discovery().topologyVersion()).append(']'); + + sb.append(nl()); - for (GridCacheMessage msg : pendingMsgs) - diagnosticLog.info("Message [waitVer=" + msg.topologyVersion() + ", msg=" + msg + ']'); + for (GridCacheMessage msg : pendingMsgs) { + sb.append("Message [waitVer=").append(msg.topologyVersion()).append(", msg=").append(msg).append(']'); + + sb.append(nl()); + } } } @@ -341,12 +348,12 @@ private void handleMessage(UUID nodeId, GridCacheMessage cacheMsg, MessageHandle append(", desc=").append(descriptorForMessage(cacheMsg)). append(']'); - msg0.append(U.nl()).append("Registered listeners:"); + msg0.append(nl()).append("Registered listeners:"); Map idxClsHandlers0 = msgHandlers.idxClsHandlers; for (Map.Entry e : idxClsHandlers0.entrySet()) - msg0.append(U.nl()).append(e.getKey()).append("=").append(Arrays.toString(e.getValue())); + msg0.append(nl()).append(e.getKey()).append("=").append(Arrays.toString(e.getValue())); if (cctx.kernalContext().isStopping()) { if (log.isDebugEnabled()) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index b3cbd174bac4e..8506cdebd0d08 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -1537,7 +1537,12 @@ public void dumpDebugInfo(@Nullable GridDhtPartitionsExchangeFuture exchFut) thr cctx.affinity().dumpDebugInfo(); - cctx.io().dumpPendingMessages(); + StringBuilder pendingMsgs = new StringBuilder(); + + cctx.io().dumpPendingMessages(pendingMsgs); + + if (pendingMsgs.length() > 0) + diagnosticLog.info(pendingMsgs.toString()); if (IgniteSystemProperties.getBoolean(IGNITE_IO_DUMP_ON_TIMEOUT, false)) cctx.gridIO().dumpStats(); From 64d6c98d88656132e89977806ca0e9152eb62427 Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 6 Jul 2017 15:55:21 +0300 Subject: [PATCH 085/155] 2.1 Added test. --- .../cache/IgniteCacheGroupsTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java index 7b420ccbe6d63..d3269c30da6af 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java @@ -3628,6 +3628,66 @@ public void testCacheIdSort() throws Exception { assertEquals(3, c); } + /** + * @throws Exception If failed. + */ + public void testDataCleanup() throws Exception { + Ignite node = startGrid(0); + + IgniteCache cache0 = node.createCache(cacheConfiguration(GROUP1, "c0", PARTITIONED, ATOMIC, 1, false)); + + for (int i = 0; i < 100; i++) + assertNull(cache0.get(i)); + + for (int i = 0; i < 100; i++) + cache0.put(i, i); + + List ccfgs = new ArrayList<>(); + + ccfgs.add(cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1, false)); + ccfgs.add(cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1, true)); + ccfgs.add(cacheConfiguration(GROUP1, "c1", PARTITIONED, TRANSACTIONAL, 1, false)); + ccfgs.add(cacheConfiguration(GROUP1, "c1", PARTITIONED, TRANSACTIONAL, 1, true)); + + for (CacheConfiguration ccfg : ccfgs) { + IgniteCache cache = node.createCache(ccfg); + + for (int i = 0; i < 100; i++) + assertNull(cache.get(i)); + + for (int i = 0; i < 100; i++) + cache.put(i, i); + + for (int i = 0; i < 100; i++) + assertEquals(i, cache.get(i)); + + node.destroyCache(ccfg.getName()); + + cache = node.createCache(ccfg); + + for (int i = 0; i < 100; i++) + assertNull(cache.get(i)); + + node.destroyCache(ccfg.getName()); + } + + for (int i = 0; i < 100; i++) + assertEquals(i, cache0.get(i)); + + node.destroyCache(cache0.getName()); + + cache0 = node.createCache(cacheConfiguration(GROUP1, "c0", PARTITIONED, ATOMIC, 1, false)); + + for (int i = 0; i < 100; i++) + assertNull(cache0.get(i)); + + for (int i = 0; i < 100; i++) + cache0.put(i, i); + + for (int i = 0; i < 100; i++) + assertEquals(i, cache0.get(i)); + } + /** * @throws Exception If failed. */ From 0b4b5dc0e6763f4cf27ed8c9d38c6f4b10037ef6 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 7 Jul 2017 11:00:52 +0300 Subject: [PATCH 086/155] Added missing header to ByteBufferExpander. --- .../persistence/wal/ByteBufferExpander.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferExpander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferExpander.java index 75d3a98c366d7..829cd5c25b278 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferExpander.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferExpander.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.cache.persistence.wal; import java.nio.ByteBuffer; From 516e73a4baf21c86f46bee03a00e336db3ffe567 Mon Sep 17 00:00:00 2001 From: vsisko Date: Fri, 7 Jul 2017 15:19:08 +0700 Subject: [PATCH 087/155] IGNITE-5710 Change H2 version in depend to Ignite version. --- .../frontend/app/data/pom-dependencies.json | 5 ++++- .../configuration/generator/Maven.service.js | 21 ++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/modules/web-console/frontend/app/data/pom-dependencies.json b/modules/web-console/frontend/app/data/pom-dependencies.json index 3e0543bb64861..945e3f55f0886 100644 --- a/modules/web-console/frontend/app/data/pom-dependencies.json +++ b/modules/web-console/frontend/app/data/pom-dependencies.json @@ -14,7 +14,10 @@ "Generic": {"groupId": "com.mchange", "artifactId": "c3p0", "version": "0.9.5.2"}, "MySQL": {"groupId": "mysql", "artifactId": "mysql-connector-java", "version": "5.1.40"}, "PostgreSQL": {"groupId": "org.postgresql", "artifactId": "postgresql", "version": "9.4.1212.jre7"}, - "H2": {"groupId": "com.h2database", "artifactId": "h2", "version": "1.4.191"}, + "H2": {"groupId": "com.h2database", "artifactId": "h2", "version": [ + {"range": ["1.0.0", "2.0.0"], "version": "1.4.191"}, + {"range": "2.0.0", "version": "1.4.195"} + ]}, "Oracle": {"groupId": "com.oracle.jdbc", "artifactId": "ojdbc7", "version": "12.1.0.2", "jar": "ojdbc7.jar"}, "DB2": {"groupId": "ibm", "artifactId": "jdbc", "version": "4.21.29", "jar": "db2jcc4.jar"}, "SQLServer": {"groupId": "microsoft", "artifactId": "jdbc", "version": "4.2", "jar": "sqljdbc41.jar"} diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js index da098fc0341a0..700da4a6d3ac8 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js @@ -16,6 +16,9 @@ */ import StringBuilder from './StringBuilder'; +import VersionService from 'app/modules/configuration/Version.service'; + +const versionService = new VersionService(); // Java built-in class names. import POM_DEPENDENCIES from 'app/data/pom-dependencies.json'; @@ -39,13 +42,17 @@ export default class IgniteMavenGenerator { deps.add({groupId, artifactId, version, jar}); } - pickDependency(deps, key, dfltVer) { + pickDependency(deps, key, dfltVer, igniteVer) { + const extractVersion = (version) => { + return _.isArray(version) ? _.find(version, (v) => versionService.since(igniteVer, v.range)).version : version; + }; + if (!_.has(POM_DEPENDENCIES, key)) return; const {groupId, artifactId, version, jar} = POM_DEPENDENCIES[key]; - this.addDependency(deps, groupId || 'org.apache.ignite', artifactId, version || dfltVer, jar); + this.addDependency(deps, groupId || 'org.apache.ignite', artifactId, extractVersion(version) || dfltVer, jar); } addResource(sb, dir, exclude) { @@ -139,9 +146,9 @@ export default class IgniteMavenGenerator { * @param deps Already added dependencies. * @param storeFactory Store factory to add dependency. */ - storeFactoryDependency(deps, storeFactory) { + storeFactoryDependency(deps, storeFactory, igniteVer) { if (storeFactory.dialect && (!storeFactory.connectVia || storeFactory.connectVia === 'DataSource')) - this.pickDependency(deps, storeFactory.dialect); + this.pickDependency(deps, storeFactory.dialect, null, igniteVer); } collectDependencies(cluster, targetVer) { @@ -167,7 +174,7 @@ export default class IgniteMavenGenerator { _.forEach(caches, (cache) => { if (cache.cacheStoreFactory && cache.cacheStoreFactory.kind) - this.storeFactoryDependency(storeDeps, cache.cacheStoreFactory[cache.cacheStoreFactory.kind]); + this.storeFactoryDependency(storeDeps, cache.cacheStoreFactory[cache.cacheStoreFactory.kind], igniteVer); if (_.get(cache, 'nodeFilter.kind') === 'Exclude') this.addDependency(deps, 'org.apache.ignite', 'ignite-extdata-p2p', igniteVer); @@ -177,14 +184,14 @@ export default class IgniteMavenGenerator { const store = cluster.discovery.Jdbc; if (store.dataSourceBean && store.dialect) - this.storeFactoryDependency(storeDeps, cluster.discovery.Jdbc); + this.storeFactoryDependency(storeDeps, cluster.discovery.Jdbc, igniteVer); } _.forEach(cluster.checkpointSpi, (spi) => { if (spi.kind === 'S3') this.pickDependency(deps, spi.kind, igniteVer); else if (spi.kind === 'JDBC') - this.storeFactoryDependency(storeDeps, spi.JDBC); + this.storeFactoryDependency(storeDeps, spi.JDBC, igniteVer); }); if (_.get(cluster, 'hadoopConfiguration.mapReducePlanner.kind') === 'Weighted' || From 3259e2bb9d6532dd78e71314dc55a7a021090f36 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 7 Jul 2017 11:27:31 +0300 Subject: [PATCH 088/155] ignite-5075 Fixed row size calculation in canUpdateOldRow --- .../cache/IgniteCacheOffheapManagerImpl.java | 7 +++++-- .../cache/persistence/freelist/FreeListImpl.java | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index b51fc104a9245..6d16b604f27b9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -1179,12 +1179,15 @@ private boolean canUpdateOldRow(GridCacheContext cctx, @Nullable CacheDataRow ol if (oldRow.expireTime() != dataRow.expireTime()) return false; - int oldLen = FreeListImpl.getRowSize(oldRow); + // Use grp.sharedGroup() flag since it is possible cacheId is not yet set here. + boolean sizeWithCacheId = grp.sharedGroup(); + + int oldLen = FreeListImpl.getRowSize(oldRow, sizeWithCacheId); if (oldLen > updateValSizeThreshold) return false; - int newLen = FreeListImpl.getRowSize(dataRow); + int newLen = FreeListImpl.getRowSize(dataRow, sizeWithCacheId); return oldLen == newLen; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java index 139c6f3e08224..844bc0227d332 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java @@ -101,7 +101,7 @@ private final class UpdateRowHandler extends PageHandler throws IgniteCheckedException { DataPageIO io = (DataPageIO)iox; - int rowSize = getRowSize(row); + int rowSize = getRowSize(row, row.cacheId() != 0); boolean updated = io.updateRow(pageAddr, itemId, pageSize(), null, row, rowSize); @@ -147,7 +147,7 @@ private final class WriteRowHandler extends PageHandler { throws IgniteCheckedException { DataPageIO io = (DataPageIO)iox; - int rowSize = getRowSize(row); + int rowSize = getRowSize(row, row.cacheId() != 0); int oldFreeSpace = io.getFreeSpace(pageAddr); assert oldFreeSpace > 0 : oldFreeSpace; @@ -453,7 +453,7 @@ private long allocateDataPage(int part) throws IgniteCheckedException { /** {@inheritDoc} */ @Override public void insertDataRow(CacheDataRow row) throws IgniteCheckedException { - int rowSize = getRowSize(row); + int rowSize = getRowSize(row, row.cacheId() != 0); int written = 0; @@ -579,17 +579,18 @@ public int emptyDataPages() { /** * @param row Row. + * @param withCacheId If {@code true} adds cache ID size. * @return Entry size on page. * @throws IgniteCheckedException If failed. */ - public static int getRowSize(CacheDataRow row) throws IgniteCheckedException { + public static int getRowSize(CacheDataRow row, boolean withCacheId) throws IgniteCheckedException { KeyCacheObject key = row.key(); CacheObject val = row.value(); int keyLen = key.valueBytesLength(null); int valLen = val.valueBytesLength(null); - return keyLen + valLen + CacheVersionIO.size(row.version(), false) + 8 + (row.cacheId() == 0 ? 0 : 4); + return keyLen + valLen + CacheVersionIO.size(row.version(), false) + 8 + (withCacheId ? 4 : 0); } /** {@inheritDoc} */ From ab5267134966b466a2d4bfba15d0c7e3ebe91139 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 7 Jul 2017 13:14:18 +0300 Subject: [PATCH 089/155] 2.1 More simple tx cancel on node stop --- .../processors/cache/GridCacheProcessor.java | 2 ++ .../distributed/dht/GridDhtTxFinishFuture.java | 4 +++- .../distributed/near/GridNearTxFinishFuture.java | 4 ++-- .../cache/transactions/IgniteTxAdapter.java | 4 ++-- .../cache/transactions/IgniteTxLocalAdapter.java | 15 +++++++++------ 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 716482e4d4957..321e6dd31ed6c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -964,6 +964,8 @@ public void blockGateways() { // No new caches should be added after this point. exch.onKernalStop(cancel); + sharedCtx.mvcc().onStop(); + for (CacheGroupContext grp : cacheGrps.values()) grp.onKernalStop(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java index d8180b452f289..5311ddc84495d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java @@ -225,7 +225,9 @@ public void onResult(UUID nodeId, GridDhtTxFinishResponse res) { if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING)) { try { - this.tx.tmFinish(err == null); + boolean nodeStop = err != null && X.hasCause(err, NodeStoppingException.class); + + this.tx.tmFinish(err == null, nodeStop); } catch (IgniteCheckedException finishErr) { U.error(log, "Failed to finish tx: " + tx, e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java index 7f6f79337058c..c45eb7bd26178 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java @@ -327,7 +327,7 @@ else if (err != null) finishOnePhase(commit); try { - tx.tmFinish(commit); + tx.tmFinish(commit, nodeStop); } catch (IgniteCheckedException e) { U.error(log, "Failed to finish tx: " + tx, e); @@ -338,7 +338,7 @@ else if (err != null) } if (super.onDone(tx0, err)) { - if (error() instanceof IgniteTxHeuristicCheckedException) { + if (error() instanceof IgniteTxHeuristicCheckedException && !nodeStop) { AffinityTopologyVersion topVer = tx.topologyVersion(); for (IgniteTxEntry e : tx.writeMap().values()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java index 51956ac6cd5bd..880d9b91299aa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java @@ -435,9 +435,9 @@ protected void uncommit(boolean nodeStopping) { break; } } - } - cctx.tm().uncommitTx(this); + cctx.tm().uncommitTx(this); + } } catch (Exception ex) { U.error(log, "Failed to do uncommit.", ex); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java index 49b67da4194b0..e7ebaaea7670d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java @@ -898,17 +898,20 @@ assert ownsLock(txEntry.cached()): * Commits transaction to transaction manager. Used for one-phase commit transactions only. * * @param commit If {@code true} commits transaction, otherwise rollbacks. + * @param nodeStop If {@code true} tx is cancelled on node stop. * @throws IgniteCheckedException If failed. */ - public void tmFinish(boolean commit) throws IgniteCheckedException { + public void tmFinish(boolean commit, boolean nodeStop) throws IgniteCheckedException { assert onePhaseCommit(); if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) { - // Unlock all locks. - if (commit) - cctx.tm().commitTx(this); - else - cctx.tm().rollbackTx(this); + if (!nodeStop) { + // Unlock all locks. + if (commit) + cctx.tm().commitTx(this); + else + cctx.tm().rollbackTx(this); + } state(commit ? COMMITTED : ROLLED_BACK); From 85d8c6572d78c7d6796906d92b566e2fa5b116be Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Fri, 7 Jul 2017 13:13:55 +0300 Subject: [PATCH 090/155] IGNITE-5520 - Fixed IgniteChangeGlobalStateFailoverTest --- .../IgniteChangeGlobalStateFailOverTest.java | 56 ++++++++++++------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java index 92d1f2198b3b8..02a21f475f8e4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java @@ -21,11 +21,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.util.typedef.internal.U; import static java.lang.Thread.sleep; import static org.apache.ignite.testframework.GridTestUtils.runAsync; @@ -144,8 +144,6 @@ public void testActivateDeActivateOnFixTopology() throws Exception { * @throws Exception If failed. */ public void testActivateDeActivateOnJoiningNode() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-5520"); - final Ignite igB1 = backUp(0); final Ignite igB2 = backUp(1); final Ignite igB3 = backUp(2); @@ -162,14 +160,17 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { final AtomicBoolean stop = new AtomicBoolean(); - final AtomicBoolean canAct = new AtomicBoolean(true); final AtomicInteger seqIdx = new AtomicInteger(backUpNodes()); + final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); + try { final IgniteInternalFuture af = runAsync(new Callable() { @Override public Void call() throws Exception { while (!stop.get()) { - if (canAct.get()) { + rwLock.readLock().lock(); + + try { Ignite ig = randomBackUp(false); long start = System.currentTimeMillis(); @@ -182,13 +183,12 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { for (Ignite ign : allBackUpNodes()) assertTrue(ign.active()); - - canAct.set(false); } - else - U.sleep(100); - + finally { + rwLock.readLock().unlock(); + } } + return null; } }); @@ -196,7 +196,9 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { final IgniteInternalFuture df = runAsync(new Callable() { @Override public Void call() throws Exception { while (!stop.get()) { - if (!canAct.get()) { + rwLock.writeLock().lock(); + + try { Ignite ig = randomBackUp(false); long start = System.currentTimeMillis(); @@ -209,20 +211,28 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { for (Ignite ign : allBackUpNodes()) assertTrue(!ign.active()); - - canAct.set(true); } - else - U.sleep(100); + finally { + rwLock.writeLock().unlock(); + } } + return null; } }); IgniteInternalFuture jf1 = runAsync(new Callable() { @Override public Void call() throws Exception { - while (!stop.get()) - startBackUp(seqIdx.incrementAndGet()); + while (!stop.get()) { + rwLock.readLock().lock(); + + try { + startBackUp(seqIdx.incrementAndGet()); + } + finally { + rwLock.readLock().unlock(); + } + } return null; } @@ -230,8 +240,16 @@ public void testActivateDeActivateOnJoiningNode() throws Exception { IgniteInternalFuture jf2 = runAsync(new Callable() { @Override public Void call() throws Exception { - while (!stop.get()) - startBackUp(seqIdx.incrementAndGet()); + while (!stop.get()) { + rwLock.readLock().lock(); + + try { + startBackUp(seqIdx.incrementAndGet()); + } + finally { + rwLock.readLock().unlock(); + } + } return null; } From f589628f4846f14efc8e702b61856cc90a3d0dc7 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 7 Jul 2017 13:52:38 +0300 Subject: [PATCH 091/155] Rename ActionData -> CacheActionData. --- .../cache/CacheAffinitySharedManager.java | 10 +++--- .../processors/cache/ClusterCachesInfo.java | 2 +- .../processors/cache/ExchangeActions.java | 34 +++++++++---------- .../processors/cache/GridCacheProcessor.java | 2 +- .../GridDhtPartitionsExchangeFuture.java | 2 +- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index 8d08c3f064c53..548d7955ede8a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -701,7 +701,7 @@ public void onCacheChangeRequest( } }); - for (ExchangeActions.ActionData action : exchActions.cacheStartRequests()) { + for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) { DynamicCacheDescriptor cacheDesc = action.descriptor(); DynamicCacheChangeRequest req = action.request(); @@ -768,7 +768,7 @@ public void onCacheChangeRequest( Set gprs = new HashSet<>(); - for (ExchangeActions.ActionData action : exchActions.cacheStartRequests()) { + for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) { Integer grpId = action.descriptor().groupId(); if (gprs.add(grpId)) { @@ -786,7 +786,7 @@ public void onCacheChangeRequest( } } - for (ExchangeActions.ActionData action : exchActions.cacheStopRequests()) + for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) cctx.cache().blockGateway(action.request().cacheName(), true, action.request().restart()); for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) { @@ -2308,10 +2308,10 @@ void updateCachesInfo(ExchangeActions exchActions) { assert old == null : old; } - for (ExchangeActions.ActionData req : exchActions.cacheStopRequests()) + for (ExchangeActions.CacheActionData req : exchActions.cacheStopRequests()) registeredCaches.remove(req.descriptor().cacheId()); - for (ExchangeActions.ActionData req : exchActions.cacheStartRequests()) + for (ExchangeActions.CacheActionData req : exchActions.cacheStartRequests()) registeredCaches.put(req.descriptor().cacheId(), req.descriptor()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java index 5aca8c9811ae1..738e4ac64e46f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java @@ -624,7 +624,7 @@ else if (req.stop()) { // If all caches in group will be destroyed it is not necessary to destroy single cache // because group will be stopped anyway. if (req.destroy()) { - for (ExchangeActions.ActionData action : exchangeActions.cacheStopRequests()) { + for (ExchangeActions.CacheActionData action : exchangeActions.cacheStopRequests()) { if (action.descriptor().groupId() == grpDesc.groupId()) action.request().destroy(false); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java index e9ece5afb5576..1cc6438471d21 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java @@ -41,13 +41,13 @@ public class ExchangeActions { private List cacheGrpsToStop; /** */ - private Map cachesToStart; + private Map cachesToStart; /** */ - private Map cachesToStop; + private Map cachesToStop; /** */ - private Map cachesToResetLostParts; + private Map cachesToResetLostParts; /** */ private StateChangeRequest stateChangeReq; @@ -60,7 +60,7 @@ boolean checkStopRequestConsistency(int grpId) { Boolean destroy = null; // Check that caches associated with that group will be all stopped only or all destroyed. - for (ExchangeActions.ActionData action : cacheStopRequests()) { + for (CacheActionData action : cacheStopRequests()) { if (action.descriptor().groupId() == grpId) { if (destroy == null) destroy = action.request().destroy(); @@ -89,15 +89,15 @@ public boolean clientOnlyExchange() { /** * @return New caches start requests. */ - public Collection cacheStartRequests() { - return cachesToStart != null ? cachesToStart.values() : Collections.emptyList(); + public Collection cacheStartRequests() { + return cachesToStart != null ? cachesToStart.values() : Collections.emptyList(); } /** * @return Stop cache requests. */ - Collection cacheStopRequests() { - return cachesToStop != null ? cachesToStop.values() : Collections.emptyList(); + Collection cacheStopRequests() { + return cachesToStop != null ? cachesToStop.values() : Collections.emptyList(); } /** @@ -114,7 +114,7 @@ public void completeRequestFutures(GridCacheSharedContext ctx) { */ public boolean systemCachesStarting() { if (cachesToStart != null) { - for (ActionData data : cachesToStart.values()) { + for (CacheActionData data : cachesToStart.values()) { if (CU.isSystemCache(data.request().cacheName())) return true; } @@ -127,9 +127,9 @@ public boolean systemCachesStarting() { * @param map Actions map. * @param ctx Context. */ - private void completeRequestFutures(Map map, GridCacheSharedContext ctx) { + private void completeRequestFutures(Map map, GridCacheSharedContext ctx) { if (map != null) { - for (ActionData req : map.values()) + for (CacheActionData req : map.values()) ctx.cache().completeCacheStartFuture(req.req, true, null); } } @@ -159,7 +159,7 @@ public Set cachesToResetLostPartitions() { */ public boolean cacheStopped(int cacheId) { if (cachesToStop != null) { - for (ActionData cache : cachesToStop.values()) { + for (CacheActionData cache : cachesToStop.values()) { if (cache.desc.cacheId() == cacheId) return true; } @@ -174,7 +174,7 @@ public boolean cacheStopped(int cacheId) { */ public boolean cacheStarted(int cacheId) { if (cachesToStart != null) { - for (ActionData cache : cachesToStart.values()) { + for (CacheActionData cache : cachesToStart.values()) { if (cache.desc.cacheId() == cacheId) return true; } @@ -217,7 +217,7 @@ public boolean activate() { * @param desc Cache descriptor. * @return Actions map. */ - private Map add(Map map, + private Map add(Map map, DynamicCacheChangeRequest req, DynamicCacheDescriptor desc) { assert req != null; @@ -226,7 +226,7 @@ private Map add(Map map, if (map == null) map = new HashMap<>(); - ActionData old = map.put(req.cacheName(), new ActionData(req, desc)); + CacheActionData old = map.put(req.cacheName(), new CacheActionData(req, desc)); assert old == null : old; @@ -347,7 +347,7 @@ public boolean empty() { /** * */ - public static class ActionData { + public static class CacheActionData { /** */ private final DynamicCacheChangeRequest req; @@ -358,7 +358,7 @@ public static class ActionData { * @param req Request. * @param desc Cache descriptor. */ - ActionData(DynamicCacheChangeRequest req, DynamicCacheDescriptor desc) { + CacheActionData(DynamicCacheChangeRequest req, DynamicCacheDescriptor desc) { assert req != null; assert desc != null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 321e6dd31ed6c..0a69d728ea855 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -2126,7 +2126,7 @@ public void onExchangeDone( } } - for (ExchangeActions.ActionData action : exchActions.cacheStopRequests()) { + for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) { stopGateway(action.request()); sharedCtx.database().checkpointReadLock(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 215110190db27..97fcb12bf5bc6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -767,7 +767,7 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) throws IgniteCheck if (cctx.database().persistenceEnabled() && !cctx.kernalContext().clientNode()) { List startDescs = new ArrayList<>(); - for (ExchangeActions.ActionData startReq : exchActions.cacheStartRequests()) + for (ExchangeActions.CacheActionData startReq : exchActions.cacheStartRequests()) startDescs.add(startReq.descriptor()); cctx.database().readCheckpointAndRestoreMemory(startDescs); From 30922ed8ac518749a2559aff51fe183a890041bb Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 7 Jul 2017 18:03:18 +0700 Subject: [PATCH 092/155] ignite-2.1 Added map with previous snapshots. --- .../pagemem/snapshot/SnapshotOperation.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java index 863107a3ac618..fa18cd7a01045 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.Serializable; import java.util.Collection; +import java.util.Map; import java.util.Set; /** @@ -50,6 +51,9 @@ public class SnapshotOperation implements Serializable { /** Optional list of dependent snapshot IDs. */ private final Set dependentSnapshotIds; + /** Optional map of previous snapshots grouped by caches. */ + private final Map> prevSnapshots; + /** * @param type Type. * @param snapshotId Snapshot id. @@ -58,6 +62,7 @@ public class SnapshotOperation implements Serializable { * @param msg Extra user message. * @param extraParam Additional parameter. * @param dependentSnapshotIds Optional list of dependent snapshot IDs. + * @param prevSnapshots Optional map of previous snapshots grouped by caches. */ public SnapshotOperation( SnapshotOperationType type, @@ -66,7 +71,8 @@ public SnapshotOperation( Set cacheNames, String msg, Object extraParam, - Set dependentSnapshotIds + Set dependentSnapshotIds, + Map> prevSnapshots ) { this.type = type; this.snapshotId = snapshotId; @@ -75,6 +81,7 @@ public SnapshotOperation( this.msg = msg; this.extraParam = extraParam; this.dependentSnapshotIds = dependentSnapshotIds; + this.prevSnapshots = prevSnapshots; } /** @@ -130,6 +137,13 @@ public Set dependentSnapshotIds() { return dependentSnapshotIds; } + /** + * @return Cache names grouped by previous snapshot IDs. + */ + public Map> previousSnapshots() { + return prevSnapshots; + } + /** * @param op Op. */ @@ -215,6 +229,7 @@ public static File getMovingPathParameter(SnapshotOperation op) { ", msg='" + msg + '\'' + ", extraParam=" + extraParam + ", dependentSnapshotIds=" + dependentSnapshotIds + + ", prevSnapshots=" + prevSnapshots + '}'; } } From 99fd75dd7766318af88a38b5640a648bba3ccdf1 Mon Sep 17 00:00:00 2001 From: Sergey Kalashnikov Date: Fri, 7 Jul 2017 14:38:36 +0300 Subject: [PATCH 093/155] IGNITE-5483: Added limited support for Java8 LocalDateTime. This closes #2248. --- .../query/h2/DmlStatementsProcessor.java | 19 +++++++++++++++++++ .../processors/query/h2/H2DatabaseType.java | 10 ++++++++++ .../processors/query/h2/H2RowDescriptor.java | 13 +++++++++++++ 3 files changed, 42 insertions(+) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 98d123f4f9dec..0c1dbf983d7ce 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -20,6 +20,8 @@ import java.lang.reflect.Array; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -79,8 +81,13 @@ import org.h2.command.dml.Merge; import org.h2.command.dml.Update; import org.h2.table.Column; +import org.h2.util.DateTimeUtils; +import org.h2.util.LocalDateTimeUtils; import org.h2.value.DataType; import org.h2.value.Value; +import org.h2.value.ValueDate; +import org.h2.value.ValueTime; +import org.h2.value.ValueTimestamp; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -711,6 +718,18 @@ private static Object convert(Object val, GridH2RowDescriptor desc, Class exp return U.unmarshal(desc.context().marshaller(), (byte[]) val, U.resolveClassLoader(desc.context().gridConfig())); + if (LocalDateTimeUtils.isJava8DateApiPresent()) { + if (val instanceof Timestamp && LocalDateTimeUtils.isLocalDateTime(expCls)) + return LocalDateTimeUtils.valueToLocalDateTime(ValueTimestamp.get((Timestamp)val)); + + if (val instanceof Date && LocalDateTimeUtils.isLocalDate(expCls)) + return LocalDateTimeUtils.valueToLocalDate(ValueDate.fromDateValue( + DateTimeUtils.dateValueFromDate(((Date)val).getTime()))); + + if (val instanceof Time && LocalDateTimeUtils.isLocalTime(expCls)) + return LocalDateTimeUtils.valueToLocalTime(ValueTime.get((Time)val)); + } + // We have to convert arrays of reference types manually - see https://issues.apache.org/jira/browse/IGNITE-4327 // Still, we only can convert from Object[] to something more precise. if (type == Value.ARRAY && currCls != expCls) { diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DatabaseType.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DatabaseType.java index 47c7eb9c83304..8e4e6392d6bd2 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DatabaseType.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DatabaseType.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.query.h2; import org.apache.ignite.internal.util.typedef.internal.S; +import org.h2.util.LocalDateTimeUtils; import org.h2.value.DataType; import java.math.BigDecimal; @@ -142,6 +143,15 @@ public static H2DatabaseType fromClass(Class cls) { if (DataType.isGeometryClass(cls)) return GEOMETRY; + if (LocalDateTimeUtils.isJava8DateApiPresent()) { + if (LocalDateTimeUtils.isLocalDate(cls)) + return DATE; + else if (LocalDateTimeUtils.isLocalTime(cls)) + return TIME; + else if (LocalDateTimeUtils.isLocalDateTime(cls)) + return TIMESTAMP; + } + return cls.isArray() && !cls.getComponentType().isPrimitive() ? ARRAY : OTHER; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2RowDescriptor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2RowDescriptor.java index a9bbd23cedfbb..dab83d187181e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2RowDescriptor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2RowDescriptor.java @@ -36,6 +36,7 @@ import org.h2.mvstore.cache.CacheLongKeyLIRS; import org.h2.result.SearchRow; import org.h2.result.SimpleRow; +import org.h2.util.LocalDateTimeUtils; import org.h2.value.DataType; import org.h2.value.Value; import org.h2.value.ValueArray; @@ -246,14 +247,26 @@ public class H2RowDescriptor implements GridH2RowDescriptor { UUID uuid = (UUID)obj; return ValueUuid.get(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); case Value.DATE: + if (LocalDateTimeUtils.isLocalDate(obj.getClass())) + return LocalDateTimeUtils.localDateToDateValue(obj); + return ValueDate.get((Date)obj); + case Value.TIME: + if (LocalDateTimeUtils.isLocalTime(obj.getClass())) + return LocalDateTimeUtils.localTimeToTimeValue(obj); + return ValueTime.get((Time)obj); + case Value.TIMESTAMP: if (obj instanceof java.util.Date && !(obj instanceof Timestamp)) obj = new Timestamp(((java.util.Date)obj).getTime()); + if (LocalDateTimeUtils.isLocalDateTime(obj.getClass())) + return LocalDateTimeUtils.localDateTimeToValue(obj); + return ValueTimestamp.get((Timestamp)obj); + case Value.DECIMAL: return ValueDecimal.get((BigDecimal)obj); case Value.STRING: From 4fc8124d9538625cb0a95c918634b43ec1319bee Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 7 Jul 2017 14:43:28 +0300 Subject: [PATCH 094/155] Fixed formatting. --- .../dht/GridDhtPartitionTopologyImpl.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index 3e98b01376d45..320ef06dd3182 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -440,18 +440,16 @@ else if (localNode(p, aff)) U.writeLock(lock); - try { - - - if (stopping) - return; + try { + if (stopping) + return; - GridDhtPartitionExchangeId exchId = exchFut.exchangeId();assert topVer.equals(exchId.topologyVersion()) : "Invalid topology version [topVer=" + - topVer + ", exchId=" + exchId + ']'; + GridDhtPartitionExchangeId exchId = exchFut.exchangeId();assert topVer.equals(exchId.topologyVersion()) : "Invalid topology version [topVer=" + + topVer + ", exchId=" + exchId + ']'; if (exchId.isLeft() && exchFut.serverNodeDiscoveryEvent()) removeNode(exchId.nodeId()); - + ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache(); if (log.isDebugEnabled()) @@ -461,10 +459,11 @@ else if (localNode(p, aff)) cntrMap.clear(); - boolean grpStarted = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom());// If this is the oldest node. - if (oldest != null && (loc.equals(oldest) || grpStarted)) { - if (node2part == null) { - node2part = new GridDhtPartitionFullMap(oldest.id(), oldest.order(), updateSeq); + boolean grpStarted = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom());// If this is the oldest node. + + if (oldest != null && (loc.equals(oldest) || grpStarted)) { + if (node2part == null) { + node2part = new GridDhtPartitionFullMap(oldest.id(), oldest.order(), updateSeq); if (log.isDebugEnabled()) log.debug("Created brand new full topology map on oldest node [exchId=" + @@ -486,16 +485,17 @@ else if (!node2part.nodeId().equals(loc.id())) { } } - if (grpStarted || - exchFut.discoveryEvent().type() == EVT_DISCOVERY_CUSTOM_EVT || - exchFut.serverNodeDiscoveryEvent()) {if (affReady) - initPartitions0(exchFut, updateSeq); - else { - List> aff = grp.affinity().idealAssignment(); + if (grpStarted || + exchFut.discoveryEvent().type() == EVT_DISCOVERY_CUSTOM_EVT || + exchFut.serverNodeDiscoveryEvent()) { + if (affReady) + initPartitions0(exchFut, updateSeq); + else { + List> aff = grp.affinity().idealAssignment(); - createPartitions(aff, updateSeq); + createPartitions(aff, updateSeq); + } } - } consistencyCheck(); From 7adb11109bab5d83ed4f376b0cad42b026dd0a71 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 7 Jul 2017 14:49:13 +0300 Subject: [PATCH 095/155] Fixed formatting. --- .../dht/GridDhtPartitionTopologyImpl.java | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index 320ef06dd3182..f49dccfe0706d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -444,23 +444,28 @@ else if (localNode(p, aff)) if (stopping) return; - GridDhtPartitionExchangeId exchId = exchFut.exchangeId();assert topVer.equals(exchId.topologyVersion()) : "Invalid topology version [topVer=" + - topVer + ", exchId=" + exchId + ']'; + GridDhtPartitionExchangeId exchId = exchFut.exchangeId(); + + assert topVer.equals(exchId.topologyVersion()) : "Invalid topology version [topVer=" + topVer + + ", exchId=" + exchId + ']'; if (exchId.isLeft() && exchFut.serverNodeDiscoveryEvent()) removeNode(exchId.nodeId()); ClusterNode oldest = discoCache.oldestAliveServerNodeWithCache(); - if (log.isDebugEnabled()) - log.debug("Partition map beforeExchange [exchId=" + exchId + ", fullMap=" + fullMapString() + ']'); + if (log.isDebugEnabled()) { + log.debug("Partition map beforeExchange [exchId=" + exchId + + ", fullMap=" + fullMapString() + ']'); + } long updateSeq = this.updateSeq.incrementAndGet(); cntrMap.clear(); - boolean grpStarted = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom());// If this is the oldest node. + boolean grpStarted = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom()); + // If this is the oldest node. if (oldest != null && (loc.equals(oldest) || grpStarted)) { if (node2part == null) { node2part = new GridDhtPartitionFullMap(oldest.id(), oldest.order(), updateSeq); @@ -470,18 +475,28 @@ else if (localNode(p, aff)) exchId + ", fullMap=" + fullMapString() + ']'); } else if (!node2part.valid()) { - node2part = new GridDhtPartitionFullMap(oldest.id(), oldest.order(), updateSeq, node2part, false); - - if (log.isDebugEnabled()) - log.debug("Created new full topology map on oldest node [exchId=" + exchId + ", fullMap=" + - node2part + ']'); + node2part = new GridDhtPartitionFullMap(oldest.id(), + oldest.order(), + updateSeq, + node2part, + false); + + if (log.isDebugEnabled()) { + log.debug("Created new full topology map on oldest node [exchId=" + exchId + + ", fullMap=" + node2part + ']'); + } } else if (!node2part.nodeId().equals(loc.id())) { - node2part = new GridDhtPartitionFullMap(oldest.id(), oldest.order(), updateSeq, node2part, false); - - if (log.isDebugEnabled()) - log.debug("Copied old map into new map on oldest node (previous oldest node left) [exchId=" + - exchId + ", fullMap=" + fullMapString() + ']'); + node2part = new GridDhtPartitionFullMap(oldest.id(), + oldest.order(), + updateSeq, + node2part, + false); + + if (log.isDebugEnabled()) { + log.debug("Copied old map into new map on oldest node (previous oldest node left) [" + + "exchId=" + exchId + ", fullMap=" + fullMapString() + ']'); + } } } @@ -499,9 +514,10 @@ else if (!node2part.nodeId().equals(loc.id())) { consistencyCheck(); - if (log.isDebugEnabled()) - log.debug("Partition map after beforeExchange [exchId=" + exchId + ", fullMap=" + - fullMapString() + ']'); + if (log.isDebugEnabled()) { + log.debug("Partition map after beforeExchange [exchId=" + exchId + + ", fullMap=" + fullMapString() + ']'); + } } finally { lock.writeLock().unlock(); From 2a5390b1c083819f059e449b34a5979ec35b7755 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Fri, 7 Jul 2017 15:01:43 +0300 Subject: [PATCH 096/155] IGNITE-5159: DDL example. This closes #2227. --- .../datagrid/CacheQueryDdlExample.java | 119 ++++++++++++++++++ .../examples/CacheExamplesSelfTest.java | 8 ++ 2 files changed, 127 insertions(+) create mode 100644 examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java diff --git a/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java b/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java new file mode 100644 index 0000000000000..84a67cd0c18ea --- /dev/null +++ b/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.examples.datagrid; + +import java.util.List; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.examples.ExampleNodeStartup; + +/** + * Example to showcase DDL capabilities of Ignite's SQL engine. + *

+ * Remote nodes could be started from command line as follows: + * {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}. + *

+ * Alternatively you can run {@link ExampleNodeStartup} in either same or another JVM. + */ +public class CacheQueryDdlExample { + /** Dummy cache name. */ + private static final String DUMMY_CACHE_NAME = "dummy_cache"; + + /** + * Executes example. + * + * @param args Command line arguments, none required. + * @throws Exception If example execution failed. + */ + @SuppressWarnings({"unused", "ThrowFromFinallyBlock"}) + public static void main(String[] args) throws Exception { + try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) { + print("Cache query DDL example started."); + + // Create dummy cache to act as an entry point for SQL queries (new SQL API which do not require this + // will appear in future versions, JDBC and ODBC drivers do not require it already). + CacheConfiguration cacheCfg = new CacheConfiguration<>(DUMMY_CACHE_NAME) + .setSqlSchema("PUBLIC").setIndexedTypes(Integer.class, Integer.class); + + try ( + IgniteCache cache = ignite.getOrCreateCache(cacheCfg) + ) { + // Create table based on PARTITIONED template with one backup. + cache.query(new SqlFieldsQuery( + "CREATE TABLE person (id LONG PRIMARY KEY, name VARCHAR, city_id LONG) " + + "WITH \"backups=1\"")).getAll(); + + // Create reference City table based on REPLICATED template. + cache.query(new SqlFieldsQuery( + "CREATE TABLE city (id LONG PRIMARY KEY, name VARCHAR) WITH \"template=replicated\"")).getAll(); + + // Create an index. + cache.query(new SqlFieldsQuery("CREATE INDEX on Person (city_id)")).getAll(); + + print("Created database objects."); + + SqlFieldsQuery qry = new SqlFieldsQuery("INSERT INTO person (id, name, city_id) values (?, ?, ?)"); + + cache.query(qry.setArgs(1L, "John Doe", 3L)).getAll(); + cache.query(qry.setArgs(2L, "Jane Roe", 2L)).getAll(); + cache.query(qry.setArgs(3L, "Mary Major", 1L)).getAll(); + cache.query(qry.setArgs(4L, "Richard Miles", 2L)).getAll(); + + qry = new SqlFieldsQuery("INSERT INTO city (id, name) VALUES (?, ?)"); + + cache.query(qry.setArgs(1L, "Forest Hill")).getAll(); + cache.query(qry.setArgs(2L, "Denver")).getAll(); + cache.query(qry.setArgs(3L, "St. Petersburg")).getAll(); + + print("Populated data."); + + List> res = cache.query(new SqlFieldsQuery( + "SELECT p.name, c.name FROM Person p INNER JOIN City c on c.id = p.city_id")).getAll(); + + print("Query results:"); + + for (Object next : res) + System.out.println(">>> " + next); + + cache.query(new SqlFieldsQuery("drop table Person")).getAll(); + cache.query(new SqlFieldsQuery("drop table City")).getAll(); + + print("Dropped database objects."); + } + finally { + // Distributed cache can be removed from cluster only by #destroyCache() call. + ignite.destroyCache(DUMMY_CACHE_NAME); + } + + print("Cache query DDL example finished."); + } + } + + /** + * Prints message. + * + * @param msg Message to print before all objects are printed. + */ + private static void print(String msg) { + System.out.println(); + System.out.println(">>> " + msg); + } +} diff --git a/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java index 3447dffb41c96..f65d97cacfabe 100644 --- a/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java @@ -23,6 +23,7 @@ import org.apache.ignite.examples.datagrid.CacheContinuousQueryExample; import org.apache.ignite.examples.datagrid.CacheDataStreamerExample; import org.apache.ignite.examples.datagrid.CachePutGetExample; +import org.apache.ignite.examples.datagrid.CacheQueryDdlExample; import org.apache.ignite.examples.datagrid.CacheQueryDmlExample; import org.apache.ignite.examples.datagrid.CacheQueryExample; import org.apache.ignite.examples.datagrid.CacheTransactionExample; @@ -135,6 +136,13 @@ public void testCacheQueryDmlExample() throws Exception { CacheQueryDmlExample.main(EMPTY_ARGS); } + /** + * @throws Exception If failed. + */ + public void testCacheQUeryDdlExample() throws Exception { + CacheQueryDdlExample.main(EMPTY_ARGS); + } + /** * @throws Exception If failed. */ From a9387adef490086406b05fe961ff0f3151045caa Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Fri, 7 Jul 2017 15:21:20 +0300 Subject: [PATCH 097/155] IGNITE-5717 .NET: Reduce MemoryPolicyConfiguration.MaxSize for persistence tests as a workaround for OOM on default settings --- .../Cache/PersistentStoreTest.cs | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs index 96ae47c18dae6..e9cbce8924a1a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs @@ -18,6 +18,7 @@ namespace Apache.Ignite.Core.Tests.Cache { using System.IO; + using Apache.Ignite.Core.Cache.Configuration; using Apache.Ignite.Core.Common; using Apache.Ignite.Core.Impl; using Apache.Ignite.Core.PersistentStore; @@ -51,7 +52,7 @@ public void TearDown() [Test] public void TestCacheDataSurvivesNodeRestart() { - var cfg = new IgniteConfiguration(TestUtils.GetTestConfiguration()) + var cfg = new IgniteConfiguration(GetTestConfiguration()) { PersistentStoreConfiguration = new PersistentStoreConfiguration { @@ -106,7 +107,7 @@ public void TestCacheDataSurvivesNodeRestart() [Test] public void TestGridActivationWithPersistence() { - var cfg = new IgniteConfiguration(TestUtils.GetTestConfiguration()) + var cfg = new IgniteConfiguration(GetTestConfiguration()) { PersistentStoreConfiguration = new PersistentStoreConfiguration() }; @@ -177,5 +178,36 @@ private static void CheckIsActive(IIgnite ignite, bool isActive) ex.Message.Substring(0, 62)); } } + + ///

+ /// Gets the test configuration. + /// + private static IgniteConfiguration GetTestConfiguration() + { + return new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + MemoryConfiguration = GetMemoryConfig() + }; + } + + /// + /// Gets the memory configuration with reduced MaxMemory to work around persistence bug. + /// + private static MemoryConfiguration GetMemoryConfig() + { + // TODO: Remove this method and use default config (IGNITE-5717). + return new MemoryConfiguration + { + MemoryPolicies = new[] + { + new MemoryPolicyConfiguration + { + Name = MemoryConfiguration.DefaultDefaultMemoryPolicyName, + InitialSize = 512*1024*1024, + MaxSize = 512*1024*1024 + } + } + }; + } } } From 13399d530f3458c3bf59634857cb05f801e8cc34 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 7 Jul 2017 15:24:09 +0300 Subject: [PATCH 098/155] 2.1 Fixed assert (cacheName can be null here). --- .../processors/datastructures/DataStructuresProcessor.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java index 4399fe2f4c95a..5564b796c7db4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java @@ -326,12 +326,10 @@ void onRemoved(GridCacheInternalKey key, GridCacheRemovable obj) { * @return {@code True} if cache with such name is used to store data structures. */ public static boolean isDataStructureCache(String cacheName) { - assert cacheName != null; - - return cacheName.startsWith(ATOMICS_CACHE_NAME) || + return cacheName != null && (cacheName.startsWith(ATOMICS_CACHE_NAME) || cacheName.startsWith(DS_CACHE_NAME_PREFIX) || cacheName.equals(DEFAULT_DS_GROUP_NAME) || - cacheName.equals(DEFAULT_VOLATILE_DS_GROUP_NAME); + cacheName.equals(DEFAULT_VOLATILE_DS_GROUP_NAME)); } /** From 0d6fb1ad73e8ed448dabe7c0cc631222835b52c4 Mon Sep 17 00:00:00 2001 From: Sergey Kalashnikov Date: Fri, 7 Jul 2017 15:54:52 +0300 Subject: [PATCH 099/155] IGNITE-5204: Fixed NPE on certain data with index inlining. This closes #2115. --- .../processors/query/h2/database/InlineIndexHelper.java | 2 +- .../processors/query/h2/database/InlineIndexHelperTest.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java index 1fd45f3ccf030..19cf857fece29 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java @@ -441,7 +441,7 @@ public int put(long pageAddr, int off, Value val, int maxSize) { size = (short)s.length; else { s = trimUTF8(s, maxSize - 3); - size = (short)(s.length | 0x8000); + size = (short)(s == null ? 0 : s.length | 0x8000); } if (s == null) { diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java index 25790234f0a65..a2a3a7204cc6b 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java @@ -167,6 +167,10 @@ public void testStringTruncate() throws Exception { assertTrue(ih.isValueFull(pageAddr, off)); assertEquals("aaa", ih.get(pageAddr, off, 3 + 5).getString()); + + ih.put(pageAddr, off, ValueString.get("\u20acaaa"), 3 + 2); + + assertNull(ih.get(pageAddr, off, 3 + 2)); } finally { if (page != 0L) From a1cac1b7cc1c3afdcc12afb08bf1ecdcb6f28828 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 7 Jul 2017 15:55:18 +0300 Subject: [PATCH 100/155] Removed unused method GridDhtPartitionTopology.checkEvictions(). --- .../dht/GridClientPartitionTopology.java | 5 --- .../dht/GridDhtPartitionTopology.java | 7 ++-- .../dht/GridDhtPartitionTopologyImpl.java | 36 ------------------- 3 files changed, 2 insertions(+), 46 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java index 9e9f4fbc7361d..e7519610f360c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java @@ -828,11 +828,6 @@ private boolean isStaleUpdate(GridDhtPartitionMap currentMap, GridDhtPartitionMa return Collections.emptyList(); } - /** {@inheritDoc} */ - @Override public void checkEvictions() { - // No-op. - } - /** * Updates value for single partition. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java index bf4e844edfe0b..5f76d12a16fae 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java @@ -279,11 +279,6 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, */ public Collection lostPartitions(); - /** - * - */ - public void checkEvictions(); - /** * @param skipZeros If {@code true} then filters out zero counters. * @return Partition update counters. @@ -324,6 +319,7 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, /** * Make nodes from provided set owners for a given partition. * State of all current owners that aren't contained in the set will be reset to MOVING. + * * @param p Partition ID. * @param updateSeq If should increment sequence when updated. * @param owners Set of new owners. @@ -333,6 +329,7 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId, /** * Callback on exchange done. + * * @param assignment New affinity assignment. */ public void onExchangeDone(AffinityAssignment assignment); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index f49dccfe0706d..cf0dd5f3b4cd8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -1743,42 +1743,6 @@ else if (plc != PartitionLossPolicy.IGNORE) { return result; } - /** - * @param updateSeq Update sequence. - * @return {@code True} if state changed. - */ - private boolean checkEvictions(long updateSeq) { - AffinityTopologyVersion affVer = grp.affinity().lastVersion(); - - boolean changed = false; - - if (!affVer.equals(AffinityTopologyVersion.NONE) && affVer.compareTo(topVer) >= 0) { - List> aff = grp.affinity().assignments(topVer); - - changed = checkEvictions(updateSeq, aff); - - updateRebalanceVersion(aff); - } - - return changed; - } - - /** {@inheritDoc} */ - @Override public void checkEvictions() { - lock.writeLock().lock(); - - try { - long updateSeq = this.updateSeq.incrementAndGet(); - - node2part.newUpdateSequence(updateSeq); - - checkEvictions(updateSeq); - } - finally { - lock.writeLock().unlock(); - } - } - /** * @param updateSeq Update sequence. * @param aff Affinity assignments. From 99713feea764fc8c3e5b247a24698a2c04d3bcf4 Mon Sep 17 00:00:00 2001 From: Sergey Kalashnikov Date: Fri, 7 Jul 2017 16:30:37 +0300 Subject: [PATCH 101/155] IGNITE-5204: SQL: fixed incorrect partition calculation in case of unicast optimization, when WHERE clause argument type was different from key or affinity key type. This close #2107. --- .../cache/query/CacheQueryPartitionInfo.java | 46 +++- .../query/h2/DmlStatementsProcessor.java | 9 +- .../internal/processors/query/h2/H2Utils.java | 27 +++ .../processors/query/h2/IgniteH2Indexing.java | 30 ++- .../query/h2/sql/GridSqlQuerySplitter.java | 5 +- .../query/IgniteSqlRoutingTest.java | 211 ++++++++++++++++-- 6 files changed, 290 insertions(+), 38 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQueryPartitionInfo.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQueryPartitionInfo.java index 1329d5c397f07..42bf0707ca512 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQueryPartitionInfo.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQueryPartitionInfo.java @@ -39,22 +39,35 @@ */ public class CacheQueryPartitionInfo { /** */ - private int partId; + private final int partId; /** */ - private String cacheName; + private final String cacheName; /** */ - private int paramIdx; + private final String tableName; + + /** */ + private final int dataType; + + /** */ + private final int paramIdx; /** * @param partId Partition id, or -1 if parameter binding required. * @param cacheName Cache name required for partition calculation. + * @param tableName Table name required for proper type conversion. + * @param dataType Required data type id for the query parameter. * @param paramIdx Query parameter index required for partition calculation. */ - public CacheQueryPartitionInfo(int partId, String cacheName, int paramIdx) { + public CacheQueryPartitionInfo(int partId, String cacheName, String tableName, int dataType, int paramIdx) { + // In case partition is not known, both cacheName and tableName must be provided. + assert (partId >= 0) ^ ((cacheName != null) && (tableName != null)); + this.partId = partId; this.cacheName = cacheName; + this.tableName = tableName; + this.dataType = dataType; this.paramIdx = paramIdx; } @@ -72,6 +85,20 @@ public String cacheName() { return cacheName; } + /** + * @return Table name. + */ + public String tableName() { + return tableName; + } + + /** + * @return Required data type for the query parameter. + */ + public int dataType() { + return dataType; + } + /** * @return Query parameter index required for partition calculation. */ @@ -81,7 +108,9 @@ public int paramIdx() { /** {@inheritDoc} */ @Override public int hashCode() { - return partId ^ paramIdx ^ (cacheName == null ? 0 : cacheName.hashCode()); + return partId ^ dataType ^ paramIdx ^ + (cacheName == null ? 0 : cacheName.hashCode()) ^ + (tableName == null ? 0 : tableName.hashCode()); } /** {@inheritDoc} */ @@ -97,10 +126,13 @@ public int paramIdx() { if (partId >= 0) return partId == other.partId; - if (other.cacheName == null) + if (other.cacheName == null || other.tableName == null) return false; - return other.cacheName.equals(cacheName) && other.paramIdx == paramIdx; + return other.cacheName.equals(cacheName) && + other.tableName.equals(tableName) && + other.dataType == dataType && + other.paramIdx == paramIdx; } /** {@inheritDoc} */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 0c1dbf983d7ce..4f7c288f86c22 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -748,14 +748,7 @@ private static Object convert(Object val, GridH2RowDescriptor desc, Class exp return newArr; } - int objType = DataType.getTypeFromClass(val.getClass()); - - if (objType == type) - return val; - - Value h2Val = desc.wrap(val, objType); - - return h2Val.convertTo(type).getObject(); + return H2Utils.convert(val, desc, type); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java index ee88acf6a9951..157e1ba7ac8ae 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Utils.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata; import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase; @@ -29,6 +30,8 @@ import org.h2.jdbc.JdbcConnection; import org.h2.result.SortOrder; import org.h2.table.IndexColumn; +import org.h2.value.DataType; +import org.h2.value.Value; import java.lang.reflect.Constructor; import java.sql.Connection; @@ -235,6 +238,30 @@ public static void setupConnection(Connection conn, boolean distributedJoins, bo s.setJoinBatchEnabled(distributedJoins); } + /** + * Convert value to column's expected type by means of H2. + * + * @param val Source value. + * @param desc Row descriptor. + * @param type Expected column type to convert to. + * @return Converted object. + * @throws IgniteCheckedException if failed. + */ + public static Object convert(Object val, GridH2RowDescriptor desc, int type) + throws IgniteCheckedException { + if (val == null) + return null; + + int objType = DataType.getTypeFromClass(val.getClass()); + + if (objType == type) + return val; + + Value h2Val = desc.wrap(val, objType); + + return h2Val.convertTo(type).getObject(); + } + /** * Private constructor. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 255c5f1e10cd9..40eae1765792c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -2265,7 +2265,10 @@ public void awaitForReadyTopologyVersion(AffinityTopologyVersion topVer) throws /** * Bind query parameters and calculate partitions derived from the query. * + * @param partInfoList Collection of query derived partition info. + * @param params Query parameters. * @return Partitions. + * @throws IgniteCheckedException, If fails. */ private int[] calculateQueryPartitions(CacheQueryPartitionInfo[] partInfoList, Object[] params) throws IgniteCheckedException { @@ -2273,9 +2276,8 @@ private int[] calculateQueryPartitions(CacheQueryPartitionInfo[] partInfoList, O ArrayList list = new ArrayList<>(partInfoList.length); for (CacheQueryPartitionInfo partInfo: partInfoList) { - int partId = partInfo.partition() < 0 ? - kernalContext().affinity().partition(partInfo.cacheName(), params[partInfo.paramIdx()]) : - partInfo.partition(); + int partId = (partInfo.partition() >= 0) ? partInfo.partition() : + bindPartitionInfoParameter(partInfo, params); int i = 0; @@ -2298,6 +2300,28 @@ private int[] calculateQueryPartitions(CacheQueryPartitionInfo[] partInfoList, O return result; } + /** + * Bind query parameter to partition info and calculate partition. + * + * @param partInfo Partition Info. + * @param params Query parameters. + * @return Partition. + * @throws IgniteCheckedException, If fails. + */ + private int bindPartitionInfoParameter(CacheQueryPartitionInfo partInfo, Object[] params) + throws IgniteCheckedException { + assert partInfo != null; + assert partInfo.partition() < 0; + + GridH2RowDescriptor desc = dataTable(partInfo.cacheName(), + partInfo.tableName()).rowDescriptor(); + + Object param = H2Utils.convert(params[partInfo.paramIdx()], + desc, partInfo.dataType()); + + return kernalContext().affinity().partition(partInfo.cacheName(), param); + } + /** {@inheritDoc} */ @Override public Collection runningQueries(long duration) { Collection res = new ArrayList<>(); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index aebf59650badd..1578f9fb78dc0 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -2318,14 +2318,15 @@ private static CacheQueryPartitionInfo extractPartitionFromEquality(GridSqlOpera GridSqlConst constant = (GridSqlConst)right; return new CacheQueryPartitionInfo(ctx.affinity().partition(tbl.cacheName(), - constant.value().getObject()), null, -1); + constant.value().getObject()), null, null, -1, -1); } assert right instanceof GridSqlParameter; GridSqlParameter param = (GridSqlParameter) right; - return new CacheQueryPartitionInfo(-1, tbl.cacheName(), param.index()); + return new CacheQueryPartitionInfo(-1, tbl.cacheName(), tbl.getName(), + column.column().getType(), param.index()); } /** diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlRoutingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlRoutingTest.java index fddd3f4b78c21..323eb7a6eb50a 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlRoutingTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlRoutingTest.java @@ -33,19 +33,26 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import java.nio.ByteBuffer; +import java.sql.Timestamp; +import java.text.DateFormat; import java.text.MessageFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; +/** Tests for query partitions derivation. */ public class IgniteSqlRoutingTest extends GridCommonAbstractTest { /** IP finder. */ private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); @@ -62,10 +69,10 @@ public class IgniteSqlRoutingTest extends GridCommonAbstractTest { /** */ private static int NODE_COUNT = 4; - /** broadcast query to ensure events came from all nodes */ + /** Broadcast query to ensure events came from all nodes. */ private static String FINAL_QRY = "select count(1) from {0} where name=?"; - /** Param to distinguish the final query event */ + /** Param to distinguish the final query event. */ private static String FINAL_QRY_PARAM = "Abracadabra"; /** {@inheritDoc} */ @@ -108,15 +115,19 @@ public class IgniteSqlRoutingTest extends GridCommonAbstractTest { startGrid(NODE_CLIENT); + awaitPartitionMapExchange(); + fillCaches(); } + /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { super.afterTestsStopped(); stopAllGrids(); } + /** */ private CacheConfiguration buildCacheConfiguration(String name) { if (name.equals(CACHE_PERSON)) { CacheConfiguration ccfg = new CacheConfiguration(CACHE_PERSON); @@ -176,7 +187,7 @@ public void testUnicastQuerySelectAffinityKeyEqualsConstant() throws Exception { IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_CALL); List> result = runQueryEnsureUnicast(cache, - new SqlFieldsQuery("select id, name, duration from Call where personId=100 order by id"), 1); + new SqlFieldsQuery("select id, name, duration from Call where personId=100 order by id"), 1); assertEquals(2, result.size()); @@ -189,7 +200,8 @@ public void testUnicastQuerySelectAffinityKeyEqualsParameter() throws Exception IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_CALL); List> result = runQueryEnsureUnicast(cache, - new SqlFieldsQuery("select id, name, duration from Call where personId=? order by id").setArgs(100), 1); + new SqlFieldsQuery("select id, name, duration from Call where personId=? order by id") + .setArgs(100), 1); assertEquals(2, result.size()); @@ -203,7 +215,7 @@ public void testUnicastQuerySelectKeyEqualsParameterReused() throws Exception { for (int key : new int[] {0, 250, 500, 750, 1000} ) { List> result = runQueryEnsureUnicast(cache, - new SqlFieldsQuery("select name, age from Person where _key=?").setArgs(key), 1); + new SqlFieldsQuery("select name, age from Person where _key=?").setArgs(key), 1); assertEquals(1, result.size()); @@ -220,8 +232,8 @@ public void testUnicastQuerySelectKeyEqualsParameter() throws Exception { CallKey callKey = new CallKey(5, 1); List> result = runQueryEnsureUnicast(cache, - new SqlFieldsQuery("select name, duration from Call where _key=?") - .setArgs(callKey), 1); + new SqlFieldsQuery("select name, duration from Call where _key=?") + .setArgs(callKey), 1); assertEquals(1, result.size()); @@ -230,7 +242,7 @@ public void testUnicastQuerySelectKeyEqualsParameter() throws Exception { checkResultsRow(result, 0, call.name, call.duration); } - /** Check group, having, ordering allowed to be unicast requests */ + /** Check group, having, ordering allowed to be unicast requests. */ public void testUnicastQueryGroups() throws Exception { IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_CALL); @@ -258,8 +270,8 @@ public void testUnicastQuerySelectKeyEqualAndFieldParameter() throws Exception { CallKey callKey = new CallKey(5, 1); List> result = runQueryEnsureUnicast(cache, - new SqlFieldsQuery("select name, duration from Call where _key=? and duration=?") - .setArgs(callKey, 100), 1); + new SqlFieldsQuery("select name, duration from Call where _key=? and duration=?") + .setArgs(callKey, 100), 1); assertEquals(1, result.size()); @@ -276,8 +288,8 @@ public void testUnicastQuerySelect2KeyEqualsAndFieldParameter() throws Exception CallKey callKey2 = new CallKey(1000, 1); List> result = runQueryEnsureUnicast(cache, - new SqlFieldsQuery("select name, duration from Call where (_key=? and duration=?) or (_key=?)") - .setArgs(callKey1, 100, callKey2), 2); + new SqlFieldsQuery("select name, duration from Call where (_key=? and duration=?) or (_key=?)") + .setArgs(callKey1, 100, callKey2), 2); assertEquals(2, result.size()); @@ -290,6 +302,68 @@ public void testUnicastQuerySelect2KeyEqualsAndFieldParameter() throws Exception checkResultsRow(result, 1, call.name, call.duration); } + /** */ + public void testUnicastQueryKeyTypeConversionParameter() throws Exception { + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); + + // Pass string argument to expression with integer + List> result = runQueryEnsureUnicast(cache, + new SqlFieldsQuery("select name, age from Person where _key = ?") + .setArgs("5"), 1); + + Person person = cache.get(5); + + assertEquals(1, result.size()); + + assertEquals(person.name, result.get(0).get(0)); + assertEquals(person.age, result.get(0).get(1)); + } + + /** */ + public void testUnicastQueryKeyTypeConversionConstant() throws Exception { + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); + + // Use string within expression against integer key + List> result = runQueryEnsureUnicast(cache, + new SqlFieldsQuery("select name, age from Person where _key = '5'"), 1); + + Person person = cache.get(5); + + assertEquals(1, result.size()); + + assertEquals(person.name, result.get(0).get(0)); + assertEquals(person.age, result.get(0).get(1)); + } + + /** */ + public void testUnicastQueryAffinityKeyTypeConversionParameter() throws Exception { + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_CALL); + + // Pass string argument to expression with integer + List> result = runQueryEnsureUnicast(cache, + new SqlFieldsQuery("select id, name, duration from Call where personId=? order by id") + .setArgs("100"), 1); + + assertEquals(2, result.size()); + + checkResultsRow(result, 0, 1, "caller1", 100); + checkResultsRow(result, 1, 2, "caller2", 200); + } + + /** */ + public void testUnicastQueryAffinityKeyTypeConversionConstant() throws Exception { + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_CALL); + + // Use string within expression against integer key + List> result = runQueryEnsureUnicast(cache, + new SqlFieldsQuery("select id, name, duration from Call where personId='100' order by id"), 1); + + assertEquals(2, result.size()); + + checkResultsRow(result, 0, 1, "caller1", 100); + checkResultsRow(result, 1, 2, "caller2", 200); + } + /** */ public void testBroadcastQuerySelectKeyEqualsOrFieldParameter() throws Exception { IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_CALL); @@ -297,12 +371,112 @@ public void testBroadcastQuerySelectKeyEqualsOrFieldParameter() throws Exception CallKey callKey = new CallKey(5, 1); List> result = runQueryEnsureBroadcast(cache, - new SqlFieldsQuery("select name, duration from Call where _key=? or duration=?") - .setArgs(callKey, 100)); + new SqlFieldsQuery("select name, duration from Call where _key=? or duration=?") + .setArgs(callKey, 100)); assertEquals(cache.size() / 2, result.size()); } + /** */ + public void testUuidKeyAsByteArrayParameter() throws Exception { + String cacheName = "uuidCache"; + + CacheConfiguration ccfg = new CacheConfiguration<>(cacheName); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + ccfg.setIndexedTypes(UUID.class, UUID.class); + + IgniteCache cache = grid(NODE_CLIENT).createCache(ccfg); + + try { + int count = 10; + + UUID values[] = new UUID[count]; + + for (int i = 0; i < count; i++) { + UUID val = UUID.randomUUID(); + + cache.put(val, val); + + values[i] = val; + } + + for (UUID val : values) { + byte[] arr = convertUuidToByteArray(val); + + List> result = cache.query(new SqlFieldsQuery( + "select _val from UUID where _key = ?").setArgs(arr)).getAll(); + + assertEquals(1, result.size()); + assertEquals(val, result.get(0).get(0)); + } + } + finally { + cache.destroy(); + } + } + + /** */ + public void testDateKeyAsTimestampParameter() throws Exception { + String cacheName = "dateCache"; + + CacheConfiguration ccfg = new CacheConfiguration<>(cacheName); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + ccfg.setIndexedTypes(Date.class, Date.class); + + IgniteCache cache = grid(NODE_CLIENT).createCache(ccfg); + + try { + int count = 30; + + Date values[] = new Date[count]; + + DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy"); + + for (int i = 0; i < count; i++) { + Date val = dateFormat.parse(String.format("%02d/06/2017", i + 1)); + + cache.put(val, val); + + values[i] = val; + } + + for (Date val : values) { + Timestamp ts = new Timestamp(val.getTime()); + + List> result = cache.query(new SqlFieldsQuery( + "select _val from Date where _key = ?").setArgs(ts)).getAll(); + + assertEquals(1, result.size()); + assertEquals(val, result.get(0).get(0)); + } + } + finally { + cache.destroy(); + } + } + + /** + * Convert UUID to byte[]. + * + * @param val UUID to convert. + * @return Result. + */ + private byte[] convertUuidToByteArray(UUID val) { + assert val != null; + + ByteBuffer bb = ByteBuffer.wrap(new byte[16]); + + bb.putLong(val.getMostSignificantBits()); + + bb.putLong(val.getLeastSignificantBits()); + + return bb.array(); + } + /** */ private void fillCaches() { IgniteCache callCache = grid(NODE_CLIENT).cache(CACHE_CALL); @@ -335,15 +509,15 @@ private void checkResultsRow(List> results, int rowId, Object ... expect assertEquals(expected[col], row.get(col)); } - /** Run query and check that only one node did generate 'query executed' event for it */ + /** Run query and check that only one node did generate 'query executed' event for it. */ private List> runQueryEnsureUnicast(IgniteCache cache, SqlFieldsQuery qry, int nodeCnt) throws Exception { try (EventCounter evtCounter = new EventCounter(nodeCnt)) { List> result = cache.query(qry).getAll(); // do broadcast 'marker' query to ensure that we received all events from previous qry cache.query(new SqlFieldsQuery( - MessageFormat.format(FINAL_QRY, cache.getName())) - .setArgs(FINAL_QRY_PARAM)).getAll(); + MessageFormat.format(FINAL_QRY, cache.getName())) + .setArgs(FINAL_QRY_PARAM)).getAll(); // wait for all events from 'marker' query evtCounter.await(); @@ -353,6 +527,7 @@ private List> runQueryEnsureUnicast(IgniteCache cache, SqlFieldsQue } } + /** */ private List> runQueryEnsureBroadcast(IgniteCache cache, SqlFieldsQuery qry) throws Exception { final CountDownLatch execLatch = new CountDownLatch(NODE_COUNT); @@ -492,7 +667,7 @@ private static class CallKey { private int id; /** */ - public CallKey(int personId, int id) { + private CallKey(int personId, int id) { this.personId = personId; this.id = id; } From 3c887378eb64d2d236073410070082e5699e8334 Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Fri, 7 Jul 2017 16:52:31 +0300 Subject: [PATCH 102/155] IGNITE-5582: Implemented Compute::Broadcast for C++ (cherry picked from commit fa974286e8f066a8d6aa57519edf5ec7761be095) --- .../cpp/core-test/src/compute_test.cpp | 91 +++++- .../platforms/cpp/core/include/Makefile.am | 2 + .../cpp/core/include/ignite/compute/compute.h | 66 +++++ .../ignite/impl/compute/compute_impl.h | 161 +++++++---- .../ignite/impl/compute/compute_job_result.h | 54 +++- .../ignite/impl/compute/compute_task_holder.h | 204 +------------- .../multiple_job_compute_task_holder.h | 265 ++++++++++++++++++ .../compute/single_job_compute_task_holder.h | 212 ++++++++++++++ .../cpp/core/project/vs/core.vcxproj | 2 + .../cpp/core/project/vs/core.vcxproj.filters | 6 + 10 files changed, 811 insertions(+), 252 deletions(-) create mode 100644 modules/platforms/cpp/core/include/ignite/impl/compute/multiple_job_compute_task_holder.h create mode 100644 modules/platforms/cpp/core/include/ignite/impl/compute/single_job_compute_task_holder.h diff --git a/modules/platforms/cpp/core-test/src/compute_test.cpp b/modules/platforms/cpp/core-test/src/compute_test.cpp index 8c57ef17404f9..1fd76709c1f44 100644 --- a/modules/platforms/cpp/core-test/src/compute_test.cpp +++ b/modules/platforms/cpp/core-test/src/compute_test.cpp @@ -476,7 +476,7 @@ BOOST_AUTO_TEST_CASE(IgniteRunAsyncLocalError) BOOST_CHECK_EXCEPTION(res.GetValue(), IgniteError, IsTestError); } -BOOST_AUTO_TEST_CASE(IgniteRunTestRemote) +BOOST_AUTO_TEST_CASE(IgniteRunRemote) { Ignite node2 = MakeNode("ComputeNode2"); Compute compute = node.GetCompute(); @@ -489,7 +489,7 @@ BOOST_AUTO_TEST_CASE(IgniteRunTestRemote) BOOST_CHECK_EQUAL(Func3::res, "42.24"); } -BOOST_AUTO_TEST_CASE(IgniteRunTestRemoteError) +BOOST_AUTO_TEST_CASE(IgniteRunRemoteError) { Ignite node2 = MakeNode("ComputeNode2"); Compute compute = node.GetCompute(); @@ -509,5 +509,92 @@ BOOST_AUTO_TEST_CASE(IgniteRunTestRemoteError) BOOST_CHECK_EXCEPTION(res.GetValue(), IgniteError, IsTestError); } +BOOST_AUTO_TEST_CASE(IgniteBroadcastLocalSync) +{ + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Broadcasting");; + std::vector res = compute.Broadcast(Func2(8, 5)); + + BOOST_CHECK_EQUAL(res.size(), 1); + BOOST_CHECK_EQUAL(res[0], "8.5"); +} + +BOOST_AUTO_TEST_CASE(IgniteBroadcastLocalAsync) +{ + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Broadcasting");; + Future< std::vector > res = compute.BroadcastAsync(Func2(312, 245)); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECKPOINT("Waiting with timeout"); + res.WaitFor(100); + + BOOST_CHECK(!res.IsReady()); + + std::vector value = res.GetValue(); + + BOOST_CHECK_EQUAL(value.size(), 1); + BOOST_CHECK_EQUAL(value[0], "312.245"); +} + +BOOST_AUTO_TEST_CASE(IgniteBroadcastSyncLocalError) +{ + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Broadcasting"); + + BOOST_CHECK_EXCEPTION(compute.Broadcast(Func2(MakeTestError())), IgniteError, IsTestError); +} + +BOOST_AUTO_TEST_CASE(IgniteBroadcastAsyncLocalError) +{ + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Broadcasting"); + Future res = compute.BroadcastAsync(Func2(MakeTestError())); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECKPOINT("Waiting with timeout"); + res.WaitFor(100); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECK_EXCEPTION(res.GetValue(), IgniteError, IsTestError); +} + +BOOST_AUTO_TEST_CASE(IgniteBroadcastRemote) +{ + Ignite node2 = MakeNode("ComputeNode2"); + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Broadcasting"); + std::vector res = compute.Broadcast(Func2(8, 5)); + + BOOST_CHECK_EQUAL(res.size(), 2); + BOOST_CHECK_EQUAL(res[0], "8.5"); + BOOST_CHECK_EQUAL(res[1], "8.5"); +} + +BOOST_AUTO_TEST_CASE(IgniteBroadcastRemoteError) +{ + Ignite node2 = MakeNode("ComputeNode2"); + Compute compute = node.GetCompute(); + + BOOST_CHECKPOINT("Broadcasting"); + Future< std::vector > res = compute.BroadcastAsync(Func2(MakeTestError())); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECKPOINT("Waiting with timeout"); + res.WaitFor(100); + + BOOST_CHECK(!res.IsReady()); + + BOOST_CHECK_EXCEPTION(res.GetValue(), IgniteError, IsTestError); +} BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/modules/platforms/cpp/core/include/Makefile.am b/modules/platforms/cpp/core/include/Makefile.am index 50772cbe6d532..1e9369f8fbaf0 100644 --- a/modules/platforms/cpp/core/include/Makefile.am +++ b/modules/platforms/cpp/core/include/Makefile.am @@ -61,6 +61,8 @@ nobase_include_HEADERS = \ ignite/impl/compute/compute_job_holder.h \ ignite/impl/compute/compute_job_result.h \ ignite/impl/compute/compute_task_holder.h \ + ignite/impl/compute/single_job_compute_task_holder.h \ + ignite/impl/compute/multiple_job_compute_task_holder.h \ ignite/impl/handle_registry.h \ ignite/impl/ignite_binding_impl.h \ ignite/impl/ignite_environment.h \ diff --git a/modules/platforms/cpp/core/include/ignite/compute/compute.h b/modules/platforms/cpp/core/include/ignite/compute/compute.h index 75c8c85145e1e..9b4c9b9652ec6 100644 --- a/modules/platforms/cpp/core/include/ignite/compute/compute.h +++ b/modules/platforms/cpp/core/include/ignite/compute/compute.h @@ -157,6 +157,72 @@ namespace ignite return impl.Get()->RunAsync(action); } + /** + * Broadcasts provided ComputeFunc to all nodes in the cluster group. + * + * @tparam R Function return type. BinaryType should be specialized + * for the type if it is not primitive. + * @tparam F Compute function type. Should implement ComputeFunc + * class. + * @param func Compute function to call. + * @return Vector containing computation results. + * @throw IgniteError in case of error. + */ + template + std::vector Broadcast(const F& func) + { + return impl.Get()->BroadcastAsync(func).GetValue(); + } + + /** + * Broadcasts provided ComputeFunc to all nodes in the cluster group. + * + * @tparam F Compute function type. Should implement ComputeFunc + * class. + * @param func Compute function to call. + * @throw IgniteError in case of error. + */ + template + void Broadcast(const F& func) + { + impl.Get()->BroadcastAsync(func).GetValue(); + } + + /** + * Asyncronuously broadcasts provided ComputeFunc to all nodes in the + * cluster group. + * + * @tparam R Function return type. BinaryType should be specialized + * for the type if it is not primitive. + * @tparam F Compute function type. Should implement ComputeFunc + * class. + * @param func Compute function to call. + * @return Future that can be used to access computation results once + * they are ready. + * @throw IgniteError in case of error. + */ + template + Future< std::vector > BroadcastAsync(const F& func) + { + return impl.Get()->BroadcastAsync(func); + } + + /** + * Asyncronuously broadcasts provided ComputeFunc to all nodes in the + * cluster group. + * + * @tparam F Compute function type. Should implement ComputeFunc + * class. + * @param func Compute function to call. + * @return Future that can be used to wait for action to complete. + * @throw IgniteError in case of error. + */ + template + Future BroadcastAsync(const F& func) + { + return impl.Get()->BroadcastAsync(func); + } + private: /** Implementation. */ common::concurrent::SharedPointer impl; diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h index 63f9a46199955..4ba1c1cb9b61c 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_impl.h @@ -26,11 +26,10 @@ #include #include #include -#include +#include +#include #include -#include - namespace ignite { namespace impl @@ -50,7 +49,9 @@ namespace ignite { enum Type { - Unicast = 5 + BROADCAST = 2, + + UNICAST = 5, }; }; @@ -66,41 +67,113 @@ namespace ignite * Asyncronuously calls provided ComputeFunc on a node within * the underlying cluster group. * - * @tparam F Compute function type. Should implement ComputeFunc - * class. - * @tparam R Call return type. BinaryType should be specialized for - * the type if it is not primitive. Should not be void. For + * @tparam F Compute function type. Should implement + * ComputeFunc class. + * @tparam R Call return type. BinaryType should be specialized + * for the type if it is not primitive. Should not be void. For * non-returning methods see Compute::Run(). * @param func Compute function to call. - * @return Future that can be used to acess computation result once - * it's ready. - * @throw IgniteError in case of error. + * @return Future that can be used to acess computation result + * once it's ready. */ template Future CallAsync(const F& func) { - common::concurrent::SharedPointer mem = GetEnvironment().AllocateMemory(); - interop::InteropOutputStream out(mem.Get()); - binary::BinaryWriterImpl writer(&out, GetEnvironment().GetTypeManager()); + typedef ComputeJobHolderImpl JobType; + typedef SingleJobComputeTaskHolder TaskType; + + return PerformTask(Operation::UNICAST, func); + } + + /** + * Asyncronuously runs provided ComputeFunc on a node within + * the underlying cluster group. + * + * @tparam F Compute action type. Should implement + * ComputeFunc class. + * @param action Compute action to call. + * @return Future that can be used to wait for action + * to complete. + */ + template + Future RunAsync(const F& action) + { + typedef ComputeJobHolderImpl JobType; + typedef SingleJobComputeTaskHolder TaskType; + + return PerformTask(Operation::UNICAST, action); + } + + /** + * Asyncronuously broadcasts provided ComputeFunc to all nodes + * in the underlying cluster group. + * + * @tparam F Compute function type. Should implement + * ComputeFunc class. + * @tparam R Call return type. BinaryType should be specialized + * for the type if it is not primitive. Should not be void. For + * non-returning methods see Compute::Run(). + * @param func Compute function to call. + * @return Future that can be used to acess computation result + * once it's ready. + */ + template + Future< std::vector > BroadcastAsync(const F& func) + { + typedef ComputeJobHolderImpl JobType; + typedef MultipleJobComputeTaskHolder TaskType; + + return PerformTask, F, JobType, TaskType>(Operation::BROADCAST, func); + } + + /** + * Asyncronuously broadcasts provided ComputeFunc to all nodes + * in the underlying cluster group. + * + * @tparam F Compute function type. Should implement + * ComputeFunc class. + * @param func Compute function to call. + * @return Future that can be used to acess computation result + * once it's ready. + */ + template + Future BroadcastAsync(const F& func) + { + typedef ComputeJobHolderImpl JobType; + typedef MultipleJobComputeTaskHolder TaskType; + + return PerformTask(Operation::BROADCAST, func); + } - common::concurrent::SharedPointer job(new ComputeJobHolderImpl(func)); + private: + /** + * Perform job. + * + * @tparam F Compute function type. Should implement + * ComputeFunc class. + * @tparam R Call return type. BinaryType should be specialized + * for the type if it is not primitive. + * @tparam J Job type. + * @tparam T Task type. + * + * @param operation Operation type. + * @param func Function. + * @return Future that can be used to acess computation result + * once it's ready. + */ + template + Future PerformTask(Operation::Type operation, const F& func) + { + common::concurrent::SharedPointer job(new J(func)); int64_t jobHandle = GetEnvironment().GetHandleRegistry().Allocate(job); - ComputeTaskHolderImpl* taskPtr = new ComputeTaskHolderImpl(jobHandle); + T* taskPtr = new T(jobHandle); common::concurrent::SharedPointer task(taskPtr); int64_t taskHandle = GetEnvironment().GetHandleRegistry().Allocate(task); - writer.WriteInt64(taskHandle); - writer.WriteInt32(1); - writer.WriteInt64(jobHandle); - writer.WriteObject(func); - - out.Synchronize(); - - jobject target = InStreamOutObject(Operation::Unicast, *mem.Get()); - std::auto_ptr cancelable(new CancelableImpl(GetEnvironmentPointer(), target)); + std::auto_ptr cancelable = PerformTask(operation, jobHandle, taskHandle, func); common::Promise& promise = taskPtr->GetPromise(); promise.SetCancelTarget(cancelable); @@ -109,48 +182,38 @@ namespace ignite } /** - * Asyncronuously runs provided ComputeFunc on a node within - * the underlying cluster group. + * Perform job. * - * @tparam F Compute action type. Should implement ComputeAction - * class. - * @param action Compute action to call. - * @return Future that can be used to wait for action to complete. - * @throw IgniteError in case of error. + * @tparam F Compute function type. Should implement + * ComputeFunc class. + * + * @param operation Operation type. + * @param jobHandle Job Handle. + * @param taskHandle Task Handle. + * @param func Function. + * @return Cancelable auto pointer. */ template - Future RunAsync(const F& action) + std::auto_ptr PerformTask(Operation::Type operation, int64_t jobHandle, + int64_t taskHandle, const F& func) { common::concurrent::SharedPointer mem = GetEnvironment().AllocateMemory(); interop::InteropOutputStream out(mem.Get()); binary::BinaryWriterImpl writer(&out, GetEnvironment().GetTypeManager()); - common::concurrent::SharedPointer job(new ComputeJobHolderImpl(action)); - - int64_t jobHandle = GetEnvironment().GetHandleRegistry().Allocate(job); - - ComputeTaskHolderImpl* taskPtr = new ComputeTaskHolderImpl(jobHandle); - common::concurrent::SharedPointer task(taskPtr); - - int64_t taskHandle = GetEnvironment().GetHandleRegistry().Allocate(task); - writer.WriteInt64(taskHandle); writer.WriteInt32(1); writer.WriteInt64(jobHandle); - writer.WriteObject(action); + writer.WriteObject(func); out.Synchronize(); - jobject target = InStreamOutObject(Operation::Unicast, *mem.Get()); + jobject target = InStreamOutObject(operation, *mem.Get()); std::auto_ptr cancelable(new CancelableImpl(GetEnvironmentPointer(), target)); - common::Promise& promise = taskPtr->GetPromise(); - promise.SetCancelTarget(cancelable); - - return promise.GetFuture(); + return cancelable; } - private: IGNITE_NO_COPY_ASSIGNMENT(ComputeImpl); }; } diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_result.h b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_result.h index 087452259270c..9d3dfead8a5ee 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_result.h +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_job_result.h @@ -36,6 +36,28 @@ namespace ignite { namespace compute { + struct ComputeJobResultPolicy + { + enum Type + { + /** + * Wait for results if any are still expected. If all results have been received - + * it will start reducing results. + */ + WAIT = 0, + + /** + * Ignore all not yet received results and start reducing results. + */ + REDUCE = 1, + + /** + * Fail-over job to execute on another node. + */ + FAILOVER = 2 + }; + }; + /** * Used to hold compute job result. */ @@ -64,16 +86,36 @@ namespace ignite res = val; } + /** + * Get result value. + * + * @return Result. + */ + const ResultType& GetResult() const + { + return res; + } + /** * Set error. * * @param error Error to set. */ - void SetError(const IgniteError error) + void SetError(const IgniteError& error) { err = error; } + /** + * Get error. + * + * @return Error. + */ + const IgniteError& GetError() const + { + return err; + } + /** * Set promise to a state which corresponds to result. * @@ -191,6 +233,16 @@ namespace ignite err = error; } + /** + * Get error. + * + * @return Error. + */ + const IgniteError& GetError() const + { + return err; + } + /** * Set promise to a state which corresponds to result. * diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_task_holder.h b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_task_holder.h index f627f2792b334..66276d16152af 100644 --- a/modules/platforms/cpp/core/include/ignite/impl/compute/compute_task_holder.h +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/compute_task_holder.h @@ -17,17 +17,14 @@ /** * @file - * Declares ignite::impl::compute::ComputeTaskHolder class and - * ignite::impl::compute::ComputeTaskHolderImpl class template. + * Declares ignite::impl::compute::ComputeTaskHolder. */ -#ifndef _IGNITE_IMPL_COMPUTE_COMPUTE_TASK_IMPL -#define _IGNITE_IMPL_COMPUTE_COMPUTE_TASK_IMPL +#ifndef _IGNITE_IMPL_COMPUTE_COMPUTE_TASK_HOLDER +#define _IGNITE_IMPL_COMPUTE_COMPUTE_TASK_HOLDER #include -#include -#include #include namespace ignite @@ -36,28 +33,6 @@ namespace ignite { namespace compute { - struct ComputeJobResultPolicy - { - enum Type - { - /** - * Wait for results if any are still expected. If all results have been received - - * it will start reducing results. - */ - WAIT = 0, - - /** - * Ignore all not yet received results and start reducing results. - */ - REDUCE = 1, - - /** - * Fail-over job to execute on another node. - */ - FAILOVER = 2 - }; - }; - /** * Compute task holder. Internal helper class. * Used to handle tasks in general way, without specific types. @@ -120,179 +95,8 @@ namespace ignite /** Related job handle. */ int64_t handle; }; - - /** - * Compute task holder type-specific implementation. - */ - template - class ComputeTaskHolderImpl : public ComputeTaskHolder - { - public: - typedef F JobType; - typedef R ResultType; - - /** - * Constructor. - * - * @param handle Job handle. - */ - ComputeTaskHolderImpl(int64_t handle) : - ComputeTaskHolder(handle) - { - // No-op. - } - - /** - * Destructor. - */ - virtual ~ComputeTaskHolderImpl() - { - // No-op. - } - - /** - * Process local job result. - * - * @param job Job. - * @return Policy. - */ - virtual int32_t JobResultLocal(ComputeJobHolder& job) - { - typedef ComputeJobHolderImpl ActualComputeJobHolder; - - ActualComputeJobHolder& job0 = static_cast(job); - - res = job0.GetResult(); - - return ComputeJobResultPolicy::WAIT; - } - - /** - * Process remote job result. - * - * @param job Job. - * @param reader Reader for stream with result. - * @return Policy. - */ - virtual int32_t JobResultRemote(ComputeJobHolder& job, binary::BinaryReaderImpl& reader) - { - res.Read(reader); - - return ComputeJobResultPolicy::WAIT; - } - - /** - * Reduce results of related jobs. - */ - virtual void Reduce() - { - res.SetPromise(promise); - } - - /** - * Get result promise. - * - * @return Reference to result promise. - */ - common::Promise& GetPromise() - { - return promise; - } - - private: - /** Result. */ - ComputeJobResult res; - - /** Task result promise. */ - common::Promise promise; - }; - - /** - * Compute task holder type-specific implementation. - */ - template - class ComputeTaskHolderImpl : public ComputeTaskHolder - { - public: - typedef F JobType; - - /** - * Constructor. - * - * @param handle Job handle. - */ - ComputeTaskHolderImpl(int64_t handle) : - ComputeTaskHolder(handle) - { - // No-op. - } - - /** - * Destructor. - */ - virtual ~ComputeTaskHolderImpl() - { - // No-op. - } - - /** - * Process local job result. - * - * @param job Job. - * @return Policy. - */ - virtual int32_t JobResultLocal(ComputeJobHolder& job) - { - typedef ComputeJobHolderImpl ActualComputeJobHolder; - - ActualComputeJobHolder& job0 = static_cast(job); - - res = job0.GetResult(); - - return ComputeJobResultPolicy::WAIT; - } - - /** - * Process remote job result. - * - * @param job Job. - * @param reader Reader for stream with result. - * @return Policy. - */ - virtual int32_t JobResultRemote(ComputeJobHolder& job, binary::BinaryReaderImpl& reader) - { - res.Read(reader); - - return ComputeJobResultPolicy::WAIT; - } - - /** - * Reduce results of related jobs. - */ - virtual void Reduce() - { - res.SetPromise(promise); - } - - /** - * Get result promise. - * - * @return Reference to result promise. - */ - common::Promise& GetPromise() - { - return promise; - } - - private: - /** Result. */ - ComputeJobResult res; - - /** Task result promise. */ - common::Promise promise; - }; } } } -#endif //_IGNITE_IMPL_COMPUTE_COMPUTE_TASK_IMPL +#endif //_IGNITE_IMPL_COMPUTE_COMPUTE_TASK_HOLDER diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/multiple_job_compute_task_holder.h b/modules/platforms/cpp/core/include/ignite/impl/compute/multiple_job_compute_task_holder.h new file mode 100644 index 0000000000000..9fb13f1af4776 --- /dev/null +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/multiple_job_compute_task_holder.h @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file + * Declares ignite::impl::compute::MultipleJobComputeTaskHolder class template. + */ + +#ifndef _IGNITE_IMPL_COMPUTE_MULTIPLE_JOB_COMPUTE_TASK +#define _IGNITE_IMPL_COMPUTE_MULTIPLE_JOB_COMPUTE_TASK + +#include +#include + +#include +#include +#include + +namespace ignite +{ + namespace impl + { + namespace compute + { + /** + * Multiple Job Compute task holder type-specific implementation. + * Used for broadcast. + * + * @tparam F Function type. + * @tparam R Function result type. + */ + template + class MultipleJobComputeTaskHolder : public ComputeTaskHolder + { + public: + typedef F JobType; + typedef R ResultType; + + /** + * Constructor. + * + * @param handle Job handle. + */ + MultipleJobComputeTaskHolder(int64_t handle) : + ComputeTaskHolder(handle), + result(new std::vector()), + error(), + promise() + { + // No-op. + } + + /** + * Destructor. + */ + virtual ~MultipleJobComputeTaskHolder() + { + // No-op. + } + + /** + * Process local job result. + * + * @param job Job. + * @return Policy. + */ + virtual int32_t JobResultLocal(ComputeJobHolder& job) + { + typedef ComputeJobHolderImpl ActualComputeJobHolder; + + ActualComputeJobHolder& job0 = static_cast(job); + + ProcessResult(job0.GetResult()); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Process remote job result. + * + * @param job Job. + * @param reader Reader for stream with result. + * @return Policy. + */ + virtual int32_t JobResultRemote(ComputeJobHolder& job, binary::BinaryReaderImpl& reader) + { + ComputeJobResult res; + + res.Read(reader); + + ProcessResult(res); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Reduce results of related jobs. + */ + virtual void Reduce() + { + if (error.GetCode() == IgniteError::IGNITE_SUCCESS) + promise.SetValue(result); + else + promise.SetError(error); + } + + /** + * Get result promise. + * + * @return Reference to result promise. + */ + common::Promise< std::vector >& GetPromise() + { + return promise; + } + + private: + /** + * Process result. + * + * @param res Result. + */ + void ProcessResult(const ComputeJobResult& res) + { + const IgniteError& err = res.GetError(); + + if (err.GetCode() == IgniteError::IGNITE_SUCCESS) + result->push_back(res.GetResult()); + else + error = err; + } + + /** Result. */ + std::auto_ptr< std::vector > result; + + /** Error. */ + IgniteError error; + + /** Task result promise. */ + common::Promise< std::vector > promise; + }; + + /** + * Compute task holder type-specific implementation. + */ + template + class MultipleJobComputeTaskHolder : public ComputeTaskHolder + { + public: + typedef F JobType; + + /** + * Constructor. + * + * @param handle Job handle. + */ + MultipleJobComputeTaskHolder(int64_t handle) : + ComputeTaskHolder(handle) + { + // No-op. + } + + /** + * Destructor. + */ + virtual ~MultipleJobComputeTaskHolder() + { + // No-op. + } + + /** + * Process local job result. + * + * @param job Job. + * @return Policy. + */ + virtual int32_t JobResultLocal(ComputeJobHolder& job) + { + typedef ComputeJobHolderImpl ActualComputeJobHolder; + + ActualComputeJobHolder& job0 = static_cast(job); + + ProcessResult(job0.GetResult()); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Process remote job result. + * + * @param job Job. + * @param reader Reader for stream with result. + * @return Policy. + */ + virtual int32_t JobResultRemote(ComputeJobHolder& job, binary::BinaryReaderImpl& reader) + { + ComputeJobResult res; + + res.Read(reader); + + ProcessResult(res); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Reduce results of related jobs. + */ + virtual void Reduce() + { + if (error.GetCode() == IgniteError::IGNITE_SUCCESS) + promise.SetValue(); + else + promise.SetError(error); + } + + /** + * Get result promise. + * + * @return Reference to result promise. + */ + common::Promise& GetPromise() + { + return promise; + } + + private: + /** + * Process result. + * + * @param res Result. + */ + void ProcessResult(const ComputeJobResult& res) + { + const IgniteError& err = res.GetError(); + + if (err.GetCode() != IgniteError::IGNITE_SUCCESS) + error = err; + } + + /** Error. */ + IgniteError error; + + /** Task result promise. */ + common::Promise promise; + }; + } + } +} + +#endif //_IGNITE_IMPL_COMPUTE_MULTIPLE_JOB_COMPUTE_TASK diff --git a/modules/platforms/cpp/core/include/ignite/impl/compute/single_job_compute_task_holder.h b/modules/platforms/cpp/core/include/ignite/impl/compute/single_job_compute_task_holder.h new file mode 100644 index 0000000000000..9b0506a46d98e --- /dev/null +++ b/modules/platforms/cpp/core/include/ignite/impl/compute/single_job_compute_task_holder.h @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file + * Declares ignite::impl::compute::SingleJobComputeTaskHolder class template. + */ + +#ifndef _IGNITE_IMPL_COMPUTE_SINGLE_JOB_COMPUTE_TASK +#define _IGNITE_IMPL_COMPUTE_SINGLE_JOB_COMPUTE_TASK + +#include + +#include +#include +#include + +namespace ignite +{ + namespace impl + { + namespace compute + { + /** + * Compute task holder type-specific implementation. + */ + template + class SingleJobComputeTaskHolder : public ComputeTaskHolder + { + public: + typedef F JobType; + typedef R ResultType; + + /** + * Constructor. + * + * @param handle Job handle. + */ + SingleJobComputeTaskHolder(int64_t handle) : + ComputeTaskHolder(handle) + { + // No-op. + } + + /** + * Destructor. + */ + virtual ~SingleJobComputeTaskHolder() + { + // No-op. + } + + /** + * Process local job result. + * + * @param job Job. + * @return Policy. + */ + virtual int32_t JobResultLocal(ComputeJobHolder& job) + { + typedef ComputeJobHolderImpl ActualComputeJobHolder; + + ActualComputeJobHolder& job0 = static_cast(job); + + res = job0.GetResult(); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Process remote job result. + * + * @param job Job. + * @param reader Reader for stream with result. + * @return Policy. + */ + virtual int32_t JobResultRemote(ComputeJobHolder& job, binary::BinaryReaderImpl& reader) + { + res.Read(reader); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Reduce results of related jobs. + */ + virtual void Reduce() + { + res.SetPromise(promise); + } + + /** + * Get result promise. + * + * @return Reference to result promise. + */ + common::Promise& GetPromise() + { + return promise; + } + + private: + /** Result. */ + ComputeJobResult res; + + /** Task result promise. */ + common::Promise promise; + }; + + /** + * Compute task holder type-specific implementation. + */ + template + class SingleJobComputeTaskHolder : public ComputeTaskHolder + { + public: + typedef F JobType; + + /** + * Constructor. + * + * @param handle Job handle. + */ + SingleJobComputeTaskHolder(int64_t handle) : + ComputeTaskHolder(handle) + { + // No-op. + } + + /** + * Destructor. + */ + virtual ~SingleJobComputeTaskHolder() + { + // No-op. + } + + /** + * Process local job result. + * + * @param job Job. + * @return Policy. + */ + virtual int32_t JobResultLocal(ComputeJobHolder& job) + { + typedef ComputeJobHolderImpl ActualComputeJobHolder; + + ActualComputeJobHolder& job0 = static_cast(job); + + res = job0.GetResult(); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Process remote job result. + * + * @param job Job. + * @param reader Reader for stream with result. + * @return Policy. + */ + virtual int32_t JobResultRemote(ComputeJobHolder& job, binary::BinaryReaderImpl& reader) + { + res.Read(reader); + + return ComputeJobResultPolicy::WAIT; + } + + /** + * Reduce results of related jobs. + */ + virtual void Reduce() + { + res.SetPromise(promise); + } + + /** + * Get result promise. + * + * @return Reference to result promise. + */ + common::Promise& GetPromise() + { + return promise; + } + + private: + /** Result. */ + ComputeJobResult res; + + /** Task result promise. */ + common::Promise promise; + }; + } + } +} + +#endif //_IGNITE_IMPL_COMPUTE_SINGLE_JOB_COMPUTE_TASK diff --git a/modules/platforms/cpp/core/project/vs/core.vcxproj b/modules/platforms/cpp/core/project/vs/core.vcxproj index 9911ffeee4d0b..3c3489c806004 100644 --- a/modules/platforms/cpp/core/project/vs/core.vcxproj +++ b/modules/platforms/cpp/core/project/vs/core.vcxproj @@ -232,6 +232,8 @@ + + diff --git a/modules/platforms/cpp/core/project/vs/core.vcxproj.filters b/modules/platforms/cpp/core/project/vs/core.vcxproj.filters index 7b84494ce5a09..27f3944785736 100644 --- a/modules/platforms/cpp/core/project/vs/core.vcxproj.filters +++ b/modules/platforms/cpp/core/project/vs/core.vcxproj.filters @@ -237,6 +237,12 @@ Code\impl\compute + + Code\impl\compute + + + Code\impl\compute + From 3314a4513c816c027dad80258ded1fe5a4c2b700 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 7 Jul 2017 17:28:47 +0300 Subject: [PATCH 103/155] Fixed cache plugin validation. --- .../processors/cache/ClusterCachesInfo.java | 30 ++++++++++++++++++- .../cache/DynamicCacheDescriptor.java | 13 -------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java index 738e4ac64e46f..949bc1989887f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java @@ -37,6 +37,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; +import org.apache.ignite.internal.GridCachePluginContext; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; @@ -53,6 +54,9 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.plugin.CachePluginContext; +import org.apache.ignite.plugin.CachePluginProvider; +import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.spi.discovery.DiscoveryDataBag; import org.jetbrains.annotations.Nullable; @@ -175,8 +179,32 @@ void onKernalStart(boolean checkConsistency) throws IgniteCheckedException { locCacheInfo.cacheData().config().getName()); } - if (checkConsistency) + if (checkConsistency) { checkCache(locCacheInfo, cacheData, cacheData.receivedFrom()); + + ClusterNode rmt = ctx.discovery().node(cacheData.receivedFrom()); + + if (rmt == null) { + for (ClusterNode node : ctx.discovery().localJoin().discoCache().serverNodes()) { + if (!node.isLocal() && ctx.discovery().cacheAffinityNode(node, locCfg.getName())) { + rmt = node; + + break; + } + } + } + + if (rmt != null) { + for (PluginProvider p : ctx.plugins().allProviders()) { + CachePluginContext pluginCtx = new GridCachePluginContext(ctx, locCfg); + + CachePluginProvider provider = p.createCacheProvider(pluginCtx); + + if (provider != null) + provider.validateRemote(locCfg, cacheData.cacheConfiguration(), rmt); + } + } + } } if (checkConsistency) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java index 315013de7e103..18abcd8947f12 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java @@ -24,7 +24,6 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; -import org.apache.ignite.internal.processors.plugin.CachePluginManager; import org.apache.ignite.internal.processors.query.QuerySchema; import org.apache.ignite.internal.processors.query.schema.message.SchemaFinishDiscoveryMessage; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -57,9 +56,6 @@ public class DynamicCacheDescriptor { /** Template configuration flag. */ private boolean template; - /** Cache plugin manager. */ - private final CachePluginManager pluginMgr; - /** */ private boolean updatesAllowed = true; @@ -138,8 +134,6 @@ public DynamicCacheDescriptor(GridKernalContext ctx, this.sql = sql; this.deploymentId = deploymentId; - pluginMgr = new CachePluginManager(ctx, cacheCfg); - cacheId = CU.cacheId(cacheCfg.getName()); synchronized (schemaMux) { @@ -241,13 +235,6 @@ public CacheObjectContext cacheObjectContext(IgniteCacheObjectProcessor proc) th return objCtx; } - /** - * @return Cache plugin manager. - */ - public CachePluginManager pluginManager() { - return pluginMgr; - } - /** * @return Updates allowed flag. */ From 517a23d252078e62a8addca55ca78497e5226b5b Mon Sep 17 00:00:00 2001 From: mcherkasov Date: Fri, 30 Jun 2017 20:23:55 +0300 Subject: [PATCH 104/155] IGNITE-5554 ServiceProcessor may process failed reassignments in timeout thread --- .../service/GridServiceProcessor.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java index d67f2d16d45c5..23a29f816f205 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; @@ -1509,18 +1510,11 @@ private void onDeployment(final GridServiceDeployment dep, final AffinityTopolog } @Override public void onTimeout() { - GridSpinBusyLock busyLock = GridServiceProcessor.this.busyLock; - - if (busyLock == null || !busyLock.enterBusy()) - return; - - try { - // Try again. - onDeployment(dep, topVer); - } - finally { - busyLock.leaveBusy(); - } + depExe.execute(new DepRunnable() { + @Override public void run0() { + onDeployment(dep, topVer); + } + }); } }); } @@ -1716,7 +1710,11 @@ private void onReassignmentFailed(final AffinityTopologyVersion topVer, } @Override public void onTimeout() { - onReassignmentFailed(topVer, retries); + depExe.execute(new Runnable() { + public void run() { + onReassignmentFailed(topVer, retries); + } + }); } }); } From 993f7fbe1d49a524e2dee626aef72e16fd5d3cda Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Fri, 7 Jul 2017 18:55:27 +0300 Subject: [PATCH 105/155] IGNITE-5701 - Some nodes have partitionUpdateCounter equal to 0 after rebalancing --- .../apache/ignite/IgniteSystemProperties.java | 2 +- .../distributed/dht/GridDhtCacheEntry.java | 6 -- .../dht/GridDhtLocalPartition.java | 45 ----------- .../dht/GridDhtPartitionTopologyImpl.java | 16 +++- .../preloader/GridDhtPartitionDemander.java | 5 +- .../preloader/GridDhtPartitionSupplier.java | 2 +- .../GridDhtPartitionSupplyMessage.java | 20 ++--- .../GridCacheDatabaseSharedManager.java | 2 +- ...IgnitePdsCacheRebalancingAbstractTest.java | 74 +++++++++++++++++++ .../wal/IgniteWalHistoryReservationsTest.java | 29 +++++++- 10 files changed, 131 insertions(+), 70 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 085712a5ded44..35b0577c45f3d 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -626,7 +626,7 @@ public final class IgniteSystemProperties { /** * WAL rebalance threshold. */ - public static final String IGNITE_PDS_WAL_REBALANCE_THRESHOLD = "IGNITE_PDS_WAL_REBALANCE_THRESHOLD"; + public static final String IGNITE_PDS_WAL_REBALANCE_THRESHOLD = "IGNITE_PDS_WAL_REBALANCE_THRESHOLD"; /** Ignite page memory concurrency level. */ public static final String IGNITE_OFFHEAP_LOCK_CONCURRENCY_LEVEL = "IGNITE_OFFHEAP_LOCK_CONCURRENCY_LEVEL"; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java index 57dd622951a55..77cc642ee7ebe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java @@ -122,12 +122,6 @@ public GridDhtCacheEntry( return locPart; } - /** {@inheritDoc} */ - @Override protected void onUpdateFinished(long cntr) { - if (cctx.shared().database().persistenceEnabled()) - locPart.onUpdateReceived(cntr); - } - /** {@inheritDoc} */ @Override public boolean isDht() { return true; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java index 8e42351a88c00..725822dd58cdf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java @@ -157,13 +157,6 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements @GridToStringExclude private final CacheDataStore store; - /** Partition updates. */ - @GridToStringExclude - private final ConcurrentNavigableMap updates = new ConcurrentSkipListMap<>(); - - /** Last applied update. */ - private final AtomicLong lastApplied = new AtomicLong(0); - /** Set if failed to move partition to RENTING state due to reservations, to be checked when * reservation is released. */ private volatile boolean shouldBeRenting; @@ -348,44 +341,6 @@ public boolean isEmpty() { return store.fullSize() == 0 && internalSize() == 0; } - /** - * @return Last applied update. - */ - public long lastAppliedUpdate() { - return lastApplied.get(); - } - - /** - * @param cntr Received counter. - */ - public void onUpdateReceived(long cntr) { - boolean changed = updates.putIfAbsent(cntr, true) == null; - - if (!changed) - return; - - while (true) { - Map.Entry entry = updates.firstEntry(); - - if (entry == null) - return; - - long first = entry.getKey(); - - long cntr0 = lastApplied.get(); - - if (first <= cntr0) - updates.remove(first); - else if (first == cntr0 + 1) - if (lastApplied.compareAndSet(cntr0, first)) - updates.remove(first); - else - break; - else - break; - } - } - /** * @return If partition is moving or owning or renting. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index cf0dd5f3b4cd8..2f548108f5a7e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -673,6 +673,11 @@ private GridDhtLocalPartition createPartition(int p) { if (loc == null || loc.state() == EVICTED) { locParts.set(p, loc = new GridDhtLocalPartition(ctx, grp, p)); + T2 cntr = cntrMap.get(p); + + if (cntr != null) + loc.updateCounter(cntr.get2()); + if (ctx.pageStore() != null) { try { ctx.pageStore().onPartitionCreated(grp.groupId(), p); @@ -1334,11 +1339,12 @@ else if (locPart.state() == OWNING || locPart.state() == MOVING) { if (cntr != null && cntr.get2() > part.updateCounter()) part.updateCounter(cntr.get2()); + else if (part.updateCounter() > 0) + this.cntrMap.put(part.id(), new T2<>(part.initialUpdateCounter(), part.updateCounter())); } } finally { lock.writeLock().unlock(); - } } @@ -1715,6 +1721,10 @@ else if (plc != PartitionLossPolicy.IGNORE) { result.add(ctx.localNodeId()); } + U.warn(log, "Partition has been scheduled for rebalancing due to outdated update counter " + + "[nodeId=" + ctx.localNodeId() + "cacheOrGroupName=" + grp.cacheOrGroupName() + + ", partId=" + locPart.id() + ", haveHistory=" + haveHistory + "]"); + } } @@ -1731,6 +1741,10 @@ else if (plc != PartitionLossPolicy.IGNORE) { result.add(e.getKey()); } } + + U.warn(log, "Partition has been scheduled for rebalancing due to outdated update counter " + + "[nodeId=" + ctx.localNodeId() + "cacheOrGroupName=" + grp.cacheOrGroupName() + + ", partId=" + locPart.id() + ", haveHistory=" + haveHistory + "]"); } if (updateSeq) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java index e7e95b2537e58..4f34aba265a3a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java @@ -640,7 +640,7 @@ public void handleSupplyMessage( assert part != null; - boolean last = supply.last().contains(p); + boolean last = supply.last().containsKey(p); if (part.state() == MOVING) { boolean reserved = part.reserve(); @@ -680,6 +680,9 @@ public void handleSupplyMessage( // If message was last for this partition, // then we take ownership. if (last) { + if (supply.isClean(p)) + part.updateCounter(supply.last().get(p)); + top.own(part); fut.partitionDone(id, p); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java index 1cc6c28cb8223..3ead9826c298f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java @@ -414,7 +414,7 @@ public void handleDemandMessage(int idx, UUID id, GridDhtPartitionDemandMessage } // Mark as last supply message. - s.last(part); + s.last(part, loc.updateCounter()); phase = SupplyContextPhase.NEW; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java index ef14a90c0f85a..90d11f589a6dc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java @@ -56,8 +56,8 @@ public class GridDhtPartitionSupplyMessage extends GridCacheGroupIdMessage imple private AffinityTopologyVersion topVer; /** Partitions that have been fully sent. */ - @GridDirectCollection(int.class) - private Collection last; + @GridDirectMap(keyType = int.class, valueType = long.class) + private Map last; /** Partitions which were not found. */ @GridToStringInclude @@ -128,19 +128,19 @@ long updateSequence() { /** * @return Flag to indicate last message for partition. */ - Collection last() { - return last == null ? Collections.emptySet() : last; + Map last() { + return last == null ? Collections.emptyMap() : last; } /** * @param p Partition which was fully sent. */ - void last(int p) { + void last(int p, long cntr) { if (last == null) - last = new HashSet<>(); + last = new HashMap<>(); - if (last.add(p)) { - msgSize += 4; + if (last.put(p, cntr) == null) { + msgSize += 12; // If partition is empty, we need to add it. if (!infos().containsKey(p)) { @@ -304,7 +304,7 @@ public int size() { writer.incrementState(); case 7: - if (!writer.writeCollection("last", last, MessageCollectionItemType.INT)) + if (!writer.writeMap("last", last, MessageCollectionItemType.INT, MessageCollectionItemType.LONG)) return false; writer.incrementState(); @@ -382,7 +382,7 @@ public int size() { reader.incrementState(); case 7: - last = reader.readCollection("last", MessageCollectionItemType.INT); + last = reader.readMap("last", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false); if (!reader.isLastRead()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 4af4daf3f0b99..d64677e44305b 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -2126,7 +2126,7 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws CacheState state = new CacheState(locParts.size()); for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions()) - state.addPartitionState(part.id(), part.dataStore().fullSize(), part.lastAppliedUpdate()); + state.addPartitionState(part.id(), part.dataStore().fullSize(), part.updateCounter()); cpRec.addCacheGroupState(grp.groupId(), state); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java index cbc2623d5db71..588b3ac52a4bf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java @@ -19,6 +19,7 @@ import java.io.Serializable; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.Callable; @@ -29,6 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.PartitionLossPolicy; import org.apache.ignite.cache.QueryEntity; @@ -41,11 +43,13 @@ import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; @@ -488,6 +492,76 @@ else if (nodesCnt.get() > maxNodesCount) assertEquals(Integer.toString(entry.getKey()), entry.getValue(), cache.get(entry.getKey())); } + /** + * @throws Exception If failed + */ + public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception { + final Ignite ig = startGrids(4); + + ig.active(true); + + int k = 0; + + try (IgniteDataStreamer ds = ig.dataStreamer(cacheName)) { + ds.allowOverwrite(true); + + for (int k0 = k; k < k0 + 10_000; k++) + ds.addData(k, k); + } + + for (int t = 0; t < 10; t++) { + IgniteInternalFuture fut = GridTestUtils.runAsync(new Runnable() { + @Override public void run() { + try { + stopGrid(3); + + IgniteEx ig0 = startGrid(3); + + awaitPartitionMapExchange(); + + ig0.cache(cacheName).rebalance().get(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + + try (IgniteDataStreamer ds = ig.dataStreamer(cacheName)) { + ds.allowOverwrite(true); + + while (!fut.isDone()) { + ds.addData(k, k); + + k++; + + U.sleep(1); + } + } + + fut.get(); + + Map cntrs = new HashMap<>(); + + for (int g = 0; g < 4; g++) { + IgniteEx ig0 = grid(g); + + for (GridDhtLocalPartition part : ig0.cachex(cacheName).context().topology().currentLocalPartitions()) { + if (cntrs.containsKey(part.id())) + assertEquals(String.valueOf(part.id()), (long) cntrs.get(part.id()), part.updateCounter()); + else + cntrs.put(part.id(), part.updateCounter()); + } + + for (int k0 = 0; k0 < k; k0++) { + assertEquals(String.valueOf(k0), k0, ig0.cache(cacheName).get(k0)); + } + } + + assertEquals(ig.affinity(cacheName).partitions(), cntrs.size()); + } + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java index 48d8c21c48417..4bea63fc0447a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java @@ -31,6 +31,7 @@ import org.apache.ignite.configuration.MemoryPolicyConfiguration; import org.apache.ignite.configuration.PersistentStoreConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.apache.ignite.internal.util.lang.GridAbsPredicate; @@ -201,7 +202,7 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception { int entryCnt = 10_000; - Ignite ig0 = startGrids(2); + IgniteEx ig0 = (IgniteEx) startGrids(2); ig0.active(true); @@ -219,7 +220,7 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception { forceCheckpoint(); - Ignite ig1 = startGrid(1); + IgniteEx ig1 = startGrid(1); IgniteCache cache1 = ig1.cache("cache1"); @@ -236,6 +237,16 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception { assertEquals("k=" + k, k, cache1.get(k)); } } + + cache.rebalance().get(); + + for (int p = 0; p < ig1.affinity("cache1").partitions(); p++) { + GridDhtLocalPartition p0 = ig0.context().cache().cache("cache1").context().topology().localPartition(p); + GridDhtLocalPartition p1 = ig1.context().cache().cache("cache1").context().topology().localPartition(p); + + assertTrue(p0.updateCounter() > 0); + assertEquals(p0.updateCounter(), p1.updateCounter()); + } } /** @@ -244,7 +255,7 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception { public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception { int entryCnt = 10_000; - Ignite ig0 = startGrids(2); + IgniteEx ig0 = (IgniteEx) startGrids(2); ig0.active(true); @@ -269,7 +280,7 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception { assertEquals("k=" + k, k, cache.get(k)); } - Ignite ig1 = startGrid(1); + IgniteEx ig1 = startGrid(1); IgniteCache cache1 = ig1.cache("cache1"); @@ -286,6 +297,16 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception { assertEquals("k=" + k, k, cache1.get(k)); } } + + cache.rebalance().get(); + + for (int p = 0; p < ig1.affinity("cache1").partitions(); p++) { + GridDhtLocalPartition p0 = ig0.context().cache().cache("cache1").context().topology().localPartition(p); + GridDhtLocalPartition p1 = ig1.context().cache().cache("cache1").context().topology().localPartition(p); + + assertTrue(p0.updateCounter() > 0); + assertEquals(p0.updateCounter(), p1.updateCounter()); + } } /** From e5c2ec5a62c0d5fc268275ea9277f6933a582966 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 7 Jul 2017 23:06:52 +0700 Subject: [PATCH 106/155] ignite-2.1 Minor. --- .../internal/visor/cache/VisorCacheMetrics.java | 12 ++---------- .../visor/node/VisorMemoryPolicyConfiguration.java | 11 ++--------- .../main/resources/META-INF/classnames.properties | 11 +++++++++-- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java index d99e0c9bdf655..5d8bc8151cfbf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java @@ -638,11 +638,6 @@ public long getRebalancingBytesRate() { return rebalancingBytesRate; } - /** {@inheritDoc} */ - @Override public byte getProtocolVersion() { - return V2; - } - /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeString(out, name); @@ -690,6 +685,7 @@ public long getRebalancingBytesRate() { out.writeLong(heapEntriesCnt); out.writeLong(offHeapAllocatedSize); out.writeLong(offHeapEntriesCnt); + out.writeLong(offHeapPrimaryEntriesCnt); out.writeInt(totalPartsCnt); out.writeInt(rebalancingPartsCnt); @@ -698,8 +694,6 @@ public long getRebalancingBytesRate() { out.writeLong(rebalancingBytesRate); out.writeObject(qryMetrics); - - out.writeLong(offHeapPrimaryEntriesCnt); } /** {@inheritDoc} */ @@ -748,6 +742,7 @@ public long getRebalancingBytesRate() { heapEntriesCnt = in.readLong(); offHeapAllocatedSize = in.readLong(); offHeapEntriesCnt = in.readLong(); + offHeapPrimaryEntriesCnt = in.readLong(); totalPartsCnt = in.readInt(); rebalancingPartsCnt = in.readInt(); @@ -756,9 +751,6 @@ public long getRebalancingBytesRate() { rebalancingBytesRate = in.readLong(); qryMetrics = (VisorQueryMetrics)in.readObject(); - - if (protoVer >= V2) - offHeapPrimaryEntriesCnt = in.readLong(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryPolicyConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryPolicyConfiguration.java index d3153a6c669b7..bed4c4bb11f4e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryPolicyConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorMemoryPolicyConfiguration.java @@ -130,33 +130,26 @@ public int getEmptyPagesPoolSize() { return emptyPagesPoolSize; } - /** {@inheritDoc} */ - @Override public byte getProtocolVersion() { - return V2; - } - /** {@inheritDoc} */ @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeString(out, name); + out.writeLong(initSize); out.writeLong(maxSize); U.writeString(out, swapFilePath); U.writeEnum(out, pageEvictionMode); out.writeDouble(evictionThreshold); out.writeInt(emptyPagesPoolSize); - out.writeLong(initSize); } /** {@inheritDoc} */ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { name = U.readString(in); + initSize = in.readLong(); maxSize = in.readLong(); swapFilePath = U.readString(in); pageEvictionMode = DataPageEvictionMode.fromOrdinal(in.readByte()); evictionThreshold = in.readDouble(); emptyPagesPoolSize = in.readInt(); - - if (protoVer >= V2) - initSize = in.readLong(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 152804088664f..e34a7729771af 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -155,6 +155,7 @@ org.apache.ignite.events.EventAdapter org.apache.ignite.events.IgfsEvent org.apache.ignite.events.JobEvent org.apache.ignite.events.TaskEvent +org.apache.ignite.events.WalSegmentArchivedEvent org.apache.ignite.hadoop.HadoopInputSplit org.apache.ignite.hadoop.HadoopMapReducePlan org.apache.ignite.igfs.IgfsConcurrentModificationException @@ -337,6 +338,7 @@ org.apache.ignite.internal.marshaller.optimized.OptimizedFieldType org.apache.ignite.internal.mem.IgniteOutOfMemoryException org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl$Segment org.apache.ignite.internal.pagemem.snapshot.FinishSnapshotOperationAckDiscoveryMessage +org.apache.ignite.internal.pagemem.snapshot.SnapshotCheckParameters org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation org.apache.ignite.internal.pagemem.snapshot.SnapshotOperationType org.apache.ignite.internal.pagemem.snapshot.StartSnapshotOperationAckDiscoveryMessage @@ -407,11 +409,9 @@ org.apache.ignite.internal.processors.cache.CacheType org.apache.ignite.internal.processors.cache.CacheWeakQueryIteratorsHolder$WeakQueryCloseableIterator org.apache.ignite.internal.processors.cache.CacheWeakQueryIteratorsHolder$WeakQueryFutureIterator org.apache.ignite.internal.processors.cache.CacheWeakQueryIteratorsHolder$WeakReferenceCloseableIterator -org.apache.ignite.internal.processors.cache.ChangeGlobalStateMessage org.apache.ignite.internal.processors.cache.ClientCacheChangeDiscoveryMessage org.apache.ignite.internal.processors.cache.ClientCacheChangeDummyDiscoveryMessage org.apache.ignite.internal.processors.cache.ClusterCachesInfo$1$1 -org.apache.ignite.internal.processors.cache.ClusterState org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest org.apache.ignite.internal.processors.cache.EntryProcessorResourceInjectorProxy @@ -880,10 +880,12 @@ org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Bool org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$DestroyBag org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Result org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO$EntryPart +org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileArchiver$1 org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$RecordsIterator org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException +org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneWalRecordsIterator org.apache.ignite.internal.processors.cache.query.CacheQueryType org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryFuture$1 org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager$1 @@ -1032,9 +1034,13 @@ org.apache.ignite.internal.processors.closure.GridClosureProcessor$T8 org.apache.ignite.internal.processors.closure.GridClosureProcessor$T9 org.apache.ignite.internal.processors.closure.GridClosureProcessor$TaskNoReduceAdapter org.apache.ignite.internal.processors.closure.GridPeerDeployAwareTaskAdapter +org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage +org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage org.apache.ignite.internal.processors.cluster.ClusterProcessor$3 org.apache.ignite.internal.processors.cluster.ClusterProcessor$3$1 +org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$1$1 +org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$2 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$3 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$4 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$6 @@ -1876,6 +1882,7 @@ org.apache.ignite.internal.visor.node.VisorNodeSuppressedErrorsTask org.apache.ignite.internal.visor.node.VisorNodeSuppressedErrorsTask$VisorNodeSuppressedErrorsJob org.apache.ignite.internal.visor.node.VisorNodeSuppressedErrorsTaskArg org.apache.ignite.internal.visor.node.VisorPeerToPeerConfiguration +org.apache.ignite.internal.visor.node.VisorPersistenceMetrics org.apache.ignite.internal.visor.node.VisorPersistentStoreConfiguration org.apache.ignite.internal.visor.node.VisorRestConfiguration org.apache.ignite.internal.visor.node.VisorSegmentationConfiguration From eb37f538f55a471258e49ee2fbf0e8f234829def Mon Sep 17 00:00:00 2001 From: vsisko Date: Fri, 7 Jul 2017 23:22:59 +0700 Subject: [PATCH 107/155] IGNITE-5369 Added support for _key and _val names on QueryEntity XML and code generation. --- .../visor/query/VisorQueryEntity.java | 32 ++++++++++++++++-- modules/web-console/backend/app/mongo.js | 2 ++ .../generator/AbstractTransformer.js | 4 +-- .../generator/ConfigurationGenerator.js | 33 ++++++++++++++----- .../states/configuration/domains/query.pug | 9 +++++ .../controllers/domains-controller.js | 7 ++-- 6 files changed, 73 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryEntity.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryEntity.java index 9f4dfe75e8314..c67918af9efb7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryEntity.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryEntity.java @@ -58,6 +58,12 @@ public class VisorQueryEntity extends VisorDataTransferObject { /** Table name. */ private String tblName; + /** Key name. Can be used in field list to denote the key as a whole. */ + private String keyFieldName; + + /** Value name. Can be used in field list to denote the entire value. */ + private String valFieldName; + /** Fields to create group indexes for. */ private List grps; @@ -108,6 +114,10 @@ private VisorQueryEntity(QueryEntity q) { for (QueryIndex qryIdx : qryIdxs) grps.add(new VisorQueryIndex(qryIdx)); + + tblName = q.getTableName(); + keyFieldName = q.getKeyFieldName(); + valFieldName = q.getValueFieldName(); } /** @@ -152,6 +162,20 @@ public String getTableName() { return tblName; } + /** + * @return Key name. Can be used in field list to denote the key as a whole. + */ + public String getKeyFieldName() { + return keyFieldName; + } + + /** + * @return Value name. Can be used in field list to denote the entire value. + */ + public String getValueFieldName() { + return valFieldName; + } + /** * @return Fields to create group indexes for. */ @@ -166,8 +190,10 @@ public List getGroups() { U.writeCollection(out, keyFields); IgfsUtils.writeStringMap(out, qryFlds); U.writeMap(out, aliases); - U.writeString(out, tblName); U.writeCollection(out, grps); + U.writeString(out, tblName); + U.writeString(out, keyFieldName); + U.writeString(out, valFieldName); } /** {@inheritDoc} */ @@ -177,8 +203,10 @@ public List getGroups() { keyFields = U.readList(in); qryFlds = IgfsUtils.readStringMap(in); aliases = U.readMap(in); - tblName = U.readString(in); grps = U.readList(in); + tblName = U.readString(in); + keyFieldName = U.readString(in); + valFieldName = U.readString(in); } /** {@inheritDoc} */ diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js index 32796e6425504..aa11e0b5d0fa9 100644 --- a/modules/web-console/backend/app/mongo.js +++ b/modules/web-console/backend/app/mongo.js @@ -106,6 +106,8 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose queryMetadata: {type: String, enum: ['Annotations', 'Configuration']}, kind: {type: String, enum: ['query', 'store', 'both']}, tableName: String, + keyFieldName: String, + valueFieldName: String, databaseSchema: String, databaseTable: String, keyType: String, diff --git a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js index 0e4537e2a1275..4d66a6467631c 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js @@ -304,8 +304,8 @@ export default class AbstractTransformer { } // Generate domain model for query group. - static domainModelQuery(domain) { - return this.toSection(this.generator.domainModelQuery(domain)); + static domainModelQuery(domain, available) { + return this.toSection(this.generator.domainModelQuery(domain, available)); } // Generate domain model for store group. diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js index f850dceaa3d88..8c9b14cc12e52 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js @@ -1672,13 +1672,30 @@ export default class IgniteConfigurationGenerator { } // Generate domain model for query group. - static domainModelQuery(domain, cfg = this.domainConfigurationBean(domain)) { + static domainModelQuery(domain, available, cfg = this.domainConfigurationBean(domain)) { if (cfg.valueOf('queryMetadata') === 'Configuration') { - const fields = _.map(domain.fields, - (e) => ({name: e.name, className: javaTypes.fullClassName(e.className)})); + const fields = _.filter(_.map(domain.fields, + (e) => ({name: e.name, className: javaTypes.fullClassName(e.className)})), (field) => { + return field.name !== domain.keyFieldName && field.name !== domain.valueFieldName; + }); + + cfg.stringProperty('tableName'); + + if (available('2.0.0')) { + cfg.stringProperty('keyFieldName') + .stringProperty('valueFieldName'); + + const keyFieldName = cfg.valueOf('keyFieldName'); + const valFieldName = cfg.valueOf('valueFieldName'); + + if (keyFieldName) + fields.push({name: keyFieldName, className: javaTypes.fullClassName(domain.keyType)}); + + if (valFieldName) + fields.push({name: valFieldName, className: javaTypes.fullClassName(domain.valueType)}); + } - cfg.stringProperty('tableName') - .mapProperty('fields', fields, 'fields', true) + cfg.mapProperty('fields', fields, 'fields', true) .mapProperty('aliases', 'aliases'); const indexes = _.map(domain.indexes, (index) => @@ -2131,12 +2148,12 @@ export default class IgniteConfigurationGenerator { } // Generate domain models configs. - static cacheDomains(domains, ccfg) { + static cacheDomains(domains, available, ccfg) { const qryEntities = _.reduce(domains, (acc, domain) => { if (_.isNil(domain.queryMetadata) || domain.queryMetadata === 'Configuration') { const qryEntity = this.domainModelGeneral(domain); - this.domainModelQuery(domain, qryEntity); + this.domainModelQuery(domain, available, qryEntity); acc.push(qryEntity); } @@ -2160,7 +2177,7 @@ export default class IgniteConfigurationGenerator { this.cacheRebalance(cache, ccfg); this.cacheNearServer(cache, ccfg); this.cacheStatistics(cache, ccfg); - this.cacheDomains(cache.domains, ccfg); + this.cacheDomains(cache.domains, available, ccfg); return ccfg; } diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/query.pug b/modules/web-console/frontend/app/modules/states/configuration/domains/query.pug index b8564dc1b671f..19cb83db3a19e 100644 --- a/modules/web-console/frontend/app/modules/states/configuration/domains/query.pug +++ b/modules/web-console/frontend/app/modules/states/configuration/domains/query.pug @@ -67,6 +67,15 @@ mixin table-index-item-edit(prefix, index, sortAvailable, idAddition) div(ng-if=`${model}.queryMetadata === 'Configuration'`) .settings-row +text('Table name:', `${model}.tableName`, '"tableName"', 'false', 'Enter table name', 'Table name for this query entity') + div(ng-if='$ctrl.available("2.0.0")') + .settings-row + +text('Key field name:', `${model}.keyFieldName`, '"keyFieldName"', 'false', 'Enter key field name', + 'Key name.
' + + 'Can be used in field list to denote the key as a whole') + .settings-row + +text('Value field name:', `${model}.valueFieldName`, '"valueFieldName"', 'false', 'Enter value field name', + 'Value name.
' + + 'Can be used in field list to denote the entire value') .settings-row +ignite-form-group(ng-model=queryFields ng-form=queryFieldsForm) ignite-form-field-label(id='queryFields') diff --git a/modules/web-console/frontend/controllers/domains-controller.js b/modules/web-console/frontend/controllers/domains-controller.js index 7a16c501d6731..7c87ce00ef017 100644 --- a/modules/web-console/frontend/controllers/domains-controller.js +++ b/modules/web-console/frontend/controllers/domains-controller.js @@ -18,10 +18,12 @@ import templateUrl from 'views/configuration/domains-import.tpl.pug'; // Controller for Domain model screen. -export default ['$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'AgentManager', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData', - function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Input, Loading, ModelNormalizer, UnsavedChangesGuard, agentMgr, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes, ActivitiesData) { +export default ['$rootScope', '$scope', '$http', '$state', '$filter', '$timeout', '$modal', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteInput', 'IgniteLoading', 'IgniteModelNormalizer', 'IgniteUnsavedChangesGuard', 'AgentManager', 'IgniteLegacyTable', 'IgniteConfigurationResource', 'IgniteErrorPopover', 'IgniteFormUtils', 'JavaTypes', 'SqlTypes', 'IgniteActivitiesData', 'IgniteVersion', + function($root, $scope, $http, $state, $filter, $timeout, $modal, LegacyUtils, Messages, Focus, Confirm, ConfirmBatch, Input, Loading, ModelNormalizer, UnsavedChangesGuard, agentMgr, LegacyTable, Resource, ErrorPopover, FormUtils, JavaTypes, SqlTypes, ActivitiesData, Version) { UnsavedChangesGuard.install($scope); + this.available = Version.available.bind(Version); + const emptyDomain = {empty: true}; let __original_value; @@ -941,6 +943,7 @@ export default ['$rootScope', '$scope', '$http', '$state', '$filter', '$timeout' const keyField = newDomain.keyFields[0]; newDomain.keyType = keyField.javaType; + newDomain.keyFieldName = keyField.javaFieldName; // Exclude key column from query fields. newDomain.fields = _.filter(newDomain.fields, (field) => field.name !== keyField.javaFieldName); From 82e5f8a6553323e793c01c54e24dda6d47188ce6 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Fri, 7 Jul 2017 19:28:54 +0300 Subject: [PATCH 108/155] IGNITE-5716 .NET: Fix 2-byte field offset handling --- .../Apache.Ignite.Core.Tests.csproj | 1 + .../Binary/BinaryFooterTest.cs | 146 ++++++++++++++++++ .../Binary/BinarySelfTest.cs | 32 ---- .../Impl/Binary/BinaryObject.cs | 2 +- .../Impl/Binary/BinaryObjectBuilder.cs | 2 +- .../Impl/Binary/BinaryObjectSchemaField.cs | 3 + .../Binary/BinaryObjectSchemaSerializer.cs | 93 ++++++++--- .../Impl/Binary/BinaryReader.cs | 49 +----- 8 files changed, 228 insertions(+), 100 deletions(-) create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj index 09eac7039f121..d91f0e6b0e57f 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj @@ -72,6 +72,7 @@ + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs new file mode 100644 index 0000000000000..36f2f65164659 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.Binary +{ + using System; + using System.Linq; + using Apache.Ignite.Core.Binary; + using Apache.Ignite.Core.Impl; + using Apache.Ignite.Core.Impl.Binary; + using NUnit.Framework; + + /// + /// Tests binary footer integrity (field offsets). + /// Writes objects of various sizes to test schema compaction + /// (where field offsets can be stored as 1, 2 or 4 bytes). + /// + public class BinaryFooterTest + { + /// + /// Tears down the test. + /// + [TearDown] + public void TearDown() + { + Ignition.StopAll(true); + } + + /// + /// Tests full footers in offline mode. + /// + [Test] + public void TestFullFooterOffline() + { + // Register type to avoid unregistered type name in binary object. + // Use new marshaller to read and write to avoid schema caching. + + TestOffsets(() => new Marshaller(new BinaryConfiguration(typeof(OffsetTest)) + { + // Compact footers do not work in offline mode. + CompactFooter = false + })); + } + + /// + /// Tests the full footer online. + /// + [Test] + public void TestFullFooterOnline() + { + var ignite = Ignition.Start(new IgniteConfiguration(TestUtils.GetTestConfiguration()) + { + BinaryConfiguration = new BinaryConfiguration + { + CompactFooter = false + } + }); + + TestOffsets(() => ((Ignite) ignite).Marshaller); + } + + /// + /// Tests the full footer online. + /// + [Test] + public void TestCompactFooterOnline() + { + var ignite = Ignition.Start(TestUtils.GetTestConfiguration()); + + TestOffsets(() => ((Ignite) ignite).Marshaller); + } + + /// + /// Tests the offsets. + /// + private static void TestOffsets(Func getMarsh) + { + // Corner cases are byte/sbyte/short/ushort max values. + foreach (var i in new[] {1, sbyte.MaxValue, byte.MaxValue, short.MaxValue, ushort.MaxValue}) + { + foreach (var j in new[] {-1, 0, 1}) + { + var arrSize = i + j; + + var dt = new OffsetTest + { + Arr = Enumerable.Range(0, arrSize).Select(x => (byte) x).ToArray(), + Int = arrSize + }; + + var bytes = getMarsh().Marshal(dt); + + var res = getMarsh().Unmarshal(bytes); + var binRes = getMarsh().Unmarshal(bytes, BinaryMode.ForceBinary); + var binFieldRes = new OffsetTest + { + Arr = binRes.GetField("arr"), + Int = binRes.GetField("int") + }; + + foreach (var r in new[] {res, binRes.Deserialize(), binFieldRes}) + { + Assert.AreEqual(dt.Arr, r.Arr); + Assert.AreEqual(dt.Int, r.Int); + } + } + } + } + + /// + /// Offset test. + /// + private class OffsetTest : IBinarizable + { + public byte[] Arr; // Array to enforce field offset. + public int Int; // Value at offset. + + public void WriteBinary(IBinaryWriter writer) + { + writer.WriteByteArray("arr", Arr); + writer.WriteInt("int", Int); + } + + public void ReadBinary(IBinaryReader reader) + { + // Read in different order to enforce full schema scan. + Int = reader.ReadInt("int"); + Arr = reader.ReadByteArray("arr"); + } + } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs index 2b22d5a89c064..e24dca0ddb5cc 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs @@ -1534,38 +1534,6 @@ public void TestSpecialArrays() Assert.AreEqual(nDateArr, obj2.NDateArr); } - /// - /// Writes objects of various sizes to test schema compaction - /// (where field offsets can be stored as 1, 2 or 4 bytes). - /// - [Test] - public void TestCompactSchema() - { - var marsh = new Marshaller(new BinaryConfiguration - { - TypeConfigurations = new List - { - new BinaryTypeConfiguration(typeof (SpecialArray)), - new BinaryTypeConfiguration(typeof (SpecialArrayMarshalAware)) - } - }); - - var dt = new SpecialArrayMarshalAware(); - - foreach (var i in new[] {1, 5, 10, 13, 14, 15, 100, 200, 1000, 5000, 15000, 30000}) - { - dt.NGuidArr = Enumerable.Range(1, i).Select(x => (Guid?) Guid.NewGuid()).ToArray(); - dt.NDateArr = Enumerable.Range(1, i).Select(x => (DateTime?) DateTime.Now.AddDays(x)).ToArray(); - - var bytes = marsh.Marshal(dt); - - var res = marsh.Unmarshal(bytes); - - CollectionAssert.AreEquivalent(dt.NGuidArr, res.NGuidArr); - CollectionAssert.AreEquivalent(dt.NDateArr, res.NDateArr); - } - } - [Test] public void TestBinaryConfigurationValidation() { diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs index 8c5cee6c4744f..370233faa8bbb 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs @@ -239,7 +239,7 @@ private void InitializeFields(IBinaryTypeDescriptor desc = null) { var hdr = BinaryObjectHeader.Read(stream, _offset); - _fields = BinaryObjectSchemaSerializer.ReadSchema(stream, _offset, hdr, desc.Schema,_marsh) + _fields = BinaryObjectSchemaSerializer.ReadSchema(stream, _offset, hdr, desc.Schema, _marsh.Ignite) .ToDictionary() ?? EmptyFields; } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs index 91fe12ae40127..c310b3afa4064 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs @@ -617,7 +617,7 @@ private IBinaryObjectBuilder SetField0(string fieldName, BinaryBuilderField val) { // New object, write in full form. var inSchema = BinaryObjectSchemaSerializer.ReadSchema(inStream, inStartPos, inHeader, - _desc.Schema, _binary.Marshaller); + _desc.Schema, _binary.Marshaller.Ignite); var outSchema = BinaryObjectSchemaHolder.Current; var schemaIdx = outSchema.PushSchema(); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs index 3c5339af662a6..be6278a380180 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs @@ -17,6 +17,7 @@ namespace Apache.Ignite.Core.Impl.Binary { + using System.Diagnostics; using System.Runtime.InteropServices; /// @@ -41,6 +42,8 @@ internal struct BinaryObjectSchemaField /// The offset. public BinaryObjectSchemaField(int id, int offset) { + Debug.Assert(offset >= 0); + Id = id; Offset = offset; } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs index e2f9ea7813abf..1d699c248f576 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs @@ -54,18 +54,17 @@ internal static class BinaryObjectSchemaSerializer /// The position. /// The header. /// The schema. - /// The marshaller. + /// The ignite. /// /// Schema. /// public static BinaryObjectSchemaField[] ReadSchema(IBinaryStream stream, int position, BinaryObjectHeader hdr, - BinaryObjectSchema schema, Marshaller marsh) + BinaryObjectSchema schema, Ignite ignite) { Debug.Assert(stream != null); Debug.Assert(schema != null); - Debug.Assert(marsh != null); - return ReadSchema(stream, position, hdr, () => GetFieldIds(hdr, schema, marsh)); + return ReadSchema(stream, position, hdr, () => GetFieldIds(hdr, schema, ignite)); } /// @@ -78,8 +77,8 @@ internal static class BinaryObjectSchemaSerializer /// /// Schema. /// - public static BinaryObjectSchemaField[] ReadSchema(IBinaryStream stream, int position, BinaryObjectHeader hdr, - Func fieldIdsFunc) + public static unsafe BinaryObjectSchemaField[] ReadSchema(IBinaryStream stream, int position, + BinaryObjectHeader hdr, Func fieldIdsFunc) { Debug.Assert(stream != null); Debug.Assert(fieldIdsFunc != null); @@ -110,7 +109,7 @@ internal static class BinaryObjectSchemaSerializer else if (offsetSize == 2) { for (var i = 0; i < schemaSize; i++) - res[i] = new BinaryObjectSchemaField(fieldIds[i], stream.ReadShort()); + res[i] = new BinaryObjectSchemaField(fieldIds[i], (ushort) stream.ReadShort()); } else { @@ -128,12 +127,22 @@ internal static class BinaryObjectSchemaSerializer else if (offsetSize == 2) { for (var i = 0; i < schemaSize; i++) - res[i] = new BinaryObjectSchemaField(stream.ReadInt(), stream.ReadShort()); + res[i] = new BinaryObjectSchemaField(stream.ReadInt(), (ushort) stream.ReadShort()); } else { - for (var i = 0; i < schemaSize; i++) - res[i] = new BinaryObjectSchemaField(stream.ReadInt(), stream.ReadInt()); + if (BitConverter.IsLittleEndian) + { + fixed (BinaryObjectSchemaField* ptr = &res[0]) + { + stream.Read((byte*) ptr, schemaSize * BinaryObjectSchemaField.Size); + } + } + else + { + for (var i = 0; i < schemaSize; i++) + res[i] = new BinaryObjectSchemaField(stream.ReadInt(), stream.ReadInt()); + } } } @@ -220,7 +229,7 @@ internal static class BinaryObjectSchemaSerializer { fixed (BinaryObjectSchemaField* ptr = &fields[offset]) { - stream.Write((byte*)ptr, count / BinaryObjectSchemaField.Size); + stream.Write((byte*)ptr, count * BinaryObjectSchemaField.Size); } } else @@ -243,22 +252,66 @@ internal static class BinaryObjectSchemaSerializer /// /// Gets the field ids. /// - private static int[] GetFieldIds(BinaryObjectHeader hdr, BinaryObjectSchema schema, Marshaller marsh) + private static int[] GetFieldIds(BinaryObjectHeader hdr, Ignite ignite) { - var fieldIds = schema.Get(hdr.SchemaId); + Debug.Assert(hdr.TypeId != BinaryUtils.TypeUnregistered); + + int[] fieldIds = null; + + if (ignite != null) + { + fieldIds = ignite.BinaryProcessor.GetSchema(hdr.TypeId, hdr.SchemaId); + } if (fieldIds == null) { - Debug.Assert(hdr.TypeId != BinaryUtils.TypeUnregistered); + throw new BinaryObjectException("Cannot find schema for object with compact footer [" + + "typeId=" + hdr.TypeId + ", schemaId=" + hdr.SchemaId + ']'); + } + return fieldIds; + } - if (marsh.Ignite != null) - fieldIds = marsh.Ignite.BinaryProcessor.GetSchema(hdr.TypeId, hdr.SchemaId); + /// + /// Reads the schema, maintains stream position. + /// + public static int[] GetFieldIds(BinaryObjectHeader hdr, Ignite ignite, IBinaryStream stream, int objectPos) + { + Debug.Assert(stream != null); - if (fieldIds == null) - throw new BinaryObjectException("Cannot find schema for object with compact footer [" + - "typeId=" + hdr.TypeId + ", schemaId=" + hdr.SchemaId + ']'); + if (hdr.IsCompactFooter) + { + // Get schema from Java + return GetFieldIds(hdr, ignite); } - return fieldIds; + + var pos = stream.Position; + + stream.Seek(objectPos + hdr.SchemaOffset, SeekOrigin.Begin); + + var count = hdr.SchemaFieldCount; + + var offsetSize = hdr.SchemaFieldOffsetSize; + + var res = new int[count]; + + for (var i = 0; i < count; i++) + { + res[i] = stream.ReadInt(); + stream.Seek(offsetSize, SeekOrigin.Current); // Skip offsets. + } + + stream.Seek(pos, SeekOrigin.Begin); + + return res; + } + + + /// + /// Gets the field ids. + /// + private static int[] GetFieldIds(BinaryObjectHeader hdr, BinaryObjectSchema schema, Ignite ignite) + { + return schema.Get(hdr.SchemaId) ?? GetFieldIds(hdr, ignite); } } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs index e792dce66764e..76237c42ffbea 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs @@ -20,7 +20,6 @@ namespace Apache.Ignite.Core.Impl.Binary using System; using System.Collections; using System.Collections.Generic; - using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using Apache.Ignite.Core.Binary; @@ -783,7 +782,8 @@ private void SetCurSchema(IBinaryTypeDescriptor desc) if (_frame.Schema == null) { - _frame.Schema = ReadSchema(desc.TypeId); + _frame.Schema = + BinaryObjectSchemaSerializer.GetFieldIds(_frame.Hdr, Marshaller.Ignite, Stream, _frame.Pos); desc.Schema.Add(_frame.Hdr.SchemaId, _frame.Schema); } @@ -794,49 +794,6 @@ private void SetCurSchema(IBinaryTypeDescriptor desc) } } - /// - /// Reads the schema. - /// - private int[] ReadSchema(int typeId) - { - if (_frame.Hdr.IsCompactFooter) - { - // Get schema from Java - var ignite = Marshaller.Ignite; - - Debug.Assert(typeId != BinaryUtils.TypeUnregistered); - - var schema = ignite == null - ? null - : ignite.BinaryProcessor.GetSchema(_frame.Hdr.TypeId, _frame.Hdr.SchemaId); - - if (schema == null) - throw new BinaryObjectException("Cannot find schema for object with compact footer [" + - "typeId=" + typeId + ", schemaId=" + _frame.Hdr.SchemaId + ']'); - - return schema; - } - - var pos = Stream.Position; - - Stream.Seek(_frame.Pos + _frame.Hdr.SchemaOffset, SeekOrigin.Begin); - - var count = _frame.Hdr.SchemaFieldCount; - - var offsetSize = _frame.Hdr.SchemaFieldOffsetSize; - - var res = new int[count]; - - for (int i = 0; i < count; i++) - { - res[i] = Stream.ReadInt(); - Stream.Seek(offsetSize, SeekOrigin.Current); - } - - Stream.Seek(pos, SeekOrigin.Begin); - - return res; - } /// /// Reads the handle object. /// @@ -942,7 +899,7 @@ private bool SeekField(string fieldName) int pos; - if (!_frame.SchemaMap.TryGetValue(fieldId, out pos)) + if (_frame.SchemaMap == null || !_frame.SchemaMap.TryGetValue(fieldId, out pos)) return false; Stream.Seek(pos + _frame.Pos, SeekOrigin.Begin); From cb5ae9617af900e953e0f6f065adcd707d2b3830 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Fri, 7 Jul 2017 20:15:47 +0300 Subject: [PATCH 109/155] .NET: Remove unused import and redundant cast --- .../Cache/Configuration/CacheConfiguration.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs index f5a517949af96..d08a191aed64a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs @@ -27,7 +27,6 @@ namespace Apache.Ignite.Core.Cache.Configuration using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; - using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Cache; using Apache.Ignite.Core.Cache.Affinity; using Apache.Ignite.Core.Cache.Affinity.Rendezvous; @@ -242,7 +241,7 @@ internal CacheConfiguration(BinaryReader reader) private void Read(BinaryReader reader) { // Make sure system marshaller is used. - Debug.Assert(((BinaryReader) reader).Marshaller == BinaryUtils.Marshaller); + Debug.Assert(reader.Marshaller == BinaryUtils.Marshaller); AtomicityMode = (CacheAtomicityMode) reader.ReadInt(); Backups = reader.ReadInt(); From 17904cb174a340ec928f4a7a4b12d294c2240d0f Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Fri, 7 Jul 2017 20:18:35 +0300 Subject: [PATCH 110/155] .NET: Fix IgniteConfigurationTest --- .../Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs index 578d92360a1fd..5e5cb1c610e3b 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs @@ -683,7 +683,7 @@ private static IgniteConfiguration GetCustomConfig() new MemoryPolicyConfiguration { Name = "myDefaultPlc", - PageEvictionMode = DataPageEvictionMode.Random2Lru, + PageEvictionMode = DataPageEvictionMode.Disabled, InitialSize = 340 * 1024 * 1024, MaxSize = 345 * 1024 * 1024, EvictionThreshold = 0.88, @@ -695,7 +695,7 @@ private static IgniteConfiguration GetCustomConfig() new MemoryPolicyConfiguration { Name = "customPlc", - PageEvictionMode = DataPageEvictionMode.RandomLru, + PageEvictionMode = DataPageEvictionMode.Disabled, MaxSize = 456 * 1024 * 1024, EvictionThreshold = 0.77, EmptyPagesPoolSize = 66, From 36716fb1f4de5d1f5baab1e27afc95583dd4c7b4 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Mon, 10 Jul 2017 11:09:43 +0300 Subject: [PATCH 111/155] GG-12466 - Clean up public API for snapshots --- ...hSnapshotOperationAckDiscoveryMessage.java | 84 ------- .../snapshot/SnapshotCheckParameters.java | 75 ------ .../pagemem/snapshot/SnapshotOperation.java | 235 ------------------ ...tSnapshotOperationAckDiscoveryMessage.java | 149 ----------- ...tartSnapshotOperationDiscoveryMessage.java | 192 -------------- .../GridCachePartitionExchangeManager.java | 9 +- .../processors/cache/GridCacheProcessor.java | 22 +- .../cache/GridCacheSharedContext.java | 6 +- .../dht/GridDhtPartitionTopologyImpl.java | 8 +- .../GridDhtPartitionsExchangeFuture.java | 54 ++-- .../GridCacheDatabaseSharedManager.java | 3 +- .../file/FilePageStoreManager.java | 2 +- .../IgniteCacheSnapshotManager.java | 25 +- .../snapshot/SnapshotDiscoveryMessage.java | 33 +++ .../snapshot/SnapshotOperation.java} | 45 ++-- .../resources/META-INF/classnames.properties | 5 - .../hashmap/GridCacheTestContext.java | 6 +- 17 files changed, 114 insertions(+), 839 deletions(-) delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/StartSnapshotOperationAckDiscoveryMessage.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/StartSnapshotOperationDiscoveryMessage.java rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/{ => snapshot}/IgniteCacheSnapshotManager.java (83%) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java rename modules/core/src/main/java/org/apache/ignite/internal/{pagemem/snapshot/SnapshotOperationType.java => processors/cache/persistence/snapshot/SnapshotOperation.java} (53%) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java deleted file mode 100644 index f6758e08d4ad6..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/FinishSnapshotOperationAckDiscoveryMessage.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.pagemem.snapshot; - -import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; -import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.lang.IgniteUuid; -import org.jetbrains.annotations.Nullable; - -/** - * - */ -public class FinishSnapshotOperationAckDiscoveryMessage implements DiscoveryCustomMessage { - /** */ - private static final long serialVersionUID = 0L; - - /** Id. */ - private final IgniteUuid id = IgniteUuid.randomUuid(); - - /** Op id. */ - private final IgniteUuid opId; - - /** Success. */ - private final boolean success; - - /** - * @param opId Op id. - * @param success Success. - */ - public FinishSnapshotOperationAckDiscoveryMessage(IgniteUuid opId, boolean success) { - this.opId = opId; - this.success = success; - } - - /** {@inheritDoc} */ - @Override public IgniteUuid id() { - return id; - } - - /** {@inheritDoc} */ - @Nullable @Override public DiscoveryCustomMessage ackMessage() { - return null; - } - - /** {@inheritDoc} */ - @Override public boolean isMutable() { - return false; - } - - /** - * @return Op id. - */ - public IgniteUuid operationId() { - return opId; - } - - /** - * @return Success. - */ - public boolean success() { - return success; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(FinishSnapshotOperationAckDiscoveryMessage.class, this, - "id", id, "opId", opId, "success", success); - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java deleted file mode 100644 index 58cb2406a7240..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotCheckParameters.java +++ /dev/null @@ -1,75 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.ignite.internal.pagemem.snapshot; - -import java.io.File; -import java.io.Serializable; -import java.util.Collection; -import org.jetbrains.annotations.Nullable; - -/** - * Tuple for passing optional parameters of {@link SnapshotOperationType#CHECK}. - */ -public class SnapshotCheckParameters implements Serializable { - /** */ - private static final long serialVersionUID = 0L; - - /** Optional paths. */ - private final Collection optionalPaths; - - /** Flag for skipping CRC check. */ - private final boolean skipCrc; - - /** - * Factory method. - * - * @return Tuple with optional parameters or null if parameters are default. - * - * @param optionalPaths Optional paths. - * @param skipCrc Skip crc. - */ - @Nullable public static SnapshotCheckParameters valueOf(Collection optionalPaths, boolean skipCrc) { - if (optionalPaths == null && !skipCrc) - return null; - - return new SnapshotCheckParameters(optionalPaths, skipCrc); - } - - /** - * @param optionalPaths Optional paths. - * @param skipCrc Flag for skipping CRC check. - */ - private SnapshotCheckParameters(Collection optionalPaths, boolean skipCrc) { - this.optionalPaths = optionalPaths; - this.skipCrc = skipCrc; - } - - /** - * @return Optional paths. - */ - public Collection optionalPaths() { - return optionalPaths; - } - - /** - * @return Flag for skipping CRC check. - */ - public boolean skipCrc() { - return skipCrc; - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java deleted file mode 100644 index fa18cd7a01045..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperation.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.ignite.internal.pagemem.snapshot; - -import java.io.File; -import java.io.Serializable; -import java.util.Collection; -import java.util.Map; -import java.util.Set; - -/** - * Description and parameters of snapshot operation - */ -public class SnapshotOperation implements Serializable { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - private final SnapshotOperationType type; - - /** Snapshot ID (the timestamp of snapshot creation). */ - private final long snapshotId; - - /** Cache group ids. */ - private final Set cacheGrpIds; - - /** Cache names. */ - private final Set cacheNames; - - /** Message. */ - private final String msg; - - /** Additional parameter. */ - private final Object extraParam; - - /** Optional list of dependent snapshot IDs. */ - private final Set dependentSnapshotIds; - - /** Optional map of previous snapshots grouped by caches. */ - private final Map> prevSnapshots; - - /** - * @param type Type. - * @param snapshotId Snapshot id. - * @param cacheGrpIds Cache group ids. - * @param cacheNames Cache names. - * @param msg Extra user message. - * @param extraParam Additional parameter. - * @param dependentSnapshotIds Optional list of dependent snapshot IDs. - * @param prevSnapshots Optional map of previous snapshots grouped by caches. - */ - public SnapshotOperation( - SnapshotOperationType type, - long snapshotId, - Set cacheGrpIds, - Set cacheNames, - String msg, - Object extraParam, - Set dependentSnapshotIds, - Map> prevSnapshots - ) { - this.type = type; - this.snapshotId = snapshotId; - this.cacheGrpIds = cacheGrpIds; - this.cacheNames = cacheNames; - this.msg = msg; - this.extraParam = extraParam; - this.dependentSnapshotIds = dependentSnapshotIds; - this.prevSnapshots = prevSnapshots; - } - - /** - * - */ - public SnapshotOperationType type() { - return type; - } - - /** - * Snapshot ID (the timestamp of snapshot creation). - * - * @return Snapshot ID. - */ - public long snapshotId() { - return snapshotId; - } - - /** - * Cache group ids included to this snapshot. - * - * @return Cache names. - */ - public Set cacheGroupIds() { - return cacheGrpIds; - } - - /** - * Cache names included to this snapshot. - */ - public Set cacheNames() { - return cacheNames; - } - - /** - * Additional info which was provided by client. - */ - public String message() { - return msg; - } - - /** - * - */ - public Object extraParameter() { - return extraParam; - } - - /** - * @return Optional dependent snapshot IDs. - */ - public Set dependentSnapshotIds() { - return dependentSnapshotIds; - } - - /** - * @return Cache names grouped by previous snapshot IDs. - */ - public Map> previousSnapshots() { - return prevSnapshots; - } - - /** - * @param op Op. - */ - public static Collection getOptionalPathsParameter(SnapshotOperation op) { - assert (op.type() == SnapshotOperationType.RESTORE || - op.type() == SnapshotOperationType.RESTORE_2_PHASE) - && (op.extraParameter() == null || op.extraParameter() instanceof Collection) - || (op.type() == SnapshotOperationType.CHECK && - (op.extraParameter() == null || op.extraParameter() instanceof SnapshotCheckParameters)); - - if (op.type() == SnapshotOperationType.CHECK) { - if (op.extraParameter() == null) - return null; - else - return ((SnapshotCheckParameters)op.extraParameter()).optionalPaths(); - } - - return (Collection)op.extraParameter(); - } - - /** - * @param op Op. - */ - public static boolean getSkipCrcParameter(SnapshotOperation op) { - assert op.type() == SnapshotOperationType.CHECK && - (op.extraParameter() == null | op.extraParameter() instanceof SnapshotCheckParameters); - - return op.extraParameter() != null && ((SnapshotCheckParameters)op.extraParameter()).skipCrc(); - } - - /** - * @param op Op. - */ - public static Boolean getFullSnapshotParameter(SnapshotOperation op) { - assert op.type() == SnapshotOperationType.CREATE && op.extraParameter() instanceof Boolean; - - return (Boolean)op.extraParameter(); - } - - /** - * @param op Op. - */ - public static File getMovingPathParameter(SnapshotOperation op) { - assert op.type() == SnapshotOperationType.MOVE && op.extraParameter() instanceof File; - - return (File)op.extraParameter(); - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (o == null || getClass() != o.getClass()) - return false; - - SnapshotOperation operation = (SnapshotOperation)o; - - if (snapshotId != operation.snapshotId) - return false; - - if (type != operation.type) - return false; - - return extraParam != null ? extraParam.equals(operation.extraParam) : operation.extraParam == null; - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - int res = type.hashCode(); - res = 31 * res + (int)(snapshotId ^ (snapshotId >>> 32)); - res = 31 * res + (extraParam != null ? extraParam.hashCode() : 0); - return res; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return "SnapshotOperation{" + - "type=" + type + - ", snapshotId=" + snapshotId + - ", cacheNames=" + cacheNames + - ", cacheGroupIds=" + cacheGrpIds + - ", msg='" + msg + '\'' + - ", extraParam=" + extraParam + - ", dependentSnapshotIds=" + dependentSnapshotIds + - ", prevSnapshots=" + prevSnapshots + - '}'; - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/StartSnapshotOperationAckDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/StartSnapshotOperationAckDiscoveryMessage.java deleted file mode 100644 index af7648d2edc7e..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/StartSnapshotOperationAckDiscoveryMessage.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.ignite.internal.pagemem.snapshot; - -import java.util.Map; -import java.util.UUID; -import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; -import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.lang.IgniteUuid; -import org.jetbrains.annotations.Nullable; - -/** - * Message indicating that a snapshot has been started. - */ -public class StartSnapshotOperationAckDiscoveryMessage implements DiscoveryCustomMessage { - /** */ - private static final long serialVersionUID = 0L; - - - private SnapshotOperation snapshotOperation; - - /** Custom message ID. */ - private IgniteUuid id = IgniteUuid.randomUuid(); - - /** Operation id. */ - private IgniteUuid opId; - - /** */ - private Exception err; - - /** */ - private UUID initiatorNodeId; - - /** Last full snapshot id for cache. */ - private Map lastFullSnapshotIdForCache; - - /** Last snapshot id for cache. */ - private Map lastSnapshotIdForCache; - - /** - * @param snapshotOperation Snapshot Operation. - * @param err Error. - */ - public StartSnapshotOperationAckDiscoveryMessage( - IgniteUuid id, - SnapshotOperation snapshotOperation, - Map lastFullSnapshotIdForCache, - Map lastSnapshotIdForCache, - Exception err, - UUID initiatorNodeId - ) { - this.opId = id; - this.snapshotOperation = snapshotOperation; - this.lastFullSnapshotIdForCache = lastFullSnapshotIdForCache; - this.lastSnapshotIdForCache = lastSnapshotIdForCache; - this.err = err; - this.initiatorNodeId = initiatorNodeId; - } - - /** {@inheritDoc} */ - @Override public IgniteUuid id() { - return id; - } - - /** - * - */ - public boolean needExchange() { - /* exchange for trigger saving cluster state*/ - return err == null && snapshotOperation.type() == SnapshotOperationType.CREATE; - } - - /** - * - */ - public IgniteUuid operationId() { - return opId; - } - - /** - * @return Initiator node id. - */ - public UUID initiatorNodeId() { - return initiatorNodeId; - } - - /** - * @return Error if start this process is not successfully. - */ - public Exception error() { - return err; - } - - /** - * @return {@code True} if message has error otherwise {@code false}. - */ - public boolean hasError() { - return err != null; - } - - public SnapshotOperation snapshotOperation() { - return snapshotOperation; - } - - /** - * @param cacheId Cache id. - */ - @Nullable public Long lastFullSnapshotId(int cacheId) { - return lastFullSnapshotIdForCache.get(cacheId); - } - - /** - * @param cacheId Cache id. - */ - @Nullable public Long lastSnapshotId(int cacheId) { - return lastSnapshotIdForCache.get(cacheId); - } - - /** {@inheritDoc} */ - @Nullable @Override public DiscoveryCustomMessage ackMessage() { - return null; - } - - /** {@inheritDoc} */ - @Override public boolean isMutable() { - return false; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(StartSnapshotOperationAckDiscoveryMessage.class, this); - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/StartSnapshotOperationDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/StartSnapshotOperationDiscoveryMessage.java deleted file mode 100644 index 4c9deb593ddfe..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/StartSnapshotOperationDiscoveryMessage.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.ignite.internal.pagemem.snapshot; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import org.apache.ignite.IgniteException; -import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; -import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.lang.IgniteUuid; -import org.jetbrains.annotations.Nullable; - -/** - * Message indicating that a snapshot has been started. - */ -public class StartSnapshotOperationDiscoveryMessage implements DiscoveryCustomMessage { - /** */ - private static final long serialVersionUID = 0L; - - /** Id. */ - private IgniteUuid id = IgniteUuid.randomUuid(); - - /** Custom message ID. */ - private IgniteUuid operationId; - - /** Snapshot operation. */ - private SnapshotOperation snapshotOperation; - - /** */ - private UUID initiatorId; - - /** Validated by coordinator. */ - private boolean validatedByCoordinator = false; - - /** Error. */ - private Exception err; - - /** Last full snapshot id for cache. */ - private Map lastFullSnapshotIdForCache = new HashMap<>(); - - /** Last snapshot id for cache. */ - private Map lastSnapshotIdForCache = new HashMap<>(); - - /** - * @param snapshotOperation Snapshot operation - * @param initiatorId initiator node id - */ - public StartSnapshotOperationDiscoveryMessage( - IgniteUuid operationId, - SnapshotOperation snapshotOperation, - UUID initiatorId - ) { - this.operationId = operationId; - this.snapshotOperation = snapshotOperation; - this.initiatorId = initiatorId; - } - - /** - * - */ - public SnapshotOperation snapshotOperation() { - return snapshotOperation; - } - - /** - * Sets error. - * - * @param err Error. - */ - public void error(Exception err) { - this.err = err; - } - - /** - * @return {@code True} if message contains error. - */ - public boolean hasError() { - return err != null; - } - - /** - * @return Error. - */ - public Exception error() { - return err; - } - - /** - * @return Initiator node id. - */ - public UUID initiatorNodeId() { - return initiatorId; - } - - /** {@inheritDoc} */ - @Override public IgniteUuid id() { - return id; - } - - /** - * @return Operation ID. - */ - public IgniteUuid operationId() { - return operationId; - } - - /** - * @param cacheId Cache id. - */ - public Long lastFullSnapshotId(int cacheId) { - return lastFullSnapshotIdForCache.get(cacheId); - } - - /** - * @param cacheId Cache id. - * @param id Id. - */ - public void lastFullSnapshotId(int cacheId, long id) { - lastFullSnapshotIdForCache.put(cacheId, id); - } - - /** - * @param cacheId Cache id. - */ - public Long lastSnapshotId(int cacheId) { - return lastSnapshotIdForCache.get(cacheId); - } - - /** @return Validated by coordinator. */ - public boolean validatedByCoordinator() { - return validatedByCoordinator; - } - - /** Validated by coordinator. */ - public void validatedByCoordinator(boolean validatedByCoordinator) { - this.validatedByCoordinator = validatedByCoordinator; - } - - /** - * @param cacheId Cache id. - * @param id Id. - */ - public void lastSnapshotId(int cacheId, long id) { - lastSnapshotIdForCache.put(cacheId, id); - } - - /** {@inheritDoc} */ - @Nullable @Override public DiscoveryCustomMessage ackMessage() { - return new StartSnapshotOperationAckDiscoveryMessage( - operationId, - snapshotOperation, - lastFullSnapshotIdForCache, - lastSnapshotIdForCache, - err != null ? err : (validatedByCoordinator? null : new IgniteException("Coordinator didn't validate operation!")), - initiatorId); - } - - /** {@inheritDoc} */ - @Override public boolean isMutable() { - return true; - } - - /** - * @param snapshotOperation new snapshot operation - */ - public void snapshotOperation(SnapshotOperation snapshotOperation) { - this.snapshotOperation = snapshotOperation; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(StartSnapshotOperationDiscoveryMessage.class, this); - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 8506cdebd0d08..f1db79a8adb4f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -32,7 +32,6 @@ import java.util.NavigableMap; import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.LinkedBlockingDeque; @@ -61,7 +60,6 @@ import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData; import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener; -import org.apache.ignite.internal.pagemem.snapshot.StartSnapshotOperationAckDiscoveryMessage; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache; import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology; @@ -79,6 +77,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionHistorySuppliersMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; @@ -87,8 +86,8 @@ import org.apache.ignite.internal.processors.query.schema.SchemaNodeLeaveExchangeWorkerTask; import org.apache.ignite.internal.processors.timeout.GridTimeoutObject; import org.apache.ignite.internal.util.GridListSet; -import org.apache.ignite.internal.util.future.GridCompoundFuture; import org.apache.ignite.internal.util.GridPartitionStateMap; +import org.apache.ignite.internal.util.future.GridCompoundFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.lang.IgnitePair; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -421,8 +420,8 @@ else if (msg.exchangeId().topologyVersion().topologyVersion() >= cctx.discovery( exchangeFuture(msg.exchangeId(), null, null, null, null) .onAffinityChangeMessage(evt.eventNode(), msg); } - else if (customMsg instanceof StartSnapshotOperationAckDiscoveryMessage - && ((StartSnapshotOperationAckDiscoveryMessage)customMsg).needExchange()) { + else if (customMsg instanceof SnapshotDiscoveryMessage + && ((SnapshotDiscoveryMessage) customMsg).needExchange()) { exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt.type()); exchFut = exchangeFuture(exchId, evt, null, null, null); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 0a69d728ea855..0488a142533c6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -69,21 +69,13 @@ import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.internal.binary.GridBinaryMarshaller; import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; -import org.apache.ignite.internal.pagemem.snapshot.StartSnapshotOperationAckDiscoveryMessage; import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.CacheJoinNodeDiscoveryData.CacheInfo; import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache; +import org.apache.ignite.internal.processors.cache.CacheJoinNodeDiscoveryData.CacheInfo; import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl; -import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; -import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; -import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheSnapshotManager; -import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy; -import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; -import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; -import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCache; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter; @@ -97,6 +89,14 @@ import org.apache.ignite.internal.processors.cache.jta.CacheJtaManagerAdapter; import org.apache.ignite.internal.processors.cache.local.GridLocalCache; import org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage; +import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager; import org.apache.ignite.internal.processors.cache.query.GridCacheLocalQueryManager; @@ -2969,8 +2969,8 @@ public boolean onCustomEvent(DiscoveryCustomMessage msg, AffinityTopologyVersion if (msg instanceof CacheAffinityChangeMessage) return sharedCtx.affinity().onCustomEvent(((CacheAffinityChangeMessage)msg)); - if (msg instanceof StartSnapshotOperationAckDiscoveryMessage && - ((StartSnapshotOperationAckDiscoveryMessage)msg).needExchange()) + if (msg instanceof SnapshotDiscoveryMessage && + ((SnapshotDiscoveryMessage)msg).needExchange()) return true; if (msg instanceof DynamicCacheChangeBatch) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java index 40b263f599113..efd90a88108c4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java @@ -42,12 +42,12 @@ import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; -import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; -import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheSnapshotManager; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal; import org.apache.ignite.internal.processors.cache.jta.CacheJtaManagerAdapter; +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; import org.apache.ignite.internal.processors.cache.store.CacheStoreManager; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index 2f548108f5a7e..a69872f827f85 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -1740,11 +1740,11 @@ else if (plc != PartitionLossPolicy.IGNORE) { result.add(e.getKey()); } - } - U.warn(log, "Partition has been scheduled for rebalancing due to outdated update counter " + - "[nodeId=" + ctx.localNodeId() + "cacheOrGroupName=" + grp.cacheOrGroupName() + - ", partId=" + locPart.id() + ", haveHistory=" + haveHistory + "]"); + U.warn(log, "Partition has been scheduled for rebalancing due to outdated update counter " + + "[nodeId=" + ctx.localNodeId() + "cacheOrGroupName=" + grp.cacheOrGroupName() + + ", partId=" + p + ", haveHistory=" + haveHistory + "]"); + } } if (updateSeq) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 97fcb12bf5bc6..90c8aafb255e5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -52,8 +52,6 @@ import org.apache.ignite.internal.events.DiscoveryCustomEvent; import org.apache.ignite.internal.managers.discovery.DiscoCache; import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; -import org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation; -import org.apache.ignite.internal.pagemem.snapshot.StartSnapshotOperationAckDiscoveryMessage; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache; import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage; @@ -66,11 +64,12 @@ import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.StateChangeRequest; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter; import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage; @@ -581,7 +580,7 @@ else if (msg instanceof DynamicCacheChangeBatch) { exchange = onCacheChangeRequest(crdNode); } - else if (msg instanceof StartSnapshotOperationAckDiscoveryMessage) { + else if (msg instanceof SnapshotDiscoveryMessage) { exchange = CU.clientNode(discoEvt.eventNode()) ? onClientNodeEvent(crdNode) : onServerNodeEvent(crdNode); @@ -658,7 +657,7 @@ else if (msg instanceof StartSnapshotOperationAckDiscoveryMessage) { } if (cctx.localNode().isClient()) - startLocalSnasphotOperation(); + tryToPerformLocalSnapshotOperation(); exchLog.info("Finished exchange init [topVer=" + topVer + ", crd=" + crdNode + ']'); } @@ -1007,26 +1006,18 @@ private void distributedExchange() throws IgniteCheckedException { } /** - + * Try to start local snapshot operation if it is needed by discovery event */ - private void startLocalSnasphotOperation() { - StartSnapshotOperationAckDiscoveryMessage snapOpMsg= getSnapshotOperationMessage(); - - if (snapOpMsg != null) { - SnapshotOperation op = snapOpMsg.snapshotOperation(); - - assert snapOpMsg.needExchange(); - - try { - IgniteInternalFuture fut = cctx.snapshot() - .startLocalSnapshotOperation(snapOpMsg.initiatorNodeId(), snapOpMsg.snapshotOperation()); + private void tryToPerformLocalSnapshotOperation() { + try { + IgniteInternalFuture fut = cctx.snapshot() + .tryStartLocalSnapshotOperation(discoEvt); - if (fut != null) - fut.get(); - } - catch (IgniteCheckedException e) { - U.error(log, "Error while starting snapshot operation", e); - } + if (fut != null) + fut.get(); + } + catch (IgniteCheckedException e) { + U.error(log, "Error while starting snapshot operation", e); } } @@ -1395,7 +1386,7 @@ public boolean serverNodeDiscoveryEvent() { grpValidRes = m; } - startLocalSnasphotOperation(); + tryToPerformLocalSnapshotOperation(); cctx.cache().onExchangeDone(exchId.topologyVersion(), exchActions, err); @@ -1455,21 +1446,6 @@ public boolean serverNodeDiscoveryEvent() { return dummy; } - /** - * - */ - private StartSnapshotOperationAckDiscoveryMessage getSnapshotOperationMessage() { - // If it's a snapshot operation request, synchronously wait for backup start. - if (discoEvt != null && discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) { - DiscoveryCustomMessage customMsg = ((DiscoveryCustomEvent)discoEvt).customMessage(); - - if (customMsg instanceof StartSnapshotOperationAckDiscoveryMessage) - return (StartSnapshotOperationAckDiscoveryMessage)customMsg; - } - - return null; - } - /** * Cleans up resources to avoid excessive memory usage. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index d64677e44305b..8fe9377cfe79c 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -81,7 +81,6 @@ import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.PageUtils; -import org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation; import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager; import org.apache.ignite.internal.pagemem.store.PageStore; import org.apache.ignite.internal.pagemem.wal.StorageException; @@ -110,6 +109,8 @@ import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index 28bf6e4e6dad8..6aa2243bacd3f 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -47,7 +47,7 @@ import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; import org.apache.ignite.internal.processors.cache.StoredCacheData; -import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheSnapshotManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.marshaller.Marshaller; import org.apache.ignite.marshaller.jdk.JdkMarshaller; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java similarity index 83% rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java index cce6f55d4a494..0a27bcd2d8f6e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java @@ -15,17 +15,17 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.cache.persistence; +package org.apache.ignite.internal.processors.cache.persistence.snapshot; import java.nio.ByteBuffer; import java.util.NavigableMap; import java.util.UUID; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.events.DiscoveryEvent; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.pagemem.FullPageId; import org.apache.ignite.internal.pagemem.PageMemory; -import org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; @@ -34,30 +34,41 @@ import org.jetbrains.annotations.Nullable; /** - * + * Snapshot manager stub. */ -public class IgniteCacheSnapshotManager extends GridCacheSharedManagerAdapter implements IgniteChangeGlobalStateSupport { +public class IgniteCacheSnapshotManager extends GridCacheSharedManagerAdapter implements IgniteChangeGlobalStateSupport { /** Snapshot started lock filename. */ public static final String SNAPSHOT_RESTORE_STARTED_LOCK_FILENAME = "snapshot-started.loc"; + /** + * Try to start local snapshot operation if it's required by discovery event. + * + * @param discoveryEvent Discovery event. + */ + @Nullable public IgniteInternalFuture tryStartLocalSnapshotOperation( + @Nullable DiscoveryEvent discoveryEvent + ) throws IgniteCheckedException { + return null; + } + /** * @param initiatorNodeId Initiator node id. * @param snapshotOperation Snapshot operation. */ @Nullable public IgniteInternalFuture startLocalSnapshotOperation( UUID initiatorNodeId, - SnapshotOperation snapshotOperation + T snapshotOperation ) throws IgniteCheckedException { return null; } /** - * @param snapOp current snapshot operation. + * @param snapshotOperation current snapshot operation. * * @return {@code true} if next operation must be snapshot, {@code false} if checkpoint must be executed. */ public boolean onMarkCheckPointBegin( - SnapshotOperation snapOp, + T snapshotOperation, NavigableMap, T2> map ) throws IgniteCheckedException { return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java new file mode 100644 index 0000000000000..d88d96e1f6ed7 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; + +/** + * Initial snapshot discovery message with possibility to trigger exchange. + */ +public interface SnapshotDiscoveryMessage extends DiscoveryCustomMessage { + /** + * Is exchange needed after receiving this message. + * + * @return True if exchange is needed, false in other case. + */ + boolean needExchange(); +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperationType.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperation.java similarity index 53% rename from modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperationType.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperation.java index cc1aeea037bf8..6722eb6569783 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/snapshot/SnapshotOperationType.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotOperation.java @@ -15,35 +15,30 @@ * limitations under the License. * */ -package org.apache.ignite.internal.pagemem.snapshot; -import org.jetbrains.annotations.Nullable; +package org.apache.ignite.internal.processors.cache.persistence.snapshot; -/** */ -public enum SnapshotOperationType { - /** Create. */ - CREATE, - /** Restore. */ - RESTORE, - /** Restore 2. */ - RESTORE_2_PHASE, - /** Move. */ - MOVE, - /** Delete. */ - DELETE, - /** Check. */ - CHECK; - - /** Enumerated values. */ - private static final SnapshotOperationType[] VALS = values(); +import java.io.Serializable; +import java.util.Set; +/** + * Initial snapshot operation interface. + */ +public interface SnapshotOperation extends Serializable { /** - * Efficiently gets enumerated value from its ordinal. + * Cache group ids included to this snapshot. * - * @param ord Ordinal value. - * @return Enumerated value or {@code null} if ordinal out of range. + * @return Cache names. + */ + Set cacheGroupIds(); + + /** + * Cache names included to this snapshot. + */ + Set cacheNames(); + + /** + * Any custom extra parameter. */ - @Nullable public static SnapshotOperationType fromOrdinal(int ord) { - return ord >= 0 && ord < VALS.length ? VALS[ord] : null; - } + Object extraParameter(); } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index e34a7729771af..8c0f400e52899 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -337,12 +337,7 @@ org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager$1 org.apache.ignite.internal.marshaller.optimized.OptimizedFieldType org.apache.ignite.internal.mem.IgniteOutOfMemoryException org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl$Segment -org.apache.ignite.internal.pagemem.snapshot.FinishSnapshotOperationAckDiscoveryMessage org.apache.ignite.internal.pagemem.snapshot.SnapshotCheckParameters -org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation -org.apache.ignite.internal.pagemem.snapshot.SnapshotOperationType -org.apache.ignite.internal.pagemem.snapshot.StartSnapshotOperationAckDiscoveryMessage -org.apache.ignite.internal.pagemem.snapshot.StartSnapshotOperationDiscoveryMessage org.apache.ignite.internal.pagemem.wal.StorageException org.apache.ignite.internal.pagemem.wal.WALIterator org.apache.ignite.internal.pagemem.wal.record.TxRecord$TxAction diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java index 12200ae0a5c48..6a1d4f4668843 100644 --- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java +++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java @@ -33,13 +33,13 @@ import org.apache.ignite.internal.processors.cache.GridCacheMvccManager; import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; -import org.apache.ignite.internal.processors.cache.GridCacheTtlManager; -import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager; -import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheSnapshotManager; +import org.apache.ignite.internal.processors.cache.GridCacheTtlManager; import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager; import org.apache.ignite.internal.processors.cache.dr.GridOsCacheDrManager; import org.apache.ignite.internal.processors.cache.jta.CacheNoopJtaManager; +import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; import org.apache.ignite.internal.processors.cache.query.GridCacheLocalQueryManager; import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager; import org.apache.ignite.internal.processors.cache.store.CacheOsStoreManager; From a53544410dd15a3a5112d6de88648db21bd3fcf3 Mon Sep 17 00:00:00 2001 From: sboikov Date: Mon, 10 Jul 2017 11:38:04 +0300 Subject: [PATCH 112/155] ignite-5446 Alway use late affinity assignment mode --- .../configuration/IgniteConfiguration.java | 15 +- .../affinity/AffinityHistoryCleanupTest.java | 182 ------------------ ...AbstractCacheInterceptorRebalanceTest.java | 2 - .../cache/GridCacheDeploymentSelfTest.java | 7 +- ...gniteCacheP2pUnmarshallingTxErrorTest.java | 22 +-- ...inityAssignmentNodeJoinValidationTest.java | 46 +---- .../CacheLateAffinityAssignmentTest.java | 2 - ...cheLoadingConcurrentGridStartSelfTest.java | 2 - ...CachePartitionedPreloadEventsSelfTest.java | 143 -------------- ...teAffDisabledMultiNodeFullApiSelfTest.java | 35 ---- ...teAffDisabledMultiNodeFullApiSelfTest.java | 34 ---- .../db/IgnitePdsWholeClusterRestartTest.java | 1 - .../GridActivationPartitionedCacheSuit.java | 2 - ...ntinuousQueryFailoverAbstractSelfTest.java | 2 - .../processors/igfs/IgfsStreamsSelfTest.java | 1 - .../IgniteCacheFullApiSelfTestSuite.java | 6 - .../testsuites/IgniteCacheTestSuite2.java | 2 - 17 files changed, 17 insertions(+), 487 deletions(-) delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedPreloadEventsSelfTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicLateAffDisabledMultiNodeFullApiSelfTest.java delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java index ed05fa448dee4..cafa675b0951c 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java @@ -191,6 +191,7 @@ public class IgniteConfiguration { public static final boolean DFLT_CACHE_SANITY_CHECK_ENABLED = true; /** Default value for late affinity assignment flag. */ + @Deprecated public static final boolean DFLT_LATE_AFF_ASSIGNMENT = true; /** Default value for active on start flag. */ @@ -452,9 +453,6 @@ public class IgniteConfiguration { /** Custom executor configurations. */ private ExecutorConfiguration[] execCfgs; - /** */ - private boolean lateAffAssignment = DFLT_LATE_AFF_ASSIGNMENT; - /** Page memory configuration. */ private MemoryConfiguration memCfg; @@ -530,7 +528,6 @@ public IgniteConfiguration(IgniteConfiguration cfg) { igniteWorkDir = cfg.getWorkDirectory(); inclEvtTypes = cfg.getIncludeEventTypes(); includeProps = cfg.getIncludeProperties(); - lateAffAssignment = cfg.isLateAffinityAssignment(); lifecycleBeans = cfg.getLifecycleBeans(); locHost = cfg.getLocalHost(); log = cfg.getGridLogger(); @@ -2721,14 +2718,14 @@ public IgniteConfiguration setPlatformConfiguration(PlatformConfiguration platfo * from assignment calculated by {@link AffinityFunction#assignPartitions}. *

* This property should have the same value for all nodes in cluster. - *

- * If not provided, default value is {@link #DFLT_LATE_AFF_ASSIGNMENT}. * * @return Late affinity assignment flag. * @see AffinityFunction + * @deprecated Starting from Ignite 2.1 late affinity assignment is always enabled. */ + @Deprecated public boolean isLateAffinityAssignment() { - return lateAffAssignment; + return true; } /** @@ -2736,10 +2733,10 @@ public boolean isLateAffinityAssignment() { * * @param lateAffAssignment Late affinity assignment flag. * @return {@code this} for chaining. + * @deprecated Starting from Ignite 2.1 late affinity assignment is always enabled. */ + @Deprecated public IgniteConfiguration setLateAffinityAssignment(boolean lateAffAssignment) { - this.lateAffAssignment = lateAffAssignment; - return this; } diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityHistoryCleanupTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityHistoryCleanupTest.java index 87c2050df1645..605cc5f57c95b 100644 --- a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityHistoryCleanupTest.java +++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityHistoryCleanupTest.java @@ -47,9 +47,6 @@ public class AffinityHistoryCleanupTest extends GridCommonAbstractTest { /** */ private boolean client; - /** */ - private boolean lateAffAssignment; - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); @@ -71,8 +68,6 @@ public class AffinityHistoryCleanupTest extends GridCommonAbstractTest { cfg.setClientMode(client); - cfg.setLateAffinityAssignment(lateAffAssignment); - return cfg; } @@ -89,183 +84,6 @@ public class AffinityHistoryCleanupTest extends GridCommonAbstractTest { public void testAffinityHistoryCleanup() throws Exception { String histProp = System.getProperty(IgniteSystemProperties.IGNITE_AFFINITY_HISTORY_SIZE); - try { - System.setProperty(IgniteSystemProperties.IGNITE_AFFINITY_HISTORY_SIZE, "5"); - - Ignite ignite = startGrid(0); - - checkHistory(ignite, F.asList(topVer(1, 0)), 1); - - for (int i = 0; i < 3; i++) { - startGrid(1); - - stopGrid(1); - } - - checkHistory(ignite, F.asList( - topVer(3, 0), - topVer(4, 0), - topVer(5, 0), - topVer(6, 0), - topVer(7, 0)), - 5); - - client = true; - - startGrid(1); - - stopGrid(1); - - checkHistory(ignite, F.asList( - topVer(3, 0), - topVer(4, 0), - topVer(5, 0), - topVer(6, 0), - topVer(7, 0), - topVer(8, 0), - topVer(9, 0)), - 5); - - startGrid(1); - - stopGrid(1); - - checkHistory(ignite, F.asList( - topVer(3, 0), - topVer(4, 0), - topVer(5, 0), - topVer(6, 0), - topVer(7, 0), - topVer(8, 0), - topVer(9, 0), - topVer(10, 0), - topVer(11, 0)), - 5); - - startGrid(1); - - checkHistory(ignite, F.asList( - topVer(3, 0), - topVer(4, 0), - topVer(5, 0), - topVer(6, 0), - topVer(7, 0), - topVer(8, 0), - topVer(9, 0), - topVer(10, 0), - topVer(11, 0), - topVer(12, 0)), - 5); - - stopGrid(1); - - checkHistory(ignite, F.asList( - topVer(8, 0), - topVer(9, 0), - topVer(10, 0), - topVer(11, 0), - topVer(12, 0), - topVer(13, 0)), - 0); - - client = false; - - startGrid(1); - - stopGrid(1); - - checkHistory(ignite, F.asList( - topVer(8, 0), - topVer(9, 0), - topVer(10, 0), - topVer(11, 0), - topVer(12, 0), - topVer(13, 0), - topVer(14, 0), - topVer(15, 0)), - 2); - - startGrid(1); - - stopGrid(1); - - checkHistory(ignite, F.asList( - topVer(8, 0), - topVer(9, 0), - topVer(10, 0), - topVer(11, 0), - topVer(12, 0), - topVer(13, 0), - topVer(14, 0), - topVer(15, 0), - topVer(16, 0), - topVer(17, 0)), - 4); - - startGrid(1); - - checkHistory(ignite, F.asList( - topVer(13, 0), - topVer(14, 0), - topVer(15, 0), - topVer(16, 0), - topVer(17, 0), - topVer(18, 0)), - 5); - - stopGrid(1); - - checkHistory(ignite, F.asList( - topVer(14, 0), - topVer(15, 0), - topVer(16, 0), - topVer(17, 0), - topVer(18, 0), - topVer(19, 0)), - 6); - - startGrid(1); - - checkHistory(ignite, F.asList( - topVer(16, 0), - topVer(17, 0), - topVer(18, 0), - topVer(19, 0), - topVer(20, 0)), - 5); - - client = true; - - startGrid(2); - - stopGrid(2); - - checkHistory(ignite, F.asList( - topVer(16, 0), - topVer(17, 0), - topVer(18, 0), - topVer(19, 0), - topVer(20, 0), - topVer(21, 0), - topVer(22, 0)), - 5); - } - finally { - if (histProp != null) - System.setProperty(IgniteSystemProperties.IGNITE_AFFINITY_HISTORY_SIZE, histProp); - else - System.clearProperty(IgniteSystemProperties.IGNITE_AFFINITY_HISTORY_SIZE); - } - } - - /** - * @throws Exception If failed. - */ - public void testAffinityHistoryCleanupLateAffinityAssignment() throws Exception { - lateAffAssignment = true; - - String histProp = System.getProperty(IgniteSystemProperties.IGNITE_AFFINITY_HISTORY_SIZE); - try { System.setProperty(IgniteSystemProperties.IGNITE_AFFINITY_HISTORY_SIZE, "5"); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java index adfe085052777..99cf1f1d2438f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridAbstractCacheInterceptorRebalanceTest.java @@ -75,8 +75,6 @@ public abstract class GridAbstractCacheInterceptorRebalanceTest extends GridComm @Override protected IgniteConfiguration getConfiguration(final String igniteInstanceName) throws Exception { final IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - cfg.setLateAffinityAssignment(true); - final CacheConfiguration ccfg = new CacheConfiguration<>(CACHE_NAME); assertNotNull(interceptor); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentSelfTest.java index c88d0cc58ad0e..ff3ab362fe726 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentSelfTest.java @@ -92,8 +92,6 @@ public class GridCacheDeploymentSelfTest extends GridCommonAbstractTest { cfg.setConnectorConfiguration(null); - cfg.setLateAffinityAssignment(false); - return cfg; } @@ -510,7 +508,10 @@ private void testCacheUndeployment(DeploymentMode depMode) throws Exception { * @return Key with described properties. * @throws IllegalStateException if such a key could not be found after 10000 iterations. */ - private int getNextKey(int start, Ignite g, ClusterNode primary, ClusterNode backup, ClusterNode near) { + private int getNextKey(int start, Ignite g, ClusterNode primary, ClusterNode backup, ClusterNode near) + throws Exception { + awaitPartitionMapExchange(); + info("Primary: " + primary); info("Backup: " + backup); info("Near: " + near); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingTxErrorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingTxErrorTest.java index 1a88d809f1b78..ba77c701dd54e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingTxErrorTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingTxErrorTest.java @@ -22,7 +22,6 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.CacheAtomicityMode; -import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.transactions.Transaction; @@ -39,26 +38,13 @@ public class IgniteCacheP2pUnmarshallingTxErrorTest extends IgniteCacheP2pUnmars return CacheAtomicityMode.TRANSACTIONAL; } - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - cfg.setLateAffinityAssignment(false); - - if (!igniteInstanceName.endsWith("0")) - cfg.getCacheConfiguration()[0].setRebalanceDelay(-1); // Allows to check GridDhtLockRequest fail. - - return cfg; - } - /** * Sends put with optimistic lock and handles fail. */ - protected void failOptimistic() { + private void failOptimistic() { IgniteCache cache = jcache(0); try (Transaction tx = grid(0).transactions().txStart(OPTIMISTIC, REPEATABLE_READ)) { - cache.put(new TestKey(String.valueOf(++key)), ""); tx.commit(); @@ -75,12 +61,10 @@ protected void failOptimistic() { /** * Sends put with pessimistic lock and handles fail. */ - protected void failPessimictic() { + private void failPessimictic() { IgniteCache cache = jcache(0); - try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, - REPEATABLE_READ)) { - + try (Transaction tx = grid(0).transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) { cache.put(new TestKey(String.valueOf(++key)), ""); assert false : "p2p marshalling failed, but error response was not sent"; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentNodeJoinValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentNodeJoinValidationTest.java index 11ac063514a00..48b33b6c20b7e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentNodeJoinValidationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentNodeJoinValidationTest.java @@ -19,8 +19,6 @@ import org.apache.ignite.Ignite; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; @@ -77,7 +75,8 @@ public void testJoinValidation2() throws Exception { * @param firstEnabled Flag value for first started node. * @throws Exception If failed. */ - public void checkNodeJoinValidation(boolean firstEnabled) throws Exception { + private void checkNodeJoinValidation(boolean firstEnabled) throws Exception { + // LateAffinity should be always enabled, setLateAffinityAssignment should be ignored. lateAff = firstEnabled; Ignite ignite = startGrid(0); @@ -86,49 +85,12 @@ public void checkNodeJoinValidation(boolean firstEnabled) throws Exception { lateAff = !firstEnabled; - try { - startGrid(1); - - fail(); - } - catch (Exception e) { - checkError(e); - } - - client = true; - - try { - startGrid(1); - - fail(); - } - catch (Exception e) { - checkError(e); - } - - assertEquals(1, ignite.cluster().nodes().size()); - - lateAff = firstEnabled; - - client = false; - startGrid(1); client = true; - Ignite client = startGrid(2); - - assertTrue(client.configuration().isClientMode()); - } - - /** - * @param e Error. - */ - private void checkError(Exception e) { - IgniteSpiException err = X.cause(e, IgniteSpiException.class); + startGrid(2); - assertNotNull(err); - assertTrue(err.getMessage().contains("Local node's cache affinity assignment mode differs " + - "from the same property on remote node")); + assertEquals(3, ignite.cluster().nodes().size()); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java index 46520caaa97ec..6174209d74792 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java @@ -142,8 +142,6 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - cfg.setLateAffinityAssignment(true); - TestRecordingCommunicationSpi commSpi; if (spiC != null) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java index 4f1b090cf1cba..68e88ce298530 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java @@ -79,8 +79,6 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - cfg.setLateAffinityAssignment(true); - ((TcpCommunicationSpi)cfg.getCommunicationSpi()).setSharedMemoryPort(-1); CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedPreloadEventsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedPreloadEventsSelfTest.java deleted file mode 100644 index bc62a722d3b2e..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedPreloadEventsSelfTest.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.distributed.dht; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.affinity.AffinityFunction; -import org.apache.ignite.cache.affinity.AffinityFunctionContext; -import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.events.Event; -import org.apache.ignite.internal.processors.cache.distributed.GridCachePreloadEventsAbstractSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysFuture; -import org.apache.ignite.internal.util.typedef.F; - -import static org.apache.ignite.cache.CacheMode.PARTITIONED; -import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_OBJECT_LOADED; - -/** - * - */ -public class GridCachePartitionedPreloadEventsSelfTest extends GridCachePreloadEventsAbstractSelfTest { - /** */ - private boolean replicatedAffinity = true; - - /** */ - private long rebalanceDelay; - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - // 'testForcePreload' is not valid with late assignment. - cfg.setLateAffinityAssignment(false); - - return cfg; - } - - /** {@inheritDoc} */ - @Override protected CacheConfiguration cacheConfiguration() { - CacheConfiguration cacheCfg = super.cacheConfiguration(); - - if (replicatedAffinity) - // replicate entries to all nodes - cacheCfg.setAffinity(notSerializableProxy(new AffinityFunction() { - /** {@inheritDoc} */ - @Override public void reset() { - } - - /** {@inheritDoc} */ - @Override public int partitions() { - return 1; - } - - /** {@inheritDoc} */ - @Override public int partition(Object key) { - return 0; - } - - /** {@inheritDoc} */ - @Override public List> assignPartitions(AffinityFunctionContext affCtx) { - List nodes = new ArrayList<>(affCtx.currentTopologySnapshot()); - - return Collections.singletonList(nodes); - } - - /** {@inheritDoc} */ - @Override public void removeNode(UUID nodeId) { - } - }, AffinityFunction.class)); - - cacheCfg.setRebalanceDelay(rebalanceDelay); - - return cacheCfg; - } - - /** {@inheritDoc} */ - @Override protected CacheMode getCacheMode() { - return PARTITIONED; - } - - /** - * Test events fired from - * {@link GridDhtForceKeysFuture} - * - * @throws Exception if failed. - */ - public void testForcePreload() throws Exception { - replicatedAffinity = false; - rebalanceDelay = -1; - - Ignite g1 = startGrid("g1"); - - Collection keys = new HashSet<>(); - - IgniteCache cache = g1.cache(DEFAULT_CACHE_NAME); - - for (int i = 0; i < 100; i++) { - keys.add(i); - cache.put(i, "val"); - } - - Ignite g2 = startGrid("g2"); - - Map> keysMap = g1.affinity(DEFAULT_CACHE_NAME).mapKeysToNodes(keys); - Collection g2Keys = keysMap.get(g2.cluster().localNode()); - - assertNotNull(g2Keys); - assertFalse("There are no keys assigned to g2", g2Keys.isEmpty()); - - for (Object key : g2Keys) - // Need to force keys loading. - assertEquals("val", g2.cache(DEFAULT_CACHE_NAME).getAndPut(key, "changed val")); - - Collection evts = g2.events().localQuery(F.alwaysTrue(), EVT_CACHE_REBALANCE_OBJECT_LOADED); - - checkPreloadEvents(evts, g2, g2Keys); - } -} \ No newline at end of file diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicLateAffDisabledMultiNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicLateAffDisabledMultiNodeFullApiSelfTest.java deleted file mode 100644 index 267de66eee233..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheAtomicLateAffDisabledMultiNodeFullApiSelfTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.distributed.near; - -import org.apache.ignite.configuration.IgniteConfiguration; - -/** - * - */ -public class GridCacheAtomicLateAffDisabledMultiNodeFullApiSelfTest extends - GridCacheAtomicMultiNodeFullApiSelfTest { - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - cfg.setLateAffinityAssignment(false); - - return cfg; - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest.java deleted file mode 100644 index 025b68bc4b147..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.distributed.near; - -import org.apache.ignite.configuration.IgniteConfiguration; - -/** - * - */ -public class GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest extends GridCachePartitionedMultiNodeFullApiSelfTest { - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - cfg.setLateAffinityAssignment(false); - - return cfg; - } -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java index b512a64db399b..c8ec304e2a5cf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java @@ -76,7 +76,6 @@ public class IgnitePdsWholeClusterRestartTest extends GridCommonAbstractTest { ccfg1.setAffinity(new RendezvousAffinityFunction(false, 32)); ccfg1.setBackups(2); - cfg.setLateAffinityAssignment(false); cfg.setActiveOnStart(false); // To avoid hostname lookup on start. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationPartitionedCacheSuit.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationPartitionedCacheSuit.java index c74aadac30a7f..303725f351ea0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationPartitionedCacheSuit.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationPartitionedCacheSuit.java @@ -28,7 +28,6 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedClientOnlyNoPrimaryFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedCopyOnReadDisabledMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedFullApiSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeP2PDisabledFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNearOnlyNoPrimaryFullApiSelfTest; @@ -45,7 +44,6 @@ public class GridActivationPartitionedCacheSuit extends GridActivationCacheAbstr addTest(GridCachePartitionedClientOnlyNoPrimaryFullApiSelfTest.class); addTest(GridCachePartitionedCopyOnReadDisabledMultiNodeFullApiSelfTest.class); addTest(GridCachePartitionedFullApiSelfTest.class); - addTest(GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest.class); addTest(GridCachePartitionedMultiNodeFullApiSelfTest.class); addTest(GridCachePartitionedMultiNodeP2PDisabledFullApiSelfTest.class); addTest(GridCachePartitionedNearDisabledFullApiSelfTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java index 937a0593a2f0d..43069cd54d89a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java @@ -134,8 +134,6 @@ public abstract class CacheContinuousQueryFailoverAbstractSelfTest extends GridC @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - cfg.setLateAffinityAssignment(true); - ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(true); ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java index d77296aaca036..e8116045764b5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java @@ -116,7 +116,6 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - cfg.setLateAffinityAssignment(false); cfg.setCacheConfiguration(); TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java index 11a4a107e86d2..164ff6a551a7b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java @@ -36,7 +36,6 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicClientOnlyMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicClientOnlyMultiNodeP2PDisabledFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicCopyOnReadDisabledMultiNodeFullApiSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicLateAffDisabledMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeP2PDisabledFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeWithGroupFullApiSelfTest; @@ -57,7 +56,6 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedFilteredPutSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedFullApiMultithreadedSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedFullApiSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeCounterSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeFullApiSelfTest; import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeP2PDisabledFullApiSelfTest; @@ -155,10 +153,6 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(GridCachePartitionedNearDisabledAtomicOnheapMultiNodeFullApiSelfTest.class); suite.addTestSuite(GridCacheAtomicOnheapMultiNodeFullApiSelfTest.class); - // Old affinity assignment mode. - suite.addTestSuite(GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest.class); - suite.addTestSuite(GridCacheAtomicLateAffDisabledMultiNodeFullApiSelfTest.class); - // Multithreaded. suite.addTestSuite(GridCacheLocalFullApiMultithreadedSelfTest.class); suite.addTestSuite(GridCacheReplicatedFullApiMultithreadedSelfTest.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java index 4c9accf0e3095..9ed7ee3780f5e 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java @@ -78,7 +78,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadStartStopSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadUnloadSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledLockSelfTest; -import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedPreloadEventsSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedTopologyChangeSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedUnloadEventsSelfTest; import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheClearDuringRebalanceTest; @@ -233,7 +232,6 @@ public static TestSuite suite() throws Exception { suite.addTest(new TestSuite(GridCacheAtomicNearEvictionEventSelfTest.class)); suite.addTest(new TestSuite(GridCachePartitionedTopologyChangeSelfTest.class)); - suite.addTest(new TestSuite(GridCachePartitionedPreloadEventsSelfTest.class)); suite.addTest(new TestSuite(GridCachePartitionedUnloadEventsSelfTest.class)); suite.addTest(new TestSuite(GridCacheColocatedOptimisticTransactionSelfTest.class)); suite.addTestSuite(GridCacheAtomicMessageCountSelfTest.class); From 17694c16d449dea4d4e39a271dd39e838c581858 Mon Sep 17 00:00:00 2001 From: Ilya Lantukh Date: Mon, 10 Jul 2017 11:59:19 +0300 Subject: [PATCH 113/155] IGNITE-5694 - Added retries for getCollection in DataStructuresProcessor - Fixes #2240. Signed-off-by: Alexey Goncharuk --- .../processors/datastructures/DataStructuresProcessor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java index 5564b796c7db4..33a2fd2133362 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java @@ -1031,7 +1031,11 @@ else if (!(oldVal instanceof DistributedCollectionMetadata)) } } - return c.applyx(cache.context()); + return retryTopologySafe(new IgniteOutClosureX() { + @Override public T applyx() throws IgniteCheckedException { + return c.applyx(cache.context()); + } + }); } /** From 0b9527e851fd7c1d63bfc427721ca8af80d54b0c Mon Sep 17 00:00:00 2001 From: sboikov Date: Mon, 10 Jul 2017 12:02:03 +0300 Subject: [PATCH 114/155] Fixed NPE in setOwners. --- .../cache/distributed/dht/GridDhtPartitionTopologyImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index 2f548108f5a7e..7c62cca59c4c1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -1722,7 +1722,7 @@ else if (plc != PartitionLossPolicy.IGNORE) { } U.warn(log, "Partition has been scheduled for rebalancing due to outdated update counter " + - "[nodeId=" + ctx.localNodeId() + "cacheOrGroupName=" + grp.cacheOrGroupName() + + "[nodeId=" + ctx.localNodeId() + ", cacheOrGroupName=" + grp.cacheOrGroupName() + ", partId=" + locPart.id() + ", haveHistory=" + haveHistory + "]"); } @@ -1743,8 +1743,8 @@ else if (plc != PartitionLossPolicy.IGNORE) { } U.warn(log, "Partition has been scheduled for rebalancing due to outdated update counter " + - "[nodeId=" + ctx.localNodeId() + "cacheOrGroupName=" + grp.cacheOrGroupName() + - ", partId=" + locPart.id() + ", haveHistory=" + haveHistory + "]"); + "[nodeId=" + ctx.localNodeId() + ", cacheOrGroupName=" + grp.cacheOrGroupName() + + ", partId=" + p + ", haveHistory=" + haveHistory + "]"); } if (updateSeq) From b085fa0db67f8cda01e6927715241b10ac43c0f2 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Mon, 10 Jul 2017 12:24:54 +0300 Subject: [PATCH 115/155] IGNITE-5491 .NET: PersistentStoreMetrics --- .../cluster/PlatformClusterGroup.java | 39 ++++- .../utils/PlatformConfigurationUtils.java | 8 +- .../Cache/PersistentStoreTest.cs | 9 +- .../IgniteConfigurationSerializerTest.cs | 11 +- .../IgniteConfigurationTest.cs | 11 +- .../Apache.Ignite.Core.csproj | 2 + .../dotnet/Apache.Ignite.Core/IIgnite.cs | 9 + .../IgniteConfigurationSection.xsd | 15 ++ .../Impl/Cluster/ClusterGroupImpl.cs | 14 ++ .../dotnet/Apache.Ignite.Core/Impl/Ignite.cs | 7 + .../PersistentStore/PersistentStoreMetrics.cs | 88 ++++++++++ .../IPersistentStoreMetrics.cs | 87 ++++++++++ .../PersistentStoreConfiguration.cs | 156 +++++++++++------- 13 files changed, 392 insertions(+), 64 deletions(-) create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Impl/PersistentStore/PersistentStoreMetrics.cs create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/IPersistentStoreMetrics.cs diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java index 3e3aa3afbdf11..f6e3d2e4722bb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java @@ -25,6 +25,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteCluster; import org.apache.ignite.MemoryMetrics; +import org.apache.ignite.PersistenceMetrics; import org.apache.ignite.binary.BinaryRawWriter; import org.apache.ignite.cluster.ClusterMetrics; import org.apache.ignite.cluster.ClusterNode; @@ -122,6 +123,9 @@ public class PlatformClusterGroup extends PlatformAbstractTarget { /** */ private static final int OP_IS_ACTIVE = 29; + /** */ + private static final int OP_PERSISTENT_STORE_METRICS = 30; + /** Projection. */ private final ClusterGroupEx prj; @@ -146,7 +150,7 @@ public PlatformClusterGroup(PlatformContext platformCtx, ClusterGroupEx prj) { break; - case OP_MEMORY_METRICS: + case OP_MEMORY_METRICS: { Collection metrics = prj.ignite().memoryMetrics(); writer.writeInt(metrics.size()); @@ -156,6 +160,15 @@ public PlatformClusterGroup(PlatformContext platformCtx, ClusterGroupEx prj) { } break; + } + + case OP_PERSISTENT_STORE_METRICS: { + PersistenceMetrics metrics = prj.ignite().persistentStoreMetrics(); + + writePersistentStoreMetrics(writer, metrics); + + break; + } default: super.processOutStream(type, writer); @@ -448,4 +461,28 @@ private static void writeMemoryMetrics(BinaryRawWriter writer, MemoryMetrics met writer.writeFloat(metrics.getLargeEntriesPagesPercentage()); writer.writeFloat(metrics.getPagesFillFactor()); } + + /** + * Writes persistent store metrics. + * + * @param writer Writer. + * @param metrics Metrics + */ + private void writePersistentStoreMetrics(BinaryRawWriter writer, PersistenceMetrics metrics) { + assert writer != null; + assert metrics != null; + + writer.writeFloat(metrics.getWalLoggingRate()); + writer.writeFloat(metrics.getWalWritingRate()); + writer.writeInt(metrics.getWalArchiveSegments()); + writer.writeFloat(metrics.getWalFsyncTimeAverage()); + writer.writeLong(metrics.getLastCheckpointingDuration()); + writer.writeLong(metrics.getLastCheckpointLockWaitDuration()); + writer.writeLong(metrics.getLastCheckpointMarkDuration()); + writer.writeLong(metrics.getLastCheckpointPagesWriteDuration()); + writer.writeLong(metrics.getLastCheckpointFsyncDuration()); + writer.writeLong(metrics.getLastCheckpointTotalPagesNumber()); + writer.writeLong(metrics.getLastCheckpointDataPagesNumber()); + writer.writeLong(metrics.getLastCheckpointCopiedOnWritePagesNumber()); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java index 92db41acd121a..d513071b09d8c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java @@ -1505,7 +1505,10 @@ private static PersistentStoreConfiguration readPersistentStoreConfiguration(Bin .setWalFlushFrequency((int) in.readLong()) .setWalFsyncDelay(in.readInt()) .setWalRecordIteratorBufferSize(in.readInt()) - .setAlwaysWriteFullPages(in.readBoolean()); + .setAlwaysWriteFullPages(in.readBoolean()) + .setMetricsEnabled(in.readBoolean()) + .setSubIntervals(in.readInt()) + .setRateTimeInterval(in.readLong()); } /** @@ -1535,6 +1538,9 @@ private static void writePersistentStoreConfiguration(BinaryRawWriter w, Persist w.writeInt(cfg.getWalFsyncDelay()); w.writeInt(cfg.getWalRecordIteratorBufferSize()); w.writeBoolean(cfg.isAlwaysWriteFullPages()); + w.writeBoolean(cfg.isMetricsEnabled()); + w.writeInt(cfg.getSubIntervals()); + w.writeLong(cfg.getRateTimeInterval()); } else { w.writeBoolean(false); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs index e9cbce8924a1a..3320dd745c2f1 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs @@ -58,7 +58,8 @@ public void TestCacheDataSurvivesNodeRestart() { PersistentStorePath = Path.Combine(_tempDir, "Store"), WalStorePath = Path.Combine(_tempDir, "WalStore"), - WalArchivePath = Path.Combine(_tempDir, "WalArchive") + WalArchivePath = Path.Combine(_tempDir, "WalArchive"), + MetricsEnabled = true } }; @@ -72,6 +73,12 @@ public void TestCacheDataSurvivesNodeRestart() var cache = ignite.CreateCache(cacheName); cache[1] = 1; + + // Check some metrics. + var metrics = ignite.GetPersistentStoreMetrics(); + Assert.Greater(metrics.WalLoggingRate, 0); + Assert.Greater(metrics.WalWritingRate, 0); + Assert.Greater(metrics.WalFsyncTimeAverage, 0); } // Verify directories. diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs index 3d0bb56d7b4fd..7e0d94104541e 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs @@ -143,7 +143,7 @@ public void TestPredefinedXml() - + "; var cfg = IgniteConfiguration.FromXml(xml); @@ -315,7 +315,9 @@ public void TestPredefinedXml() Assert.AreEqual(10, pers.WalSegments); Assert.AreEqual(11, pers.WalSegmentSize); Assert.AreEqual("baz", pers.WalStorePath); - + Assert.IsTrue(pers.MetricsEnabled); + Assert.AreEqual(3, pers.SubIntervals); + Assert.AreEqual(TimeSpan.FromSeconds(6), pers.RateTimeInterval); } /// @@ -907,7 +909,10 @@ private static IgniteConfiguration GetTestConfig() WalRecordIteratorBufferSize = 32 * 1024 * 1024, WalSegments = 6, WalSegmentSize = 5 * 1024 * 1024, - WalStorePath = Path.GetTempPath() + WalStorePath = Path.GetTempPath(), + SubIntervals = 25, + MetricsEnabled = true, + RateTimeInterval = TimeSpan.FromDays(1) }, IsActiveOnStart = false }; diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs index 5e5cb1c610e3b..4902118eec2df 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs @@ -266,6 +266,9 @@ public void TestAllConfigurationProperties() Assert.AreEqual(pers.WalSegments, resPers.WalSegments); Assert.AreEqual(pers.WalSegmentSize, resPers.WalSegmentSize); Assert.AreEqual(pers.WalStorePath, resPers.WalStorePath); + Assert.AreEqual(pers.MetricsEnabled, resPers.MetricsEnabled); + Assert.AreEqual(pers.RateTimeInterval, resPers.RateTimeInterval); + Assert.AreEqual(pers.SubIntervals, resPers.SubIntervals); } } @@ -539,6 +542,9 @@ private static void CheckDefaultProperties(PersistentStoreConfiguration cfg) Assert.AreEqual(PersistentStoreConfiguration.DefaultWalSegmentSize, cfg.WalSegmentSize); Assert.AreEqual(PersistentStoreConfiguration.DefaultWalSegments, cfg.WalSegments); Assert.AreEqual(WalMode.Default, cfg.WalMode); + Assert.IsFalse(cfg.MetricsEnabled); + Assert.AreEqual(PersistentStoreConfiguration.DefaultSubIntervals, cfg.SubIntervals); + Assert.AreEqual(PersistentStoreConfiguration.DefaultRateTimeInterval, cfg.RateTimeInterval); } /// @@ -741,7 +747,10 @@ private static IgniteConfiguration GetCustomConfig() WalRecordIteratorBufferSize = 32 * 1024 * 1024, WalSegments = 6, WalSegmentSize = 5 * 1024 * 1024, - WalStorePath = Path.GetTempPath() + WalStorePath = Path.GetTempPath(), + MetricsEnabled = true, + SubIntervals = 7, + RateTimeInterval = TimeSpan.FromSeconds(9) } }; } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj index 9df28890687bc..76132c3423f69 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj @@ -99,6 +99,8 @@ + + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IIgnite.cs b/modules/platforms/dotnet/Apache.Ignite.Core/IIgnite.cs index 8c4bee28c7dd6..bf061db0705af 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IIgnite.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IIgnite.cs @@ -32,6 +32,7 @@ namespace Apache.Ignite.Core using Apache.Ignite.Core.Log; using Apache.Ignite.Core.Lifecycle; using Apache.Ignite.Core.Messaging; + using Apache.Ignite.Core.PersistentStore; using Apache.Ignite.Core.Plugin; using Apache.Ignite.Core.Services; using Apache.Ignite.Core.Transactions; @@ -359,5 +360,13 @@ public interface IIgnite : IDisposable /// true if the grid is active; otherwise, false. /// bool IsActive(); + + /// + /// Gets the persistent store metrics. + /// + /// To enable metrics set property + /// in . + /// + IPersistentStoreMetrics GetPersistentStoreMetrics(); } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd index 919f3030e5dd7..d7fd5acade02a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd @@ -1394,6 +1394,21 @@ Whether full pages should always be written. + + + Enable persistent store metrics. + + + + + Number of sub intervals to split RateTimeInterval into. + + + + + Rate time interval. + + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs index 37b4e79c3b32a..6e07b7819b67b 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cluster/ClusterGroupImpl.cs @@ -35,9 +35,11 @@ namespace Apache.Ignite.Core.Impl.Cluster using Apache.Ignite.Core.Impl.Compute; using Apache.Ignite.Core.Impl.Events; using Apache.Ignite.Core.Impl.Messaging; + using Apache.Ignite.Core.Impl.PersistentStore; using Apache.Ignite.Core.Impl.Services; using Apache.Ignite.Core.Impl.Unmanaged; using Apache.Ignite.Core.Messaging; + using Apache.Ignite.Core.PersistentStore; using Apache.Ignite.Core.Services; using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; @@ -130,6 +132,9 @@ internal class ClusterGroupImpl : PlatformTarget, IClusterGroup /** */ private const int OpIsActive = 29; + /** */ + private const int OpGetPersistentStoreMetrics = 30; + /** Initial Ignite instance. */ private readonly Ignite _ignite; @@ -614,6 +619,15 @@ public bool IsActive() return DoOutInOp(OpIsActive) == True; } + /// + /// Gets the persistent store metrics. + /// + public IPersistentStoreMetrics GetPersistentStoreMetrics() + { + return DoInOp(OpGetPersistentStoreMetrics, stream => + new PersistentStoreMetrics(Marshaller.StartUnmarshal(stream, false))); + } + /// /// Creates new Cluster Group from given native projection. /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs index fc7894abfadda..205f6e2ba3440 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Ignite.cs @@ -45,6 +45,7 @@ namespace Apache.Ignite.Core.Impl using Apache.Ignite.Core.Lifecycle; using Apache.Ignite.Core.Log; using Apache.Ignite.Core.Messaging; + using Apache.Ignite.Core.PersistentStore; using Apache.Ignite.Core.Services; using Apache.Ignite.Core.Transactions; using UU = Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils; @@ -757,6 +758,12 @@ public bool IsActive() return _prj.IsActive(); } + /** */ + public IPersistentStoreMetrics GetPersistentStoreMetrics() + { + return _prj.GetPersistentStoreMetrics(); + } + /// /// Gets or creates near cache. /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PersistentStore/PersistentStoreMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PersistentStore/PersistentStoreMetrics.cs new file mode 100644 index 0000000000000..85a4fdfc3ed8d --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/PersistentStore/PersistentStoreMetrics.cs @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Impl.PersistentStore +{ + using System; + using System.Diagnostics; + using Apache.Ignite.Core.Binary; + using Apache.Ignite.Core.Impl.Binary; + using Apache.Ignite.Core.PersistentStore; + + /// + /// Persistent store metrics. + /// + internal class PersistentStoreMetrics : IPersistentStoreMetrics + { + /// + /// Initializes a new instance of the class. + /// + public PersistentStoreMetrics(IBinaryRawReader reader) + { + Debug.Assert(reader != null); + + WalLoggingRate = reader.ReadFloat(); + WalWritingRate = reader.ReadFloat(); + WalArchiveSegments = reader.ReadInt(); + WalFsyncTimeAverage = reader.ReadFloat(); + LastCheckpointingDuration = reader.ReadLongAsTimespan(); + LastCheckpointLockWaitDuration = reader.ReadLongAsTimespan(); + LastCheckpointMarkDuration = reader.ReadLongAsTimespan(); + LastCheckpointPagesWriteDuration = reader.ReadLongAsTimespan(); + LastCheckpointFsyncDuration = reader.ReadLongAsTimespan(); + LastCheckpointTotalPagesNumber = reader.ReadLong(); + LastCheckpointDataPagesNumber = reader.ReadLong(); + LastCheckpointCopiedOnWritePagesNumber = reader.ReadLong(); + } + + /** */ + public float WalLoggingRate { get; private set; } + + /** */ + public float WalWritingRate { get; private set; } + + /** */ + public int WalArchiveSegments { get; private set; } + + /** */ + public float WalFsyncTimeAverage { get; private set; } + + /** */ + public TimeSpan LastCheckpointingDuration { get; private set; } + + /** */ + public TimeSpan LastCheckpointLockWaitDuration { get; private set; } + + /** */ + public TimeSpan LastCheckpointMarkDuration { get; private set; } + + /** */ + public TimeSpan LastCheckpointPagesWriteDuration { get; private set; } + + /** */ + public TimeSpan LastCheckpointFsyncDuration { get; private set; } + + /** */ + public long LastCheckpointTotalPagesNumber { get; private set; } + + /** */ + public long LastCheckpointDataPagesNumber { get; private set; } + + /** */ + public long LastCheckpointCopiedOnWritePagesNumber { get; private set; } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/IPersistentStoreMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/IPersistentStoreMetrics.cs new file mode 100644 index 0000000000000..e7e8481d6bf3e --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/IPersistentStoreMetrics.cs @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.PersistentStore +{ + using System; + + /// + /// Persistent store metrics. + /// + public interface IPersistentStoreMetrics + { + /// + /// Gets the average number of WAL records per second written during the last time interval. + /// + float WalLoggingRate { get; } + + /// + /// Gets the average number of bytes per second written during the last time interval. + /// + float WalWritingRate { get; } + + /// + /// Gets the current number of WAL segments in the WAL archive. + /// + int WalArchiveSegments { get; } + + /// + /// Gets the average WAL fsync duration in microseconds over the last time interval. + /// + float WalFsyncTimeAverage { get; } + + /// + /// Gets the duration of the last checkpoint. + /// + TimeSpan LastCheckpointingDuration { get; } + + /// + /// Gets the duration of last checkpoint lock wait. + /// + TimeSpan LastCheckpointLockWaitDuration { get; } + + /// + /// Gets the duration of last checkpoint mark phase. + /// + TimeSpan LastCheckpointMarkDuration { get; } + + /// + /// Gets the duration of last checkpoint pages write phase. + /// + TimeSpan LastCheckpointPagesWriteDuration { get; } + + /// + /// Gets the duration of the sync phase of the last checkpoint. + /// + TimeSpan LastCheckpointFsyncDuration { get; } + + /// + /// Gets the total number of pages written during the last checkpoint. + /// + long LastCheckpointTotalPagesNumber { get; } + + /// + /// Gets the number of data pages written during the last checkpoint. + /// + long LastCheckpointDataPagesNumber { get; } + + /// + /// Gets the number of pages copied to a temporary checkpoint buffer during the last checkpoint. + /// + long LastCheckpointCopiedOnWritePagesNumber { get; } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs index c998ab3242426..43b17ac7effc2 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs @@ -20,6 +20,7 @@ namespace Apache.Ignite.Core.PersistentStore using System; using System.ComponentModel; using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; using Apache.Ignite.Core.Binary; using Apache.Ignite.Core.Impl.Binary; @@ -28,6 +29,73 @@ namespace Apache.Ignite.Core.PersistentStore /// public class PersistentStoreConfiguration { + /// + /// Default value for . + /// + public const long DefaultCheckpointingPageBufferSize = 256L * 1024 * 1024; + + /// + /// Default value for . + /// + public const int DefaultCheckpointingThreads = 1; + + /// + /// Default value for . + /// + public static readonly TimeSpan DefaultCheckpointingFrequency = TimeSpan.FromSeconds(180); + + /// + /// Default value for . + /// + public static readonly TimeSpan DefaultLockWaitTime = TimeSpan.FromSeconds(10); + + /// + /// Default value for . + /// + public const int DefaultWalHistorySize = 20; + + /// + /// Default value for . + /// + public const int DefaultWalSegments = 10; + + /// + /// Default value for . + /// + public const int DefaultWalSegmentSize = 64 * 1024 * 1024; + + /// + /// Default value for . + /// + public const int DefaultTlbSize = 128 * 1024; + + /// + /// Default value for . + /// + public static readonly TimeSpan DefaultWalFlushFrequency = TimeSpan.FromSeconds(2); + + /// + /// Default value for . + /// + public const int DefaultWalRecordIteratorBufferSize = 64 * 1024 * 1024; + + /// + /// Default value for . + /// + public const int DefaultWalFsyncDelayNanos = 1; + + /// + /// The default sub intervals. + /// + [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", + Justification = "Consistency with Java config")] + public const int DefaultSubIntervals = 5; + + /// + /// The default rate time interval. + /// + public static readonly TimeSpan DefaultRateTimeInterval = TimeSpan.FromSeconds(60); + /// /// Initializes a new instance of the class. /// @@ -44,6 +112,8 @@ public PersistentStoreConfiguration() WalFlushFrequency = DefaultWalFlushFrequency; WalRecordIteratorBufferSize = DefaultWalRecordIteratorBufferSize; WalFsyncDelayNanos = DefaultWalFsyncDelayNanos; + RateTimeInterval = DefaultRateTimeInterval; + SubIntervals = DefaultSubIntervals; } /// @@ -64,12 +134,15 @@ internal PersistentStoreConfiguration(IBinaryRawReader reader) WalSegmentSize = reader.ReadInt(); WalStorePath = reader.ReadString(); WalArchivePath = reader.ReadString(); - WalMode = (WalMode) reader.ReadInt(); + WalMode = (WalMode)reader.ReadInt(); TlbSize = reader.ReadInt(); WalFlushFrequency = reader.ReadLongAsTimespan(); WalFsyncDelayNanos = reader.ReadInt(); WalRecordIteratorBufferSize = reader.ReadInt(); AlwaysWriteFullPages = reader.ReadBoolean(); + MetricsEnabled = reader.ReadBoolean(); + SubIntervals = reader.ReadInt(); + RateTimeInterval = reader.ReadLongAsTimespan(); } /// @@ -90,69 +163,17 @@ internal void Write(IBinaryRawWriter writer) writer.WriteInt(WalSegmentSize); writer.WriteString(WalStorePath); writer.WriteString(WalArchivePath); - writer.WriteInt((int) WalMode); + writer.WriteInt((int)WalMode); writer.WriteInt(TlbSize); writer.WriteTimeSpanAsLong(WalFlushFrequency); writer.WriteInt(WalFsyncDelayNanos); writer.WriteInt(WalRecordIteratorBufferSize); writer.WriteBoolean(AlwaysWriteFullPages); + writer.WriteBoolean(MetricsEnabled); + writer.WriteInt(SubIntervals); + writer.WriteTimeSpanAsLong(RateTimeInterval); } - /// - /// Default value for . - /// - public const long DefaultCheckpointingPageBufferSize = 256L * 1024 * 1024; - - /// - /// Default value for . - /// - public const int DefaultCheckpointingThreads = 1; - - /// - /// Default value for . - /// - public static readonly TimeSpan DefaultCheckpointingFrequency = TimeSpan.FromSeconds(180); - - /// - /// Default value for . - /// - public static readonly TimeSpan DefaultLockWaitTime = TimeSpan.FromSeconds(10); - - /// - /// Default value for . - /// - public const int DefaultWalHistorySize = 20; - - /// - /// Default value for . - /// - public const int DefaultWalSegments = 10; - - /// - /// Default value for . - /// - public const int DefaultWalSegmentSize = 64 * 1024 * 1024; - - /// - /// Default value for . - /// - public const int DefaultTlbSize = 128 * 1024; - - /// - /// Default value for . - /// - public static readonly TimeSpan DefaultWalFlushFrequency = TimeSpan.FromSeconds(2); - - /// - /// Default value for . - /// - public const int DefaultWalRecordIteratorBufferSize = 64 * 1024 * 1024; - - /// - /// Default value for . - /// - public const int DefaultWalFsyncDelayNanos = 1; - /// /// Gets or sets the path where data and indexes will be persisted. /// @@ -247,5 +268,26 @@ internal void Write(IBinaryRawWriter writer) /// Gets or sets a value indicating whether full pages should always be written. /// public bool AlwaysWriteFullPages { get; set; } + + /// + /// Gets or sets a value indicating whether to enable persistent store metrics. + /// See . + /// + public bool MetricsEnabled { get; set; } + + /// + /// Gets or sets the length of the time interval for rate-based metrics. + /// This interval defines a window over which hits will be tracked. + /// + [DefaultValue(typeof(TimeSpan), "00:01:00")] + public TimeSpan RateTimeInterval { get; set; } + + /// + /// Number of sub-intervals to split the into to track the update history. + /// + [DefaultValue(DefaultSubIntervals)] + [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly", + Justification = "Consistency with Java config")] + public int SubIntervals { get; set; } } } From 313f86e5536f82cf34a9a95ad16fa1afbe37a63e Mon Sep 17 00:00:00 2001 From: sboikov Date: Mon, 10 Jul 2017 13:01:02 +0300 Subject: [PATCH 116/155] GridDhtPartitionsExchangeFuture minor: removed duplicated updateAlives call. --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 90c8aafb255e5..21a3a1373c488 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -545,8 +545,6 @@ public void init() throws IgniteInterruptedCheckedException { AffinityTopologyVersion topVer = topologyVersion(); - discoCache.updateAlives(cctx.discovery()); - srvNodes = new ArrayList<>(discoCache.serverNodes()); remaining.addAll(F.nodeIds(F.view(srvNodes, F.remoteNodes(cctx.localNodeId())))); From f1c8e59cb9410915a6e61ba6f4f63c6f3c795c75 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Mon, 10 Jul 2017 13:15:20 +0300 Subject: [PATCH 117/155] IGNITE-5716 .NET: Fix 2-byte field offset handling - improve tests --- .../Binary/BinaryFooterTest.cs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs index 36f2f65164659..5088e5aaee473 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs @@ -20,6 +20,8 @@ namespace Apache.Ignite.Core.Tests.Binary using System; using System.Linq; using Apache.Ignite.Core.Binary; + using Apache.Ignite.Core.Cache.Configuration; + using Apache.Ignite.Core.Cache.Query; using Apache.Ignite.Core.Impl; using Apache.Ignite.Core.Impl.Binary; using NUnit.Framework; @@ -117,16 +119,46 @@ private static void TestOffsets(Func getMarsh) Assert.AreEqual(dt.Arr, r.Arr); Assert.AreEqual(dt.Int, r.Int); } + + TestSql(dt, getMarsh()); } } } + /// + /// Tests SQL query, which verifies Java side of things. + /// + private static void TestSql(OffsetTest dt, Marshaller marsh) + { + var ignite = marsh.Ignite; + + if (ignite == null) + { + return; + } + + var cache = ignite.GetOrCreateCache( + new CacheConfiguration("offs", new QueryEntity(typeof(int), typeof(OffsetTest)))); + + // Cache operation. + cache[1] = dt; + Assert.AreEqual(dt.Int, cache[1].Int); + Assert.AreEqual(dt.Arr, cache[1].Arr); + + // SQL: read field on Java side to ensure correct offset handling. + var res = cache.QueryFields(new SqlFieldsQuery("select int from OffsetTest")).GetAll()[0][0]; + Assert.AreEqual(dt.Int, (int) res); + } + /// /// Offset test. /// private class OffsetTest : IBinarizable { + [QuerySqlField] public byte[] Arr; // Array to enforce field offset. + + [QuerySqlField] public int Int; // Value at offset. public void WriteBinary(IBinaryWriter writer) From 17d881ba0122a7f90cac9846c376300a1d001bdd Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Mon, 10 Jul 2017 13:55:47 +0300 Subject: [PATCH 118/155] GG-12418 - WAL hangs on any error during segment rollover --- .../PersistentStoreConfiguration.java | 24 +++ .../cache/persistence/file/FileIO.java | 154 ++++++++++++++ .../persistence/file/FileIODecorator.java | 98 +++++++++ .../cache/persistence/file/FileIOFactory.java | 45 ++++ .../cache/persistence/file/FilePageStore.java | 51 +++-- .../file/FilePageStoreManager.java | 2 + .../persistence/file/RandomAccessFileIO.java | 110 ++++++++++ .../file/RandomAccessFileIOFactory.java | 42 ++++ .../wal/AbstractWalRecordsIterator.java | 22 +- .../cache/persistence/wal/FileInput.java | 40 ++-- .../wal/FileWriteAheadLogManager.java | 161 ++++++++------- .../wal/reader/IgniteWalIteratorFactory.java | 13 +- .../reader/StandaloneGridKernalContext.java | 15 +- .../StandaloneIgnitePluginProcessor.java | 38 ++++ .../reader/StandaloneWalRecordsIterator.java | 37 ++-- ...itePdsRecoveryAfterFileCorruptionTest.java | 11 +- .../db/wal/IgniteWalFlushFailoverTest.java | 195 ++++++++++++++++++ .../db/wal/crc/IgniteDataIntegrityTests.java | 10 +- .../db/wal/reader/IgniteWalReaderTest.java | 9 +- .../db/wal/reader/MockWalIteratorFactory.java | 8 +- .../testsuites/IgnitePdsTestSuite2.java | 4 + 21 files changed, 919 insertions(+), 170 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneIgnitePluginProcessor.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java index b531f9d1510ec..479248364b4b9 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java @@ -16,6 +16,8 @@ */ package org.apache.ignite.configuration; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.util.typedef.internal.S; import java.io.Serializable; @@ -133,6 +135,9 @@ public class PersistentStoreConfiguration implements Serializable { /** Always write full pages. */ private boolean alwaysWriteFullPages = DFLT_WAL_ALWAYS_WRITE_FULL_PAGES; + /** Factory to provide I/O interface for files */ + private FileIOFactory fileIOFactory = new RandomAccessFileIOFactory(); + /** * Number of sub-intervals the whole {@link #setRateTimeInterval(long)} will be split into to calculate * rate-based metrics. @@ -538,6 +543,25 @@ public PersistentStoreConfiguration setAlwaysWriteFullPages(boolean alwaysWriteF return this; } + /** + * Factory to provide implementation of FileIO interface + * which is used for any file read/write operations + * + * @return File I/O factory + */ + public FileIOFactory getFileIOFactory() { + return fileIOFactory; + } + + /** + * @param fileIOFactory File I/O factory + */ + public PersistentStoreConfiguration setFileIOFactory(FileIOFactory fileIOFactory) { + this.fileIOFactory = fileIOFactory; + + return this; + } + /** * Note: setting this value with {@link WALMode#DEFAULT} may generate file size overhead for WAL segments in case * grid is used rarely. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java new file mode 100644 index 0000000000000..1e81150658596 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.file; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Interface to perform file I/O operations. + */ +public interface FileIO extends AutoCloseable { + /** + * Returns current file position. + * + * @return Current file position, + * a non-negative integer counting the number of bytes + * from the beginning of the file to the current position. + * + * @throws IOException If some I/O error occurs. + */ + public long position() throws IOException; + + /** + * Sets new current file position. + * + * @param newPosition + * The new position, a non-negative integer counting + * the number of bytes from the beginning of the file. + * + * @throws IOException If some I/O error occurs. + */ + public void position(long newPosition) throws IOException; + + /** + * Reads a sequence of bytes from this file into the {@code destinationBuffer}. + * + * @param destinationBuffer Destination byte buffer. + * + * @return Number of read bytes. + * + * @throws IOException If some I/O error occurs. + */ + public int read(ByteBuffer destinationBuffer) throws IOException; + + /** + * Reads a sequence of bytes from this file into the {@code destinationBuffer} + * starting from specified file {@code position}. + * + * @param destinationBuffer Destination byte buffer. + * @param position Starting position of file. + * + * @return Number of read bytes. + * + * @throws IOException If some I/O error occurs. + */ + public int read(ByteBuffer destinationBuffer, long position) throws IOException; + + /** + * Reads a up to {@code length} bytes from this file into the {@code buffer}. + * + * @param buffer Destination byte array. + * @param offset The start offset in array {@code b} + * at which the data is written. + * @param length Maximum number of bytes read. + * + * @return Number of read bytes. + * + * @throws IOException If some I/O error occurs. + */ + public int read(byte[] buffer, int offset, int length) throws IOException; + + /** + * Writes a sequence of bytes to this file from the {@code sourceBuffer}. + * + * @param sourceBuffer Source buffer. + * + * @return Number of written bytes. + * + * @throws IOException If some I/O error occurs. + */ + public int write(ByteBuffer sourceBuffer) throws IOException; + + /** + * Writes a sequence of bytes to this file from the {@code sourceBuffer} + * starting from specified file {@code position} + * + * @param sourceBuffer Source buffer. + * @param position Starting file position. + * + * @return Number of written bytes. + * + * @throws IOException If some I/O error occurs. + */ + public int write(ByteBuffer sourceBuffer, long position) throws IOException; + + /** + * Writes {@code length} bytes from the {@code buffer} + * starting at offset {@code off} to this file. + * + * @param buffer Source byte array. + * @param offset Start offset in the {@code buffer}. + * @param length Number of bytes to write. + * + * @throws IOException If some I/O error occurs. + */ + public void write(byte[] buffer, int offset, int length) throws IOException; + + /** + * Forces any updates of this file to be written to the storage + * device that contains it. + * + * @throws IOException If some I/O error occurs. + */ + public void force() throws IOException; + + /** + * Returns current file size in bytes. + * + * @return File size. + * + * @throws IOException If some I/O error occurs. + */ + public long size() throws IOException; + + /** + * Truncates current file to zero length + * and resets current file position to zero. + * + * @throws IOException If some I/O error occurs. + */ + public void clear() throws IOException; + + /** + * Closes current file. + * + * @throws IOException If some I/O error occurs. + */ + @Override public void close() throws IOException; +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java new file mode 100644 index 0000000000000..3e80ef8f3cbcb --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.file; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Decorator class for File I/O + */ +public class FileIODecorator implements FileIO { + + /** File I/O delegate */ + private final FileIO delegate; + + /** + * + * @param delegate File I/O delegate + */ + public FileIODecorator(FileIO delegate) { + this.delegate = delegate; + } + + /** {@inheritDoc} */ + @Override public long position() throws IOException { + return delegate.position(); + } + + /** {@inheritDoc} */ + @Override public void position(long newPosition) throws IOException { + delegate.position(newPosition); + } + + /** {@inheritDoc} */ + @Override public int read(ByteBuffer destinationBuffer) throws IOException { + return delegate.read(destinationBuffer); + } + + /** {@inheritDoc} */ + @Override public int read(ByteBuffer destinationBuffer, long position) throws IOException { + return delegate.read(destinationBuffer, position); + } + + /** {@inheritDoc} */ + @Override public int read(byte[] buffer, int offset, int length) throws IOException { + return delegate.read(buffer, offset, length); + } + + /** {@inheritDoc} */ + @Override public int write(ByteBuffer sourceBuffer) throws IOException { + return delegate.write(sourceBuffer); + } + + /** {@inheritDoc} */ + @Override public int write(ByteBuffer sourceBuffer, long position) throws IOException { + return delegate.write(sourceBuffer, position); + } + + /** {@inheritDoc} */ + @Override public void write(byte[] buffer, int offset, int length) throws IOException { + delegate.write(buffer, offset, length); + } + + /** {@inheritDoc} */ + @Override public void force() throws IOException { + delegate.force(); + } + + /** {@inheritDoc} */ + @Override public long size() throws IOException { + return delegate.size(); + } + + /** {@inheritDoc} */ + @Override public void clear() throws IOException { + delegate.clear(); + } + + /** {@inheritDoc} */ + @Override public void close() throws IOException { + delegate.close(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java new file mode 100644 index 0000000000000..0ffc65373a2fa --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.file; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; + +public interface FileIOFactory extends Serializable { + + /** + * Creates I/O interface for file with default I/O mode + * + * @param file File + * @return File I/O interface + * @throws IOException If I/O interface creation was failed + */ + FileIO create(File file) throws IOException; + + /** + * Creates I/O interface for file with specified mode + * + * @param file File + * @param mode I/O mode in + * @return File I/O interface + * @throws IOException If I/O interface creation was failed + */ + FileIO create(File file, String mode) throws IOException; + +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java index 6ddc9fc1678ee..c827e96d609f1 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java @@ -19,10 +19,8 @@ import java.io.File; import java.io.IOException; -import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.FileChannel; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -61,11 +59,11 @@ public class FilePageStore implements PageStore { /** Database configuration. */ private final MemoryConfiguration dbCfg; - /** */ - private RandomAccessFile file; + /** Factory to provide I/O interfaces for read/write operations with files */ + private final FileIOFactory ioFactory; - /** */ - private FileChannel ch; + /** I/O interface for read/write operations with file */ + private FileIO fileIO; /** */ private final AtomicLong allocated; @@ -91,11 +89,12 @@ public class FilePageStore implements PageStore { /** * @param file File. */ - public FilePageStore(byte type, File file, MemoryConfiguration cfg) { + public FilePageStore(byte type, File file, FileIOFactory factory, MemoryConfiguration cfg) { this.type = type; cfgFile = file; dbCfg = cfg; + ioFactory = factory; allocated = new AtomicLong(); @@ -136,7 +135,7 @@ private long initFile() { ByteBuffer hdr = header(type, dbCfg.getPageSize()); while (hdr.remaining() > 0) - ch.write(hdr); + fileIO.write(hdr); } catch (IOException e) { throw new IgniteException("Check file failed.", e); @@ -154,7 +153,7 @@ private long checkFile() throws IgniteCheckedException { ByteBuffer hdr = ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); while (hdr.remaining() > 0) - ch.read(hdr); + fileIO.read(hdr); hdr.rewind(); @@ -186,7 +185,7 @@ private long checkFile() throws IgniteCheckedException { " [expectedPageSize=" + dbCfg.getPageSize() + ", filePageSize=" + pageSize + "]"); - long fileSize = file.length(); + long fileSize = cfgFile.length(); if (fileSize == HEADER_SIZE) // Every file has a special meta page. fileSize = pageSize + HEADER_SIZE; @@ -214,9 +213,9 @@ public void stop(boolean cleanFile) throws IgniteCheckedException { if (!inited) return; - ch.force(false); + fileIO.force(); - file.close(); + fileIO.close(); if (cleanFile) cfgFile.delete(); @@ -241,9 +240,7 @@ public void truncate(int tag) throws IgniteCheckedException { this.tag = tag; - ch.position(0); - - file.setLength(0); + fileIO.clear(); allocated.set(initFile()); } @@ -277,7 +274,7 @@ public void finishRecover() { try { if (inited) - allocated.set(ch.size()); + allocated.set(fileIO.size()); recover = false; } @@ -303,7 +300,7 @@ public void finishRecover() { int len = pageSize; do { - int n = ch.read(pageBuf, off); + int n = fileIO.read(pageBuf, off); // If page was not written yet, nothing to read. if (n < 0) { @@ -330,7 +327,7 @@ public void finishRecover() { if ((savedCrc32 ^ curCrc32) != 0) throw new IgniteDataIntegrityViolationException("Failed to read page (CRC validation failed) " + "[id=" + U.hexLong(pageId) + ", off=" + (off - pageSize) + - ", file=" + cfgFile.getAbsolutePath() + ", fileSize=" + ch.size() + + ", file=" + cfgFile.getAbsolutePath() + ", fileSize=" + fileIO.size() + ", savedCrc=" + U.hexInt(savedCrc32) + ", curCrc=" + U.hexInt(curCrc32) + "]"); } @@ -356,7 +353,7 @@ public void finishRecover() { long off = 0; do { - int n = ch.read(buf, off); + int n = fileIO.read(buf, off); // If page was not written yet, nothing to read. if (n < 0) @@ -382,16 +379,14 @@ private void init() throws IgniteCheckedException { try { if (!inited) { - RandomAccessFile rndFile = null; + FileIO fileIO = null; IgniteCheckedException err = null; try { - file = rndFile = new RandomAccessFile(cfgFile, "rw"); - - ch = file.getChannel(); + this.fileIO = fileIO = ioFactory.create(cfgFile, "rw"); - if (file.length() == 0) + if (cfgFile.length() == 0) allocated.set(initFile()); else allocated.set(checkFile()); @@ -402,9 +397,9 @@ private void init() throws IgniteCheckedException { throw err = new IgniteCheckedException("Can't open file: " + cfgFile.getName(), e); } finally { - if (err != null && rndFile != null) + if (err != null && fileIO != null) try { - rndFile.close(); + fileIO.close(); } catch (IOException e) { err.addSuppressed(e); @@ -447,7 +442,7 @@ private void init() throws IgniteCheckedException { int len = pageSize; do { - int n = ch.write(pageBuf, off); + int n = fileIO.write(pageBuf, off); off += n; @@ -478,7 +473,7 @@ private void init() throws IgniteCheckedException { try { init(); - ch.force(false); + fileIO.force(); } catch (IOException e) { throw new IgniteCheckedException("Sync error", e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index 6aa2243bacd3f..4a56ec7e22dee 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -367,6 +367,7 @@ private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfigu FilePageStore idxStore = new FilePageStore( PageMemory.FLAG_IDX, idxFile, + pstCfg.getFileIOFactory(), cctx.kernalContext().config().getMemoryConfiguration()); FilePageStore[] partStores = new FilePageStore[grpDesc.config().getAffinity().partitions()]; @@ -375,6 +376,7 @@ private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfigu FilePageStore partStore = new FilePageStore( PageMemory.FLAG_DATA, new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)), + pstCfg.getFileIOFactory(), cctx.kernalContext().config().getMemoryConfiguration() ); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java new file mode 100644 index 0000000000000..73a560a19602c --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.file; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +/** + * File I/O implementation based on {@code java.io.RandomAccessFile}. + */ +public class RandomAccessFileIO implements FileIO { + + /** + * Random access file associated with this I/O + */ + private final RandomAccessFile file; + + /** + * File channel associated with {@code file} + */ + private final FileChannel channel; + + /** + * Creates I/O implementation for specified {@code file} + * + * @param file Random access file + */ + public RandomAccessFileIO(RandomAccessFile file) { + this.file = file; + this.channel = file.getChannel(); + } + + /** {@inheritDoc} */ + @Override public long position() throws IOException { + return channel.position(); + } + + /** {@inheritDoc} */ + @Override public void position(long newPosition) throws IOException { + channel.position(newPosition); + } + + /** {@inheritDoc} */ + @Override public int read(ByteBuffer destinationBuffer) throws IOException { + return channel.read(destinationBuffer); + } + + /** {@inheritDoc} */ + @Override public int read(ByteBuffer destinationBuffer, long position) throws IOException { + return channel.read(destinationBuffer, position); + } + + /** {@inheritDoc} */ + @Override public int read(byte[] buffer, int offset, int length) throws IOException { + return file.read(buffer, offset, length); + } + + /** {@inheritDoc} */ + @Override public int write(ByteBuffer sourceBuffer) throws IOException { + return channel.write(sourceBuffer); + } + + /** {@inheritDoc} */ + @Override public int write(ByteBuffer sourceBuffer, long position) throws IOException { + return channel.write(sourceBuffer, position); + } + + /** {@inheritDoc} */ + @Override public void write(byte[] buffer, int offset, int length) throws IOException { + file.write(buffer, offset, length); + } + + /** {@inheritDoc} */ + @Override public void force() throws IOException { + channel.force(false); + } + + /** {@inheritDoc} */ + @Override public long size() throws IOException { + return channel.size(); + } + + /** {@inheritDoc} */ + @Override public void clear() throws IOException { + channel.position(0); + file.setLength(0); + } + + /** {@inheritDoc} */ + @Override public void close() throws IOException { + file.close(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java new file mode 100644 index 0000000000000..6b731f2ad7e8f --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.file; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + +/** + * File I/O factory which provides RandomAccessFileIO implementation of FileIO. + */ +public class RandomAccessFileIOFactory implements FileIOFactory { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override public FileIO create(File file) throws IOException { + return create(file, "rw"); + } + + /** {@inheritDoc} */ + @Override public FileIO create(File file, String mode) throws IOException { + RandomAccessFile rf = new RandomAccessFile(file, mode); + + return new RandomAccessFileIO(rf); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java index f4bace171ecca..beed90b1c4996 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java @@ -21,15 +21,15 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.RandomAccessFile; import java.nio.ByteOrder; -import java.nio.channels.FileChannel; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; import org.apache.ignite.internal.util.GridCloseableIteratorAdapter; import org.apache.ignite.lang.IgniteBiTuple; @@ -71,6 +71,9 @@ public abstract class AbstractWalRecordsIterator /** Serializer of current version to read headers. */ @NotNull private final RecordSerializer serializer; + /** Factory to provide I/O interfaces for read/write operations with files */ + @NotNull protected final FileIOFactory ioFactory; + /** Utility buffer for reading records */ private final ByteBufferExpander buf; @@ -84,11 +87,13 @@ protected AbstractWalRecordsIterator( @NotNull final IgniteLogger log, @NotNull final GridCacheSharedContext sharedCtx, @NotNull final RecordSerializer serializer, + @NotNull final FileIOFactory ioFactory, final int bufSize ) { this.log = log; this.sharedCtx = sharedCtx; this.serializer = serializer; + this.ioFactory = ioFactory; // Do not allocate direct buffer for iterator. buf = new ByteBufferExpander(bufSize, ByteOrder.nativeOrder()); @@ -229,15 +234,14 @@ protected FileWriteAheadLogManager.ReadFileHandle initReadHandle( @Nullable final FileWALPointer start) throws IgniteCheckedException, FileNotFoundException { try { - RandomAccessFile rf = new RandomAccessFile(desc.file, "r"); + FileIO fileIO = ioFactory.create(desc.file, "r"); try { - FileChannel ch = rf.getChannel(); - FileInput in = new FileInput(ch, buf); + FileInput in = new FileInput(fileIO, buf); // Header record must be agnostic to the serializer version. WALRecord rec = serializer.readRecord(in, - new FileWALPointer(desc.idx, (int)ch.position(), 0)); + new FileWALPointer(desc.idx, (int)fileIO.position(), 0)); if (rec == null) return null; @@ -252,11 +256,11 @@ protected FileWriteAheadLogManager.ReadFileHandle initReadHandle( if (start != null && desc.idx == start.index()) in.seek(start.fileOffset()); - return new FileWriteAheadLogManager.ReadFileHandle(rf, desc.idx, sharedCtx.igniteInstanceName(), ser, in); + return new FileWriteAheadLogManager.ReadFileHandle(fileIO, desc.idx, sharedCtx.igniteInstanceName(), ser, in); } catch (SegmentEofException | EOFException ignore) { try { - rf.close(); + fileIO.close(); } catch (IOException ce) { throw new IgniteCheckedException(ce); @@ -266,7 +270,7 @@ protected FileWriteAheadLogManager.ReadFileHandle initReadHandle( } catch (IOException | IgniteCheckedException e) { try { - rf.close(); + fileIO.close(); } catch (IOException ce) { e.addSuppressed(ce); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java index 00c7c02b6f95c..6443a7cab4dca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java @@ -20,7 +20,7 @@ import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.jetbrains.annotations.NotNull; @@ -36,8 +36,8 @@ public final class FileInput implements ByteBufferBackedDataInput { */ private ByteBuffer buf; - /** File channel to read chunks from */ - private FileChannel ch; + /** I/O interface for read/write operations with file */ + private FileIO io; /** */ private long pos; @@ -46,28 +46,20 @@ public final class FileInput implements ByteBufferBackedDataInput { private ByteBufferExpander expBuf; /** - * @param ch Channel to read from - * @param buf Buffer for reading blocks of data into + * @param io FileIO to read from. + * @param buf Buffer for reading blocks of data into. */ - public FileInput(FileChannel ch, ByteBuffer buf) throws IOException { - assert ch != null; + public FileInput(FileIO io, ByteBufferExpander buf) throws IOException { + assert io != null; - this.ch = ch; - this.buf = buf; + this.io = io; + this.buf = buf.buffer(); - pos = ch.position(); + expBuf = buf; - clearBuffer(); - } - - /** - * @param ch Channel to read from - * @param expBuf ByteBufferWrapper with ability expand buffer dynamically. - */ - public FileInput(FileChannel ch, ByteBufferExpander expBuf) throws IOException { - this(ch, expBuf.buffer()); + pos = io.position(); - this.expBuf = expBuf; + clearBuffer(); } /** @@ -84,10 +76,10 @@ private void clearBuffer() { * @param pos Position in bytes from file begin. */ public void seek(long pos) throws IOException { - if (pos > ch.size()) + if (pos > io.size()) throw new EOFException(); - ch.position(pos); + io.position(pos); this.pos = pos; @@ -118,10 +110,10 @@ public void seek(long pos) throws IOException { buf.compact(); do { - int read = ch.read(buf); + int read = io.read(buf); if (read == -1) - throw new EOFException("EOF at position [" + ch.position() + "] expected to read [" + requested + "] bytes"); + throw new EOFException("EOF at position [" + io.position() + "] expected to read [" + requested + "] bytes"); available += read; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 162f43d674a7e..5c112fb1b72b1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -22,10 +22,8 @@ import java.io.FileFilter; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.FileChannel; import java.nio.file.Files; import java.sql.Time; import java.util.Arrays; @@ -48,6 +46,7 @@ import org.apache.ignite.configuration.PersistentStoreConfiguration; import org.apache.ignite.configuration.WALMode; import org.apache.ignite.events.EventType; +import org.apache.ignite.events.WalSegmentArchivedEvent; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager; @@ -61,7 +60,8 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.PersistenceMetricsImpl; -import org.apache.ignite.events.WalSegmentArchivedEvent; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer; import org.apache.ignite.internal.processors.timeout.GridTimeoutObject; @@ -153,6 +153,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl /** */ private volatile long oldestArchiveSegmentIdx; + /** Factory to provide I/O interfaces for read/write operations with files */ + private final FileIOFactory ioFactory; + /** Updater for {@link #currentHnd}, used for verify there are no concurrent update for current log segment handle */ private static final AtomicReferenceFieldUpdater currentHndUpd = AtomicReferenceFieldUpdater.newUpdater(FileWriteAheadLogManager.class, FileWriteHandle.class, "currentHnd"); @@ -181,6 +184,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl /** Current log segment handle */ private volatile FileWriteHandle currentHnd; + /** Environment failure. */ + private volatile Throwable envFailed; + /** * Positive (non-0) value indicates WAL can be archived even if not complete
* See {@link PersistentStoreConfiguration#setWalAutoArchiveAfterInactivity(long)}
@@ -225,6 +231,7 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) { flushFreq = psCfg.getWalFlushFrequency(); fsyncDelay = psCfg.getWalFsyncDelay(); alwaysWriteFullPages = psCfg.isAlwaysWriteFullPages(); + ioFactory = psCfg.getFileIOFactory(); walAutoArchiveAfterInactivity = psCfg.getWalAutoArchiveAfterInactivity(); evt = ctx.event(); } @@ -322,7 +329,7 @@ protected String consistentId() { archiver.shutdown(); } catch (Exception e) { - U.error(log, "Failed to gracefully close WAL segment: " + currHnd.file, e); + U.error(log, "Failed to gracefully close WAL segment: " + currentHnd.fileIO, e); } } @@ -493,6 +500,8 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { return ptr; } + checkEnvironment(); + if (isStopping()) throw new IgniteCheckedException("Stopping."); } @@ -549,6 +558,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() { end, psCfg, serializer, + ioFactory, archiver, log, tlbSize @@ -800,13 +810,13 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig int len = lastReadPtr == null ? 0 : lastReadPtr.length(); try { - RandomAccessFile file = new RandomAccessFile(curFile, "rw"); + FileIO fileIO = ioFactory.create(curFile); try { // readSerializerVersion will change the channel position. // This is fine because the FileWriteHandle consitructor will move it // to offset + len anyways. - int serVer = readSerializerVersion(file, curFile, absIdx); + int serVer = readSerializerVersion(fileIO, curFile, absIdx); RecordSerializer ser = forVersion(cctx, serVer); @@ -815,7 +825,7 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig ", offset=" + offset + ", ver=" + serVer + ']'); FileWriteHandle hnd = new FileWriteHandle( - file, + fileIO, absIdx, cctx.igniteInstanceName(), offset + len, @@ -835,7 +845,7 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig return hnd; } catch (IgniteCheckedException | IOException e) { - file.close(); + fileIO.close(); throw e; } @@ -862,10 +872,10 @@ private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException if (log.isDebugEnabled()) log.debug("Switching to a new WAL segment: " + nextFile.getAbsolutePath()); - RandomAccessFile file = new RandomAccessFile(nextFile, "rw"); + FileIO fileIO = ioFactory.create(nextFile); FileWriteHandle hnd = new FileWriteHandle( - file, + fileIO, curIdx + 1, cctx.igniteInstanceName(), 0, @@ -929,22 +939,22 @@ private void formatFile(File file) throws IgniteCheckedException { if (log.isDebugEnabled()) log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']'); - try (RandomAccessFile rnd = new RandomAccessFile(file, "rw")) { + try (FileIO fileIO = ioFactory.create(file, "rw")) { int left = psCfg.getWalSegmentSize(); if (mode == WALMode.DEFAULT) { while (left > 0) { int toWrite = Math.min(FILL_BUF.length, left); - rnd.write(FILL_BUF, 0, toWrite); + fileIO.write(FILL_BUF, 0, toWrite); left -= toWrite; } - rnd.getChannel().force(false); + fileIO.force(); } else - rnd.setLength(0); + fileIO.clear(); } catch (IOException e) { throw new IgniteCheckedException("Failed to format WAL segment file: " + file.getAbsolutePath(), e); @@ -1032,6 +1042,15 @@ public static FileDescriptor[] scan(File[] allFiles) { return descs; } + /** + * @throws StorageException If environment is no longer valid and we missed a WAL write. + */ + private void checkEnvironment() throws StorageException { + if (envFailed != null) + throw new StorageException("Failed to flush WAL buffer (environment was invalidated by a " + + "previous error)", envFailed); + } + /** * File archiver operates on absolute segment indexes. For any given absolute segment index N we can calculate * the work WAL segment: S(N) = N % psCfg.walSegments. @@ -1337,8 +1356,8 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedExc Files.move(dstTmpFile.toPath(), dstFile.toPath()); if (mode == WALMode.DEFAULT) { - try (RandomAccessFile f0 = new RandomAccessFile(dstFile, "rw")) { - f0.getChannel().force(false); + try (FileIO f0 = ioFactory.create(dstFile, "rw")) { + f0.force(); } } } @@ -1402,20 +1421,21 @@ else if (create) } /** - * @param rf Random access file. + * @param io I/O interface for file. * @param file File object. * @param idx File index to read. * @return Serializer version stored in the file. * @throws IOException If failed to read serializer version. * @throws IgniteCheckedException If failed to read serializer version. */ - private int readSerializerVersion(RandomAccessFile rf, File file, long idx) + private int readSerializerVersion(FileIO io, File file, long idx) throws IOException, IgniteCheckedException { try { ByteBuffer buf = ByteBuffer.allocate(RecordV1Serializer.HEADER_RECORD_SIZE); buf.order(ByteOrder.nativeOrder()); - FileInput in = new FileInput(rf.getChannel(), buf); + FileInput in = new FileInput(io, + new ByteBufferExpander(RecordV1Serializer.HEADER_RECORD_SIZE, ByteOrder.nativeOrder())); // Header record must be agnostic to the serializer version. WALRecord rec = serializer.readRecord(in, new FileWALPointer(idx, 0, 0)); @@ -1541,11 +1561,8 @@ public String getAbsolutePath() { * */ private abstract static class FileHandle { - /** */ - protected RandomAccessFile file; - - /** */ - protected FileChannel ch; + /** I/O interface for read/write operations with file */ + protected FileIO fileIO; /** Absolute WAL segment file index (incremental counter) */ protected final long idx; @@ -1554,17 +1571,13 @@ private abstract static class FileHandle { protected String gridName; /** - * @param file File. + * @param fileIO I/O interface for read/write operations of FileHandle. * @param idx Absolute WAL segment file index (incremental counter). */ - private FileHandle(RandomAccessFile file, long idx, String gridName) { - this.file = file; + private FileHandle(FileIO fileIO, long idx, String gridName) { + this.fileIO = fileIO; this.idx = idx; this.gridName = gridName; - - ch = file.getChannel(); - - assert ch != null; } } @@ -1585,19 +1598,19 @@ public static class ReadFileHandle extends FileHandle { private boolean workDir; /** - * @param file File to read. + * @param fileIO I/O interface for read/write operations of FileHandle. * @param idx Absolute WAL segment file index (incremental counter). * @param ser Entry serializer. * @param in File input. */ ReadFileHandle( - RandomAccessFile file, - long idx, - String gridName, - RecordSerializer ser, - FileInput in + FileIO fileIO, + long idx, + String gridName, + RecordSerializer ser, + FileInput in ) { - super(file, idx, gridName); + super(fileIO, idx, gridName); this.ser = ser; this.in = in; @@ -1608,7 +1621,7 @@ public static class ReadFileHandle extends FileHandle { */ public void close() throws IgniteCheckedException { try { - file.close(); + fileIO.close(); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -1644,10 +1657,7 @@ private class FileWriteHandle extends FileHandle { /** */ private volatile long lastFsyncPos; - /** Environment failure. */ - private volatile Throwable envFailed; - - /** Stop guard to provide warranty that only one thread will be successful in calling {@link #close(boolean)} */ + /** Stop guard to provide warranty that only one thread will be successful in calling {@link #close(boolean)}*/ private final AtomicBoolean stop = new AtomicBoolean(false); /** */ @@ -1661,12 +1671,12 @@ private class FileWriteHandle extends FileHandle { /** * Next segment available condition. - * Protection from "spurious wakeup" is provided by predicate {@link #ch}=null + * Protection from "spurious wakeup" is provided by predicate {@link #fileIO}=null */ private final Condition nextSegment = lock.newCondition(); /** - * @param file Mapped file to use. + * @param fileIO I/O file interface to use * @param idx Absolute WAL segment file index for easy access. * @param pos Position. * @param maxSegmentSize Max segment size. @@ -1674,18 +1684,18 @@ private class FileWriteHandle extends FileHandle { * @throws IOException If failed. */ private FileWriteHandle( - RandomAccessFile file, + FileIO fileIO, long idx, String gridName, long pos, long maxSegmentSize, RecordSerializer serializer ) throws IOException { - super(file, idx, gridName); + super(fileIO, idx, gridName); assert serializer != null; - ch.position(pos); + fileIO.position(pos); this.maxSegmentSize = maxSegmentSize; this.serializer = serializer; @@ -1887,6 +1897,9 @@ private boolean flush(WALRecord expHead) throws StorageException, IgniteCheckedE catch (Throwable e) { invalidateEnvironment(e); + // All workers waiting for a next segment must be woken up and stopped + signalNextAvailable(); + throw e; } } @@ -1990,7 +2003,7 @@ private void fsync(FileWALPointer ptr) throws StorageException, IgniteCheckedExc long start = metricsEnabled ? System.nanoTime() : 0; try { - ch.force(false); + fileIO.force(); } catch (IOException e) { throw new StorageException(e); @@ -2027,20 +2040,24 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx try { int switchSegmentRecSize = RecordV1Serializer.REC_TYPE_SIZE + RecordV1Serializer.FILE_WAL_POINTER_SIZE; + if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) { //it is expected there is sufficient space for this record because rollover should run early final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize); buf.put((byte)(WALRecord.RecordType.SWITCH_SEGMENT_RECORD.ordinal() + 1)); - final FileWALPointer pointer = new FileWALPointer(idx, (int)ch.position(), -1); + + final FileWALPointer pointer = new FileWALPointer(idx, (int)fileIO.position(), -1); RecordV1Serializer.putPosition(buf, pointer); + buf.rewind(); - ch.write(buf, written); + + fileIO.write(buf, written); if (mode == WALMode.DEFAULT) - ch.force(false); + fileIO.force(); } - ch.close(); + fileIO.close(); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -2064,13 +2081,15 @@ private void signalNextAvailable() { try { WALRecord rec = head.get(); - assert rec instanceof FakeRecord : "Expected head FakeRecord, actual head " + if (envFailed == null) { + assert rec instanceof FakeRecord : "Expected head FakeRecord, actual head " + (rec != null ? rec.getClass().getSimpleName() : "null"); - assert written == lastFsyncPos || mode != WALMode.DEFAULT : + assert written == lastFsyncPos || mode != WALMode.DEFAULT : "fsync [written=" + written + ", lastFsync=" + lastFsyncPos + ']'; + } - ch = null; + fileIO = null; nextSegment.signalAll(); } @@ -2086,7 +2105,7 @@ private void awaitNext() throws IgniteCheckedException { lock.lock(); try { - while (ch != null) + while (fileIO != null) U.await(nextSegment); } finally { @@ -2108,7 +2127,7 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni lock.lock(); try { - assert ch != null : "Writing to a closed segment."; + assert fileIO != null : "Writing to a closed segment."; checkEnvironment(); @@ -2151,10 +2170,10 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni assert size > 0 : size; try { - assert written == ch.position(); + assert written == fileIO.position(); do { - ch.write(buf); + fileIO.write(buf); } while (buf.hasRemaining()); @@ -2162,7 +2181,7 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni metrics.onWalBytesWritten(size); - assert written == ch.position(); + assert written == fileIO.position(); } catch (IOException e) { invalidateEnvironmentLocked(e); @@ -2214,26 +2233,17 @@ private void invalidateEnvironmentLocked(Throwable e) { } } - /** - * @throws StorageException If environment is no longer valid and we missed a WAL write. - */ - private void checkEnvironment() throws StorageException { - if (envFailed != null) - throw new StorageException("Failed to flush WAL buffer (environment was invalidated by a " + - "previous error)", envFailed); - } - /** * @return Safely reads current position of the file channel as String. Will return "null" if channel is null. */ private String safePosition() { - FileChannel ch = this.ch; + FileIO io = this.fileIO; - if (ch == null) + if (io == null) return "null"; try { - return String.valueOf(ch.position()); + return String.valueOf(io.position()); } catch (IOException e) { return "{Failed to read channel position: " + e.getMessage() + "}"; @@ -2320,6 +2330,7 @@ private RecordsIterator( @Nullable FileWALPointer end, PersistentStoreConfiguration psCfg, @NotNull RecordSerializer serializer, + FileIOFactory ioFactory, FileArchiver archiver, IgniteLogger log, int tlbSize @@ -2327,6 +2338,7 @@ private RecordsIterator( super(log, cctx, serializer, + ioFactory, psCfg.getWalRecordIteratorBufferSize()); this.walWorkDir = walWorkDir; this.walArchiveDir = walArchiveDir; @@ -2479,11 +2491,10 @@ private void releaseWorkSegment(long absIdx) { */ private void doFlush() { final FileWriteHandle hnd = currentHandle(); - try { hnd.flush(hnd.head.get()); } - catch (IgniteCheckedException e) { + catch (Exception e) { U.warn(log, "Failed to flush WAL record queue", e); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java index 8ea058560391f..4e3998bf70853 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java @@ -24,6 +24,7 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.jetbrains.annotations.NotNull; /** @@ -34,15 +35,18 @@ public class IgniteWalIteratorFactory { private final IgniteLogger log; /** Page size, in standalone iterator mode this value can't be taken from memory configuration */ private final int pageSize; + /** Factory to provide I/O interfaces for read/write operations with files */ + private final FileIOFactory ioFactory; /** * Creates WAL files iterator factory * @param log Logger. * @param pageSize Page size, size is validated */ - public IgniteWalIteratorFactory(@NotNull final IgniteLogger log, final int pageSize) { + public IgniteWalIteratorFactory(@NotNull IgniteLogger log, @NotNull FileIOFactory ioFactory, int pageSize) { this.log = log; this.pageSize = pageSize; + this.ioFactory = ioFactory; new MemoryConfiguration().setPageSize(pageSize); // just for validate } @@ -57,7 +61,7 @@ public IgniteWalIteratorFactory(@NotNull final IgniteLogger log, final int pageS * @throws IgniteCheckedException if failed to read folder */ public WALIterator iteratorArchiveDirectory(@NotNull final File walDirWithConsistentId) throws IgniteCheckedException { - return new StandaloneWalRecordsIterator(walDirWithConsistentId, log, prepareSharedCtx()); + return new StandaloneWalRecordsIterator(walDirWithConsistentId, log, prepareSharedCtx(), ioFactory); } /** @@ -69,7 +73,7 @@ public WALIterator iteratorArchiveDirectory(@NotNull final File walDirWithConsis * @throws IgniteCheckedException if failed to read files */ public WALIterator iteratorArchiveFiles(@NotNull final File ...files) throws IgniteCheckedException { - return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), false, files); + return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, false, files); } /** @@ -81,7 +85,7 @@ public WALIterator iteratorArchiveFiles(@NotNull final File ...files) throws Ign * @throws IgniteCheckedException if failed to read files */ public WALIterator iteratorWorkFiles(@NotNull final File ...files) throws IgniteCheckedException { - return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), true, files); + return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, true, files); } /** @@ -93,6 +97,7 @@ public WALIterator iteratorWorkFiles(@NotNull final File ...files) throws Ignite final StandaloneIgniteCacheDatabaseSharedManager dbMgr = new StandaloneIgniteCacheDatabaseSharedManager(); dbMgr.setPageSize(pageSize); + return new GridCacheSharedContext<>( kernalCtx, null, null, null, null, null, dbMgr, null, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java index df932e600cfb2..02b93529a755a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.ExecutorService; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridComponent; @@ -82,13 +83,25 @@ * Dummy grid kernal context */ public class StandaloneGridKernalContext implements GridKernalContext { + /** */ private IgniteLogger log; + /** */ + private IgnitePluginProcessor pluginProc; + /** * @param log Logger. */ StandaloneGridKernalContext(IgniteLogger log) { this.log = log; + + try { + pluginProc = new StandaloneIgnitePluginProcessor( + this, config()); + } + catch (IgniteCheckedException e) { + throw new IllegalStateException("Must not fail on empty providers list.", e); + } } /** {@inheritDoc} */ @@ -278,7 +291,7 @@ public class StandaloneGridKernalContext implements GridKernalContext { /** {@inheritDoc} */ @Override public IgnitePluginProcessor plugins() { - return null; + return pluginProc; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneIgnitePluginProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneIgnitePluginProcessor.java new file mode 100644 index 0000000000000..838fc85c9d497 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneIgnitePluginProcessor.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.wal.reader; + +import java.util.Collections; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor; +import org.apache.ignite.plugin.PluginProvider; + +/** + * + */ +public class StandaloneIgnitePluginProcessor extends IgnitePluginProcessor { + /** + * @param ctx Kernal context. + * @param cfg Ignite configuration. + */ + public StandaloneIgnitePluginProcessor(GridKernalContext ctx, IgniteConfiguration cfg) throws IgniteCheckedException { + super(ctx, cfg, Collections.emptyList()); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java index f17c1122aa666..ecad70abe7094 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java @@ -21,10 +21,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -33,7 +30,10 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator; +import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander; import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; @@ -83,14 +83,17 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { * @param walFilesDir Wal files directory. Should already contain node consistent ID as subfolder * @param log Logger. * @param sharedCtx Shared context. + * @param ioFactory File I/O factory. */ StandaloneWalRecordsIterator( - @NotNull final File walFilesDir, - @NotNull final IgniteLogger log, - @NotNull final GridCacheSharedContext sharedCtx) throws IgniteCheckedException { + @NotNull File walFilesDir, + @NotNull IgniteLogger log, + @NotNull GridCacheSharedContext sharedCtx, + @NotNull FileIOFactory ioFactory) throws IgniteCheckedException { super(log, sharedCtx, new RecordV1Serializer(sharedCtx), + ioFactory, BUF_SIZE); init(walFilesDir, false, null); advance(); @@ -101,17 +104,20 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator { * * @param log Logger. * @param sharedCtx Shared context. + * @param ioFactory File I/O factory. * @param workDir Work directory is scanned, false - archive * @param walFiles Wal files. */ StandaloneWalRecordsIterator( - @NotNull final IgniteLogger log, - @NotNull final GridCacheSharedContext sharedCtx, - final boolean workDir, - @NotNull final File... walFiles) throws IgniteCheckedException { + @NotNull IgniteLogger log, + @NotNull GridCacheSharedContext sharedCtx, + @NotNull FileIOFactory ioFactory, + boolean workDir, + @NotNull File... walFiles) throws IgniteCheckedException { super(log, sharedCtx, new RecordV1Serializer(sharedCtx), + ioFactory, BUF_SIZE); this.workDir = workDir; init(null, workDir, walFiles); @@ -138,10 +144,12 @@ private void init( } else { this.workDir = workDir; + if (workDir) walFileDescriptors = scanIndexesFromFileHeaders(walFiles); else walFileDescriptors = new ArrayList<>(Arrays.asList(FileWriteAheadLogManager.scan(walFiles))); + curWalSegmIdx = !walFileDescriptors.isEmpty() ? walFileDescriptors.get(0).getIdx() : 0; } curWalSegmIdx--; @@ -172,13 +180,10 @@ private List scanIndexesFromFileHeaders FileWALPointer ptr; - try (RandomAccessFile rf = new RandomAccessFile(file, "r");) { - final FileChannel ch = rf.getChannel(); - final ByteBuffer buf = ByteBuffer.allocate(HEADER_RECORD_SIZE); - - buf.order(ByteOrder.nativeOrder()); + try (FileIO fileIO = ioFactory.create(file, "r")) { + final DataInput in = new FileInput(fileIO, + new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())); - final DataInput in = new FileInput(ch, buf); // Header record must be agnostic to the serializer version. final int type = in.readUnsignedByte(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java index 6847482e05968..e086258613085 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.FileChannel; import java.util.Collection; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; @@ -43,7 +42,7 @@ import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; -import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl; @@ -191,13 +190,13 @@ private void eraseDataFromDisk( FilePageStore filePageStore = (FilePageStore)store; - FileChannel ch = U.field(filePageStore, "ch"); + FileIO fileIO = U.field(filePageStore, "fileIO"); - long size = ch.size(); + long size = fileIO.size(); - ch.write(ByteBuffer.allocate((int)size - FilePageStore.HEADER_SIZE), FilePageStore.HEADER_SIZE); + fileIO.write(ByteBuffer.allocate((int)size - FilePageStore.HEADER_SIZE), FilePageStore.HEADER_SIZE); - ch.force(false); + fileIO.force(); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java new file mode 100644 index 0000000000000..cad10ae0e15bb --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db.wal; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.MemoryConfiguration; +import org.apache.ignite.configuration.MemoryPolicyConfiguration; +import org.apache.ignite.configuration.PersistentStoreConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.GridKernalState; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionConcurrency; +import org.apache.ignite.transactions.TransactionIsolation; + +/** + * + */ +public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest { + + /** */ + private static final String TEST_CACHE = "testCache"; + + /** */ + private boolean flushByTimeout; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + deleteWorkFiles(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + deleteWorkFiles(); + } + + /** {@inheritDoc} */ + @Override protected long getTestTimeout() { + return 30_000L; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + CacheConfiguration cacheCfg = new CacheConfiguration(TEST_CACHE) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + + cfg.setCacheConfiguration(cacheCfg); + + MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration() + .setName("dfltMemPlc") + .setInitialSize(2 * 1024L * 1024L * 1024L); + + MemoryConfiguration memCfg = new MemoryConfiguration() + .setMemoryPolicies(memPlcCfg) + .setDefaultMemoryPolicyName(memPlcCfg.getName()); + + cfg.setMemoryConfiguration(memCfg); + + PersistentStoreConfiguration storeCfg = new PersistentStoreConfiguration() + .setFileIOFactory(new FailingFileIOFactory()) + .setWalMode(WALMode.BACKGROUND) + // Setting WAL Segment size to high values forces flushing by timeout. + .setWalSegmentSize(flushByTimeout ? 500_000 : 50_000); + + cfg.setPersistentStoreConfiguration(storeCfg); + + return cfg; + } + + /** + * Test flushing error recovery when flush is triggered asynchronously by timeout + * + * @throws Exception In case of fail + */ + public void testErrorOnFlushByTimeout() throws Exception { + flushByTimeout = true; + flushingErrorTest(); + } + + /** + * Test flushing error recovery when flush is triggered directly by transaction commit + * + * @throws Exception In case of fail + */ + public void testErrorOnDirectFlush() throws Exception { + flushByTimeout = false; + flushingErrorTest(); + } + + /** + * @throws Exception if failed. + */ + private void flushingErrorTest() throws Exception { + final IgniteEx grid = startGrid(0); + grid.active(true); + + IgniteCache cache = grid.cache(TEST_CACHE); + + final int iterations = 100; + + try { + for (int i = 0; i < iterations; i++) { + Transaction tx = grid.transactions().txStart( + TransactionConcurrency.PESSIMISTIC, TransactionIsolation.READ_COMMITTED); + + cache.put(i, "testValue" + i); + + Thread.sleep(100L); + + tx.commitAsync().get(); + } + } + catch (Exception expected) { + // There can be any exception. Do nothing. + } + + // We should await successful stop of node. + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override + public boolean apply() { + return grid.context().gateway().getState() == GridKernalState.STOPPED; + } + }, getTestTimeout()); + } + + /** + * @throws IgniteCheckedException + */ + private void deleteWorkFiles() throws IgniteCheckedException { + deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); + } + + /** + * Create File I/O which fails after second attempt to write to File + */ + private static class FailingFileIOFactory implements FileIOFactory { + private static final long serialVersionUID = 0L; + + private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory(); + + @Override + public FileIO create(File file) throws IOException { + return create(file, "rw"); + } + + @Override + public FileIO create(File file, String mode) throws IOException { + FileIO delegate = delegateFactory.create(file, mode); + + return new FileIODecorator(delegate) { + int writeAttempts = 2; + + @Override + public int write(ByteBuffer sourceBuffer) throws IOException { + if (--writeAttempts == 0) + throw new RuntimeException("Test exception. Unable to write to file."); + + return super.write(sourceBuffer); + } + }; + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java index 303f14e0992f6..b93c74db14760 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java @@ -23,7 +23,10 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO; +import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander; import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; @@ -47,9 +50,10 @@ public class IgniteDataIntegrityTests extends TestCase { randomAccessFile = new RandomAccessFile(file, "rw"); - fileInput = new FileInput(randomAccessFile.getChannel(), ByteBuffer.allocate(1024)); - - PureJavaCrc32 pureJavaCrc32 = new PureJavaCrc32(); + fileInput = new FileInput( + new RandomAccessFileIO(randomAccessFile), + new ByteBufferExpander(1024, ByteOrder.BIG_ENDIAN) + ); ByteBuffer buf = ByteBuffer.allocate(1024); ThreadLocalRandom curr = ThreadLocalRandom.current(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java index 06bcf08fa47ec..ca54e6fdd0e0a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java @@ -39,11 +39,12 @@ import org.apache.ignite.configuration.WALMode; import org.apache.ignite.events.Event; import org.apache.ignite.events.EventType; +import org.apache.ignite.events.WalSegmentArchivedEvent; import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; -import org.apache.ignite.events.WalSegmentArchivedEvent; import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -177,7 +178,9 @@ public void testFillWalAndReadRecords() throws Exception { final File wal = new File(db, "wal"); final File walArchive = new File(wal, "archive"); final String consistentId = "127_0_0_1_47500"; - final MockWalIteratorFactory mockItFactory = new MockWalIteratorFactory(log, PAGE_SIZE, consistentId, WAL_SEGMENTS); + + final FileIOFactory fileIOFactory = getConfiguration("").getPersistentStoreConfiguration().getFileIOFactory(); + final MockWalIteratorFactory mockItFactory = new MockWalIteratorFactory(log, fileIOFactory, PAGE_SIZE, consistentId, WAL_SEGMENTS); final WALIterator it = mockItFactory.iterator(wal, walArchive); final int cntUsingMockIter = iterateAndCount(it); @@ -188,7 +191,7 @@ public void testFillWalAndReadRecords() throws Exception { final File walArchiveDirWithConsistentId = new File(walArchive, consistentId); final File walWorkDirWithConsistentId = new File(wal, consistentId); - final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE); + final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, fileIOFactory, PAGE_SIZE); final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId)); log.info("Total records loaded using directory : " + cntArchiveDir); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java index 95079a0049b8f..ef162d2e9c33d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.jetbrains.annotations.Nullable; import org.mockito.Mockito; @@ -42,6 +43,9 @@ public class MockWalIteratorFactory { /** Logger. */ private final IgniteLogger log; + /** */ + private final FileIOFactory ioFactory; + /** Page size. */ private final int pageSize; @@ -58,8 +62,9 @@ public class MockWalIteratorFactory { * @param consistentId Consistent id. * @param segments Segments. */ - public MockWalIteratorFactory(@Nullable IgniteLogger log, int pageSize, String consistentId, int segments) { + public MockWalIteratorFactory(@Nullable IgniteLogger log, FileIOFactory ioFactory, int pageSize, String consistentId, int segments) { this.log = log == null ? Mockito.mock(IgniteLogger.class) : log; + this.ioFactory = ioFactory; this.pageSize = pageSize; this.consistentId = consistentId; this.segments = segments; @@ -80,6 +85,7 @@ public WALIterator iterator(File wal, File walArchive) throws IgniteCheckedExcep when(persistentCfg1.getWalSegments()).thenReturn(segments); when(persistentCfg1.getTlbSize()).thenReturn(PersistentStoreConfiguration.DFLT_TLB_SIZE); when(persistentCfg1.getWalRecordIteratorBufferSize()).thenReturn(PersistentStoreConfiguration.DFLT_WAL_RECORD_ITERATOR_BUFFER_SIZE); + when(persistentCfg1.getFileIOFactory()).thenReturn(ioFactory); final IgniteConfiguration cfg = Mockito.mock(IgniteConfiguration.class); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index 801870595dcc2..4f4dedfe879af 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest; +import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFailoverTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteDataIntegrityTests; import org.apache.ignite.internal.processors.cache.persistence.db.wal.reader.IgniteWalReaderTest; @@ -65,6 +66,9 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgnitePersistentStoreDataStructuresTest.class); + // Failover test + suite.addTestSuite(IgniteWalFlushFailoverTest.class); + suite.addTestSuite(IgniteWalReaderTest.class); return suite; } From 56868102ef46e37a914593fa0f81317ccb8e4cc4 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Mon, 10 Jul 2017 14:02:30 +0300 Subject: [PATCH 119/155] Fixed NPE when test kernal context is used --- .../testframework/junits/GridTestKernalContext.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java index 3d3de22af0507..6b39faaf01992 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java @@ -17,6 +17,7 @@ package org.apache.ignite.testframework.junits; +import java.util.Collections; import java.util.List; import java.util.ListIterator; import java.util.concurrent.ExecutorService; @@ -28,8 +29,10 @@ import org.apache.ignite.internal.GridKernalGatewayImpl; import org.apache.ignite.internal.GridLoggerProxy; import org.apache.ignite.internal.IgniteKernal; +import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.testframework.GridTestUtils; /** @@ -41,6 +44,13 @@ public class GridTestKernalContext extends GridKernalContextImpl { */ public GridTestKernalContext(IgniteLogger log) { this(log, new IgniteConfiguration()); + + try { + add(new IgnitePluginProcessor(this, config(), Collections.emptyList())); + } + catch (IgniteCheckedException e) { + throw new IllegalStateException("Must not fail for empty plugins list.", e); + } } /** From f2568b76417aebc0e3c20003a8120b2d2adab3eb Mon Sep 17 00:00:00 2001 From: sboikov Date: Mon, 10 Jul 2017 14:41:04 +0300 Subject: [PATCH 120/155] ignite-5075 Rename cacheId -> groupId --- .../ignite/internal/pagemem/FullPageId.java | 32 ++++---- .../ignite/internal/pagemem/PageSupport.java | 36 ++++----- .../pagemem/store/IgnitePageStoreManager.java | 40 +++++----- .../delta/DataPageInsertFragmentRecord.java | 6 +- .../record/delta/DataPageInsertRecord.java | 6 +- .../record/delta/DataPageRemoveRecord.java | 6 +- .../delta/DataPageSetFreeListPageRecord.java | 6 +- .../record/delta/DataPageUpdateRecord.java | 6 +- .../wal/record/delta/FixCountRecord.java | 6 +- .../record/delta/FixLeftmostChildRecord.java | 6 +- .../pagemem/wal/record/delta/FixRemoveId.java | 6 +- .../wal/record/delta/InitNewPageRecord.java | 6 +- .../wal/record/delta/InnerReplaceRecord.java | 6 +- .../wal/record/delta/InsertRecord.java | 6 +- .../pagemem/wal/record/delta/MergeRecord.java | 6 +- .../record/delta/MetaPageAddRootRecord.java | 6 +- .../record/delta/MetaPageCutRootRecord.java | 6 +- .../wal/record/delta/MetaPageInitRecord.java | 6 +- .../delta/MetaPageInitRootInlineRecord.java | 6 +- .../record/delta/MetaPageInitRootRecord.java | 4 +- .../MetaPageUpdateLastAllocatedIndex.java | 4 +- ...ageUpdateLastSuccessfulFullSnapshotId.java | 4 +- ...etaPageUpdateLastSuccessfulSnapshotId.java | 4 +- .../delta/MetaPageUpdateNextSnapshotId.java | 4 +- .../MetaPageUpdatePartitionDataRecord.java | 6 +- .../wal/record/delta/NewRootInitRecord.java | 6 +- .../wal/record/delta/PageDeltaRecord.java | 14 ++-- .../delta/PageListMetaResetCountRecord.java | 6 +- .../record/delta/PagesListAddPageRecord.java | 6 +- .../delta/PagesListInitNewPageRecord.java | 6 +- .../delta/PagesListRemovePageRecord.java | 2 +- .../record/delta/PagesListSetNextRecord.java | 6 +- .../delta/PagesListSetPreviousRecord.java | 6 +- .../record/delta/PartitionDestroyRecord.java | 20 ++--- .../delta/PartitionMetaStateRecord.java | 16 ++-- .../wal/record/delta/RecycleRecord.java | 6 +- .../wal/record/delta/RemoveRecord.java | 6 +- .../wal/record/delta/ReplaceRecord.java | 6 +- .../record/delta/SplitExistingPageRecord.java | 6 +- .../record/delta/SplitForwardPageRecord.java | 6 +- .../record/delta/TrackingPageDeltaRecord.java | 6 +- .../cache/persistence/DataStructure.java | 46 +++++------ .../GridCacheDatabaseSharedManager.java | 38 +++++----- .../persistence/GridCacheOffheapManager.java | 4 +- .../cache/persistence/MetadataStorage.java | 2 +- .../file/FilePageStoreManager.java | 34 ++++----- .../persistence/freelist/FreeListImpl.java | 6 +- .../cache/persistence/freelist/PagesList.java | 36 ++++----- .../persistence/pagemem/FullPageIdTable.java | 54 ++++++------- .../persistence/pagemem/PageMemoryEx.java | 4 +- .../persistence/pagemem/PageMemoryImpl.java | 72 +++++++++--------- .../cache/persistence/tree/BPlusTree.java | 16 ++-- .../wal/serializer/RecordV1Serializer.java | 76 +++++++++---------- .../impl/PageMemoryNoLoadSelfTest.java | 52 ++++++------- ...itePdsRecoveryAfterFileCorruptionTest.java | 24 +++--- ...pointSimulationWithRealCpDisabledTest.java | 74 +++++++++--------- .../db/file/IgnitePdsEvictionTest.java | 16 ++-- .../db/wal/IgniteWalRecoveryTest.java | 10 +-- .../pagemem/NoOpPageStoreManager.java | 22 +++--- 59 files changed, 470 insertions(+), 470 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java index 20677ea02b26c..00f52c1250963 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java @@ -56,28 +56,28 @@ public class FullPageId { private final long effectivePageId; /** */ - private final int cacheId; + private final int grpId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @return Hash code. */ - public static int hashCode(int cacheId, long pageId) { + public static int hashCode(int grpId, long pageId) { long effectiveId = PageIdUtils.effectivePageId(pageId); - return U.hash(hashCode0(cacheId, effectiveId)); + return U.hash(hashCode0(grpId, effectiveId)); } /** * Will not clear link bits. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param effectivePageId Effective page ID. * @return Hash code. */ - private static int hashCode0(int cacheId, long effectivePageId) { - return (int)(mix64(effectivePageId) ^ mix32(cacheId)); + private static int hashCode0(int grpId, long effectivePageId) { + return (int)(mix64(effectivePageId) ^ mix32(grpId)); } /** @@ -107,11 +107,11 @@ private static long mix64(long z) { } /** * @param pageId Page ID. - * @param cacheId Cache ID. + * @param grpId Cache group ID. */ - public FullPageId(long pageId, int cacheId) { + public FullPageId(long pageId, int grpId) { this.pageId = pageId; - this.cacheId = cacheId; + this.grpId = grpId; effectivePageId = PageIdUtils.effectivePageId(pageId); } @@ -124,10 +124,10 @@ public long pageId() { } /** - * @return Cache ID. + * @return Cache group ID. */ - public int cacheId() { - return cacheId; + public int groupId() { + return grpId; } /** {@inheritDoc} */ @@ -140,18 +140,18 @@ public int cacheId() { FullPageId that = (FullPageId)o; - return effectivePageId == that.effectivePageId && cacheId == that.cacheId; + return effectivePageId == that.effectivePageId && grpId == that.grpId; } /** {@inheritDoc} */ @Override public int hashCode() { - return hashCode0(cacheId, effectivePageId); + return hashCode0(grpId, effectivePageId); } /** {@inheritDoc} */ @Override public String toString() { return new SB("FullPageId [pageId=").appendHex(pageId) .a(", effectivePageId=").appendHex(effectivePageId) - .a(", cacheId=").a(cacheId).a(']').toString(); + .a(", grpId=").a(grpId).a(']').toString(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageSupport.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageSupport.java index f4b2d96d4901e..ed2311f3ebd26 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageSupport.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageSupport.java @@ -28,87 +28,87 @@ public interface PageSupport { * released by calling {@link #releasePage(int, long, long)}. This method will allocate page with given ID if it doesn't * exist. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @return Page pointer. * @throws IgniteCheckedException If failed. */ - public long acquirePage(int cacheId, long pageId) throws IgniteCheckedException; + public long acquirePage(int grpId, long pageId) throws IgniteCheckedException; /** * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID to release. * @param page Page pointer. */ - public void releasePage(int cacheId, long pageId, long page); + public void releasePage(int grpId, long pageId, long page); /** * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param page Page pointer. * @return Pointer for reading the page. */ - public long readLock(int cacheId, long pageId, long page); + public long readLock(int grpId, long pageId, long page); /** * Obtains read lock without checking page tag. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param page Page pointer. * @return Pointer for reading the page. */ - public long readLockForce(int cacheId, long pageId, long page); + public long readLockForce(int grpId, long pageId, long page); /** * Releases locked page. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param page Page pointer. */ - public void readUnlock(int cacheId, long pageId, long page); + public void readUnlock(int grpId, long pageId, long page); /** * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param page Page pointer. * @return Address of a buffer with contents of the given page or * {@code 0L} if attempt to take the write lock failed. */ - public long writeLock(int cacheId, long pageId, long page); + public long writeLock(int grpId, long pageId, long page); /** * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param page Page pointer. * @return Address of a buffer with contents of the given page or * {@code 0L} if attempt to take the write lock failed. */ - public long tryWriteLock(int cacheId, long pageId, long page); + public long tryWriteLock(int grpId, long pageId, long page); /** * Releases locked page. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param page Page pointer. * @param walPlc {@code True} if page should be recorded to WAL, {@code false} if the page must not * be recorded and {@code null} for the default behavior. * @param dirtyFlag Determines whether the page was modified since the last checkpoint. */ - public void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc, + public void writeUnlock(int grpId, long pageId, long page, Boolean walPlc, boolean dirtyFlag); /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param page Page pointer. * @return {@code True} if the page is dirty. */ - public boolean isDirty(int cacheId, long pageId, long page); + public boolean isDirty(int grpId, long pageId, long page); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java index fa6e9e410e7f8..a1b766f7489a7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java @@ -49,8 +49,8 @@ public interface IgnitePageStoreManager extends GridCacheSharedManager, IgniteCh * @param cacheData Cache data of the cache being started. * @throws IgniteCheckedException If failed to handle cache start callback. */ - public void initializeForCache(CacheGroupDescriptor grpDesc, - StoredCacheData cacheData) throws IgniteCheckedException; + public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cacheData) + throws IgniteCheckedException; /** * Callback called when a cache is stopping. After this callback is invoked, no data associated with @@ -84,12 +84,12 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, /** * Reads a page for the given cache ID. Cache ID may be {@code 0} if the page is a meta page. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId PageID to read. * @param pageBuf Page buffer to write to. * @throws IgniteCheckedException If failed to read the page. */ - public void read(int cacheId, long pageId, ByteBuffer pageBuf) throws IgniteCheckedException; + public void read(int grpId, long pageId, ByteBuffer pageBuf) throws IgniteCheckedException; /** * Checks if partition store exists. @@ -104,77 +104,77 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, /** * Reads a header of a page store. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param partId Partition ID. * @param buf Buffer to write to. * @throws IgniteCheckedException If failed. */ - public void readHeader(int cacheId, int partId, ByteBuffer buf) throws IgniteCheckedException; + public void readHeader(int grpId, int partId, ByteBuffer buf) throws IgniteCheckedException; /** * Writes the page for the given cache ID. Cache ID may be {@code 0} if the page is a meta page. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param pageBuf Page buffer to write. * @throws IgniteCheckedException If failed to write page. */ - public void write(int cacheId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException; + public void write(int grpId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException; /** * Gets page offset within the page store file. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @return Page offset. * @throws IgniteCheckedException If failed. */ - public long pageOffset(int cacheId, long pageId) throws IgniteCheckedException; + public long pageOffset(int grpId, long pageId) throws IgniteCheckedException; /** * Makes sure that all previous writes to the store has been written to disk. * - * @param cacheId Cache ID to sync. + * @param grpId Cache group ID to sync. * @param partId Partition ID to sync. * @throws IgniteCheckedException If IO error occurred while running sync. */ - public void sync(int cacheId, int partId) throws IgniteCheckedException; + public void sync(int grpId, int partId) throws IgniteCheckedException; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param partId Partition ID. * @throws IgniteCheckedException If failed. */ - public void ensure(int cacheId, int partId) throws IgniteCheckedException; + public void ensure(int grpId, int partId) throws IgniteCheckedException; /** * Allocates a page for the given page space. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param partId Partition ID. Used only if {@code flags} is equal to {@link PageMemory#FLAG_DATA}. * @param flags Page allocation flags. * @return Allocated page ID. * @throws IgniteCheckedException If IO exception occurred while allocating a page ID. */ - public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException; + public long allocatePage(int grpId, int partId, byte flags) throws IgniteCheckedException; /** * Gets total number of allocated pages for the given space. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param partId Partition ID. * @return Number of allocated pages. * @throws IgniteCheckedException If failed. */ - public int pages(int cacheId, int partId) throws IgniteCheckedException; + public int pages(int grpId, int partId) throws IgniteCheckedException; /** * Gets meta page ID for specified cache. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @return Meta page ID. */ - public long metaPageId(int cacheId); + public long metaPageId(int grpId); /** * @return Saved cache configurations. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java index 042fbe473d083..3b2ced1d5b72d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java @@ -32,18 +32,18 @@ public class DataPageInsertFragmentRecord extends PageDeltaRecord { private final byte[] payload; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param payload Fragment payload. * @param lastLink Link to the last entry fragment. */ public DataPageInsertFragmentRecord( - final int cacheId, + final int grpId, final long pageId, final byte[] payload, final long lastLink ) { - super(cacheId, pageId); + super(grpId, pageId); this.lastLink = lastLink; this.payload = payload; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java index 17425feb579a3..e480cc82d681f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java @@ -29,16 +29,16 @@ public class DataPageInsertRecord extends PageDeltaRecord { private byte[] payload; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param payload Remainder of the record. */ public DataPageInsertRecord( - int cacheId, + int grpId, long pageId, byte[] payload ) { - super(cacheId, pageId); + super(grpId, pageId); this.payload = payload; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java index 3aabe08a89a00..b80600c97b468 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java @@ -30,12 +30,12 @@ public class DataPageRemoveRecord extends PageDeltaRecord { private int itemId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param itemId Item ID. */ - public DataPageRemoveRecord(int cacheId, long pageId, int itemId) { - super(cacheId, pageId); + public DataPageRemoveRecord(int grpId, long pageId, int itemId) { + super(grpId, pageId); this.itemId = itemId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java index 82281de664e63..a7fd31f0d954d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java @@ -31,12 +31,12 @@ public class DataPageSetFreeListPageRecord extends PageDeltaRecord { private long freeListPage; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param freeListPage Free list page ID. */ - public DataPageSetFreeListPageRecord(int cacheId, long pageId, long freeListPage) { - super(cacheId, pageId); + public DataPageSetFreeListPageRecord(int grpId, long pageId, long freeListPage) { + super(grpId, pageId); this.freeListPage = freeListPage; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java index d4e2811ad13ae..6207f41fbf9ea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java @@ -32,18 +32,18 @@ public class DataPageUpdateRecord extends PageDeltaRecord { private byte[] payload; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param itemId Item ID. * @param payload Record data. */ public DataPageUpdateRecord( - int cacheId, + int grpId, long pageId, int itemId, byte[] payload ) { - super(cacheId, pageId); + super(grpId, pageId); this.payload = payload; this.itemId = itemId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixCountRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixCountRecord.java index 3ab1c4a04a234..80e06adb5d364 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixCountRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixCountRecord.java @@ -30,11 +30,11 @@ public class FixCountRecord extends PageDeltaRecord { private int cnt; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. */ - public FixCountRecord(int cacheId, long pageId, int cnt) { - super(cacheId, pageId); + public FixCountRecord(int grpId, long pageId, int cnt) { + super(grpId, pageId); this.cnt = cnt; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixLeftmostChildRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixLeftmostChildRecord.java index 15ee7dd8a0446..a34e8e3fe3471 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixLeftmostChildRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixLeftmostChildRecord.java @@ -30,12 +30,12 @@ public class FixLeftmostChildRecord extends PageDeltaRecord { private long rightId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param rightId Right ID. */ - public FixLeftmostChildRecord(int cacheId, long pageId, long rightId) { - super(cacheId, pageId); + public FixLeftmostChildRecord(int grpId, long pageId, long rightId) { + super(grpId, pageId); this.rightId = rightId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixRemoveId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixRemoveId.java index b6ea410a75546..1ec845d178e70 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixRemoveId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixRemoveId.java @@ -30,12 +30,12 @@ public class FixRemoveId extends PageDeltaRecord { private long rmvId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param rmvId Remove ID. */ - public FixRemoveId(int cacheId, long pageId, long rmvId) { - super(cacheId, pageId); + public FixRemoveId(int grpId, long pageId, long rmvId) { + super(grpId, pageId); this.rmvId = rmvId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java index f883e068daea9..15bd2da5c6491 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java @@ -39,14 +39,14 @@ public class InitNewPageRecord extends PageDeltaRecord { protected long newPageId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param ioType IO type. * @param ioVer IO version. * @param newPageId New page ID. */ - public InitNewPageRecord(int cacheId, long pageId, int ioType, int ioVer, long newPageId) { - super(cacheId, pageId); + public InitNewPageRecord(int grpId, long pageId, int ioType, int ioVer, long newPageId) { + super(grpId, pageId); this.ioType = ioType; this.ioVer = ioVer; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InnerReplaceRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InnerReplaceRecord.java index 35d23c19c8204..6e6879dc0e366 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InnerReplaceRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InnerReplaceRecord.java @@ -38,14 +38,14 @@ public class InnerReplaceRecord extends PageDeltaRecord { private long rmvId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param dstIdx Destination index. * @param srcPageId Source page ID. * @param srcIdx Source index. */ - public InnerReplaceRecord(int cacheId, long pageId, int dstIdx, long srcPageId, int srcIdx, long rmvId) { - super(cacheId, pageId); + public InnerReplaceRecord(int grpId, long pageId, int dstIdx, long srcPageId, int srcIdx, long rmvId) { + super(grpId, pageId); this.dstIdx = dstIdx; this.srcPageId = srcPageId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InsertRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InsertRecord.java index d0e66a3343f30..f6ebe870902be 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InsertRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InsertRecord.java @@ -42,7 +42,7 @@ public class InsertRecord extends PageDeltaRecord { private BPlusIO io; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param io IO. * @param idx Index. @@ -50,14 +50,14 @@ public class InsertRecord extends PageDeltaRecord { * @param rightId Right ID. */ public InsertRecord( - int cacheId, + int grpId, long pageId, BPlusIO io, int idx, byte[] rowBytes, long rightId ) { - super(cacheId, pageId); + super(grpId, pageId); this.io = io; this.idx = idx; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MergeRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MergeRecord.java index 84770d5403a18..2d481d1cda67f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MergeRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MergeRecord.java @@ -44,15 +44,15 @@ public class MergeRecord extends PageDeltaRecord { private boolean emptyBranch; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param prntId Parent ID. * @param prntIdx Index in parent page. * @param rightId Right ID. * @param emptyBranch We are merging empty branch. */ - public MergeRecord(int cacheId, long pageId, long prntId, int prntIdx, long rightId, boolean emptyBranch) { - super(cacheId, pageId); + public MergeRecord(int grpId, long pageId, long prntId, int prntIdx, long rightId, boolean emptyBranch) { + super(grpId, pageId); this.prntId = prntId; this.rightId = rightId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageAddRootRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageAddRootRecord.java index 1f1e97697f3d4..ce9e06b98ed7e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageAddRootRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageAddRootRecord.java @@ -29,12 +29,12 @@ public class MetaPageAddRootRecord extends PageDeltaRecord { private long rootId; /** - * @param cacheId Cache ID. + * @param grpId Cache ID. * @param pageId Page ID. * @param rootId Root ID. */ - public MetaPageAddRootRecord(int cacheId, long pageId, long rootId) { - super(cacheId, pageId); + public MetaPageAddRootRecord(int grpId, long pageId, long rootId) { + super(grpId, pageId); this.rootId = rootId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageCutRootRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageCutRootRecord.java index 3fc42a15dfffd..1e93ab9230227 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageCutRootRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageCutRootRecord.java @@ -26,11 +26,11 @@ */ public class MetaPageCutRootRecord extends PageDeltaRecord { /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. */ - public MetaPageCutRootRecord(int cacheId, long pageId) { - super(cacheId, pageId); + public MetaPageCutRootRecord(int grpId, long pageId) { + super(grpId, pageId); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRecord.java index 692b8f1782d2f..b1e701aa67324 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRecord.java @@ -36,14 +36,14 @@ public class MetaPageInitRecord extends InitNewPageRecord { private int ioType; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param ioType IO type. * @param treeRoot Tree root. * @param reuseListRoot Reuse list root. */ - public MetaPageInitRecord(int cacheId, long pageId, int ioType, int ioVer, long treeRoot, long reuseListRoot) { - super(cacheId, pageId, ioType, ioVer, pageId); + public MetaPageInitRecord(int grpId, long pageId, int ioType, int ioVer, long treeRoot, long reuseListRoot) { + super(grpId, pageId, ioType, ioVer, pageId); assert ioType == PageIO.T_META || ioType == PageIO.T_PART_META; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootInlineRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootInlineRecord.java index 7eb8426dd38c2..2b82444f40df9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootInlineRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootInlineRecord.java @@ -30,13 +30,13 @@ public class MetaPageInitRootInlineRecord extends MetaPageInitRootRecord { private final int inlineSize; /** - * @param cacheId + * @param grpId Cache group ID. * @param pageId Meta page ID. * @param rootId * @param inlineSize Inline size. */ - public MetaPageInitRootInlineRecord(int cacheId, long pageId, long rootId, int inlineSize) { - super(cacheId, pageId, rootId); + public MetaPageInitRootInlineRecord(int grpId, long pageId, long rootId, int inlineSize) { + super(grpId, pageId, rootId); this.inlineSize = inlineSize; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootRecord.java index 0eb61b577fdb5..ebdf3d4b403fb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootRecord.java @@ -31,8 +31,8 @@ public class MetaPageInitRootRecord extends PageDeltaRecord { /** * @param pageId Meta page ID. */ - public MetaPageInitRootRecord(int cacheId, long pageId, long rootId) { - super(cacheId, pageId); + public MetaPageInitRootRecord(int grpId, long pageId, long rootId) { + super(grpId, pageId); assert pageId != rootId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java index 548735eceb226..60aebdeeb4aef 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java @@ -32,8 +32,8 @@ public class MetaPageUpdateLastAllocatedIndex extends PageDeltaRecord { /** * @param pageId Meta page ID. */ - public MetaPageUpdateLastAllocatedIndex(int cacheId, long pageId, int lastAllocatedIdx) { - super(cacheId, pageId); + public MetaPageUpdateLastAllocatedIndex(int grpId, long pageId, int lastAllocatedIdx) { + super(grpId, pageId); this.lastAllocatedIdx = lastAllocatedIdx; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulFullSnapshotId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulFullSnapshotId.java index 34be353f4a580..03f945f8c005e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulFullSnapshotId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulFullSnapshotId.java @@ -31,8 +31,8 @@ public class MetaPageUpdateLastSuccessfulFullSnapshotId extends PageDeltaRecord /** * @param pageId Meta page ID. */ - public MetaPageUpdateLastSuccessfulFullSnapshotId(int cacheId, long pageId, long lastSuccessfulFullSnapshotId) { - super(cacheId, pageId); + public MetaPageUpdateLastSuccessfulFullSnapshotId(int grpId, long pageId, long lastSuccessfulFullSnapshotId) { + super(grpId, pageId); this.lastSuccessfulFullSnapshotId = lastSuccessfulFullSnapshotId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulSnapshotId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulSnapshotId.java index 7fca5dc9516e0..07798bbac5a75 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulSnapshotId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulSnapshotId.java @@ -34,8 +34,8 @@ public class MetaPageUpdateLastSuccessfulSnapshotId extends PageDeltaRecord { * @param pageId Meta page ID. * @param snapshotTag */ - public MetaPageUpdateLastSuccessfulSnapshotId(int cacheId, long pageId, long lastSuccessfulSnapshotId, long snapshotTag) { - super(cacheId, pageId); + public MetaPageUpdateLastSuccessfulSnapshotId(int grpId, long pageId, long lastSuccessfulSnapshotId, long snapshotTag) { + super(grpId, pageId); this.lastSuccessfulSnapshotId = lastSuccessfulSnapshotId; this.lastSuccessfulSnapshotTag = snapshotTag; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java index 0cd72ece5ca34..e2d12673c5395 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java @@ -31,8 +31,8 @@ public class MetaPageUpdateNextSnapshotId extends PageDeltaRecord { /** * @param pageId Meta page ID. */ - public MetaPageUpdateNextSnapshotId(int cacheId, long pageId, long nextSnapshotId) { - super(cacheId, pageId); + public MetaPageUpdateNextSnapshotId(int grpId, long pageId, long nextSnapshotId) { + super(grpId, pageId); this.nextSnapshotId = nextSnapshotId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java index 15d1a0ce084bf..f85f46801de68 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java @@ -45,12 +45,12 @@ public class MetaPageUpdatePartitionDataRecord extends PageDeltaRecord { private long cntrsPageId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param allocatedIdxCandidate Page Allocated index candidate */ public MetaPageUpdatePartitionDataRecord( - int cacheId, + int grpId, long pageId, long updateCntr, long globalRmvId, @@ -58,7 +58,7 @@ public MetaPageUpdatePartitionDataRecord( long cntrsPageId, byte state, int allocatedIdxCandidate ) { - super(cacheId, pageId); + super(grpId, pageId); this.updateCntr = updateCntr; this.globalRmvId = globalRmvId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/NewRootInitRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/NewRootInitRecord.java index bdfdef5ea025b..f39f91c6a49dd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/NewRootInitRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/NewRootInitRecord.java @@ -41,7 +41,7 @@ public class NewRootInitRecord extends PageDeltaRecord { private long rightChildId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param io IO. * @param leftChildId Left child ID. @@ -49,7 +49,7 @@ public class NewRootInitRecord extends PageDeltaRecord { * @param rightChildId Right child ID. */ public NewRootInitRecord( - int cacheId, + int grpId, long pageId, long newRootId, BPlusInnerIO io, @@ -57,7 +57,7 @@ public NewRootInitRecord( byte[] rowBytes, long rightChildId ) { - super(cacheId, pageId); + super(grpId, pageId); assert io != null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageDeltaRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageDeltaRecord.java index ca52bd2277d28..260c2c84cf6df 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageDeltaRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageDeltaRecord.java @@ -29,18 +29,18 @@ */ public abstract class PageDeltaRecord extends WALRecord { /** */ - private int cacheId; + private int grpId; /** */ @GridToStringExclude private long pageId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. */ - protected PageDeltaRecord(int cacheId, long pageId) { - this.cacheId = cacheId; + protected PageDeltaRecord(int grpId, long pageId) { + this.grpId = grpId; this.pageId = pageId; } @@ -52,10 +52,10 @@ public long pageId() { } /** - * @return Cache ID. + * @return Cache group ID. */ - public int cacheId() { - return cacheId; + public int groupId() { + return grpId; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageListMetaResetCountRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageListMetaResetCountRecord.java index 4756c9af60634..9c12380267f8f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageListMetaResetCountRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageListMetaResetCountRecord.java @@ -26,11 +26,11 @@ */ public class PageListMetaResetCountRecord extends PageDeltaRecord { /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. */ - public PageListMetaResetCountRecord(int cacheId, long pageId) { - super(cacheId, pageId); + public PageListMetaResetCountRecord(int grpId, long pageId) { + super(grpId, pageId); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListAddPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListAddPageRecord.java index 2577ec33c5d09..6c7fc71080de9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListAddPageRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListAddPageRecord.java @@ -33,12 +33,12 @@ public class PagesListAddPageRecord extends PageDeltaRecord { private final long dataPageId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param dataPageId Data page ID to add. */ - public PagesListAddPageRecord(int cacheId, long pageId, long dataPageId) { - super(cacheId, pageId); + public PagesListAddPageRecord(int grpId, long pageId, long dataPageId) { + super(grpId, pageId); this.dataPageId = dataPageId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListInitNewPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListInitNewPageRecord.java index 673d33c0987a2..ee83b8be38b40 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListInitNewPageRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListInitNewPageRecord.java @@ -38,13 +38,13 @@ public class PagesListInitNewPageRecord extends InitNewPageRecord { private final long addDataPageId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param prevPageId Previous page ID. * @param addDataPageId Optional page ID to add. */ public PagesListInitNewPageRecord( - int cacheId, + int grpId, long pageId, int ioType, int ioVer, @@ -52,7 +52,7 @@ public PagesListInitNewPageRecord( long prevPageId, long addDataPageId ) { - super(cacheId, pageId, ioType, ioVer, newPageId); + super(grpId, pageId, ioType, ioVer, newPageId); this.prevPageId = prevPageId; this.addDataPageId = addDataPageId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListRemovePageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListRemovePageRecord.java index 23411295e28fd..8f2cd72275e82 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListRemovePageRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListRemovePageRecord.java @@ -69,6 +69,6 @@ public long removedPageId() { return S.toString(PagesListRemovePageRecord.class, this, "rmvdPageId", U.hexLong(rmvdPageId), "pageId", U.hexLong(pageId()), - "cacheId", cacheId()); + "grpId", groupId()); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetNextRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetNextRecord.java index 30600e17447c0..f46ac8eaaa7cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetNextRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetNextRecord.java @@ -29,12 +29,12 @@ public class PagesListSetNextRecord extends PageDeltaRecord { private final long nextPageId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param nextPageId Next page ID. */ - public PagesListSetNextRecord(int cacheId, long pageId, long nextPageId) { - super(cacheId, pageId); + public PagesListSetNextRecord(int grpId, long pageId, long nextPageId) { + super(grpId, pageId); this.nextPageId = nextPageId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetPreviousRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetPreviousRecord.java index 590643e094145..cc270771a423a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetPreviousRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetPreviousRecord.java @@ -29,12 +29,12 @@ public class PagesListSetPreviousRecord extends PageDeltaRecord { private final long prevPageId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param prevPageId Previous page ID. */ - public PagesListSetPreviousRecord(int cacheId, long pageId, long prevPageId) { - super(cacheId, pageId); + public PagesListSetPreviousRecord(int grpId, long pageId, long prevPageId) { + super(grpId, pageId); this.prevPageId = prevPageId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionDestroyRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionDestroyRecord.java index c3b82009f29b2..834dc58d01412 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionDestroyRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionDestroyRecord.java @@ -24,17 +24,17 @@ */ public class PartitionDestroyRecord extends WALRecord { /** */ - private int cacheId; + private int grpId; /** */ private int partId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param partId Partition ID. */ - public PartitionDestroyRecord(int cacheId, int partId) { - this.cacheId = cacheId; + public PartitionDestroyRecord(int grpId, int partId) { + this.grpId = grpId; this.partId = partId; } @@ -44,17 +44,17 @@ public PartitionDestroyRecord(int cacheId, int partId) { } /** - * @return Cache ID. + * @return Cache group ID. */ - public int cacheId() { - return cacheId; + public int groupId() { + return grpId; } /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. */ - public void cacheId(int cacheId) { - this.cacheId = cacheId; + public void groupId(int grpId) { + this.grpId = grpId; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java index 95e1a561d28f3..8ab794ce8e0cc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java @@ -27,8 +27,8 @@ public class PartitionMetaStateRecord extends WALRecord { /** State. */ private final byte state; - /** Cache id. */ - private final int cacheId; + /** Cache group ID. */ + private final int grpId; /** Partition id. */ private final int partId; @@ -37,11 +37,11 @@ public class PartitionMetaStateRecord extends WALRecord { private final long updateCounter; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param state Page ID. */ - public PartitionMetaStateRecord(int cacheId, int partId, GridDhtPartitionState state, long updateCounter) { - this.cacheId = cacheId; + public PartitionMetaStateRecord(int grpId, int partId, GridDhtPartitionState state, long updateCounter) { + this.grpId = grpId; this.partId = partId; this.state = (byte)state.ordinal(); this.updateCounter = updateCounter; @@ -60,10 +60,10 @@ public byte state() { } /** - * @return Cache ID. + * @return Cache group ID. */ - public int cacheId() { - return cacheId; + public int groupId() { + return grpId; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RecycleRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RecycleRecord.java index 92ea88e819983..b505e4d971356 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RecycleRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RecycleRecord.java @@ -29,12 +29,12 @@ public class RecycleRecord extends PageDeltaRecord { private long newPageId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param newPageId New page ID. */ - public RecycleRecord(int cacheId, long pageId, long newPageId) { - super(cacheId, pageId); + public RecycleRecord(int grpId, long pageId, long newPageId) { + super(grpId, pageId); this.newPageId = newPageId; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RemoveRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RemoveRecord.java index ac5d442aa30b0..53f9cb0886bed 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RemoveRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RemoveRecord.java @@ -34,13 +34,13 @@ public class RemoveRecord extends PageDeltaRecord { private int cnt; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param idx Index. * @param cnt Count. */ - public RemoveRecord(int cacheId, long pageId, int idx, int cnt) { - super(cacheId, pageId); + public RemoveRecord(int grpId, long pageId, int idx, int cnt) { + super(grpId, pageId); this.idx = idx; this.cnt = cnt; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/ReplaceRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/ReplaceRecord.java index 2f504adce9c42..2fbc0ddc76a55 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/ReplaceRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/ReplaceRecord.java @@ -35,14 +35,14 @@ public class ReplaceRecord extends PageDeltaRecord { private int idx; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param io IO. * @param rowBytes Row bytes. * @param idx Index. */ - public ReplaceRecord(int cacheId, long pageId, BPlusIO io, byte[] rowBytes, int idx) { - super(cacheId, pageId); + public ReplaceRecord(int grpId, long pageId, BPlusIO io, byte[] rowBytes, int idx) { + super(grpId, pageId); this.io = io; this.rowBytes = rowBytes; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitExistingPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitExistingPageRecord.java index 5cb3023119296..87638a69e4039 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitExistingPageRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitExistingPageRecord.java @@ -37,13 +37,13 @@ public class SplitExistingPageRecord extends PageDeltaRecord { private long fwdId; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @param mid Bisection index. * @param fwdId New forward page ID. */ - public SplitExistingPageRecord(int cacheId, long pageId, int mid, long fwdId) { - super(cacheId, pageId); + public SplitExistingPageRecord(int grpId, long pageId, int mid, long fwdId) { + super(grpId, pageId); this.mid = mid; this.fwdId = fwdId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitForwardPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitForwardPageRecord.java index 39f2669ce6c61..7f855f7e3bb11 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitForwardPageRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitForwardPageRecord.java @@ -45,7 +45,7 @@ public class SplitForwardPageRecord extends PageDeltaRecord { private int cnt; /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Real forward page ID. * @param fwdId Virtual forward page ID. * @param ioType IO Type. @@ -55,7 +55,7 @@ public class SplitForwardPageRecord extends PageDeltaRecord { * @param cnt Initial elements count in the page being split. */ public SplitForwardPageRecord( - int cacheId, + int grpId, long pageId, long fwdId, int ioType, @@ -64,7 +64,7 @@ public SplitForwardPageRecord( int mid, int cnt ) { - super(cacheId, pageId); + super(grpId, pageId); this.fwdId = fwdId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/TrackingPageDeltaRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/TrackingPageDeltaRecord.java index 05e96ff99367d..3d88e8cb4ae82 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/TrackingPageDeltaRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/TrackingPageDeltaRecord.java @@ -35,13 +35,13 @@ public class TrackingPageDeltaRecord extends PageDeltaRecord { private final long lastSuccessfulSnapshotId; /** - * @param cacheId Cache id. + * @param grpId Cache group id. * @param pageId Page id. * @param nextSnapshotId * @param lastSuccessfulSnapshotId */ - public TrackingPageDeltaRecord(int cacheId, long pageId, long pageIdToMark, long nextSnapshotId, long lastSuccessfulSnapshotId) { - super(cacheId, pageId); + public TrackingPageDeltaRecord(int grpId, long pageId, long pageIdToMark, long nextSnapshotId, long lastSuccessfulSnapshotId) { + super(grpId, pageId); this.pageIdToMark = pageIdToMark; this.nextSnapshotId = nextSnapshotId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java index b2a8f3636bd16..9161d69b3bfe8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java @@ -45,7 +45,7 @@ public abstract class DataStructure implements PageLockListener { public static Random rnd; /** */ - protected final int cacheId; + protected final int grpId; /** */ protected final PageMemory pageMem; @@ -57,7 +57,7 @@ public abstract class DataStructure implements PageLockListener { protected ReuseList reuseList; /** - * @param cacheId Cache ID. + * @param cacheId Cache group ID. * @param pageMem Page memory. * @param wal Write ahead log manager. */ @@ -68,16 +68,16 @@ public DataStructure( ) { assert pageMem != null; - this.cacheId = cacheId; + this.grpId = cacheId; this.pageMem = pageMem; this.wal = wal; } /** - * @return Cache ID. + * @return Cache group ID. */ - public final int getCacheId() { - return cacheId; + public final int groupId() { + return grpId; } /** @@ -114,7 +114,7 @@ protected final long allocatePage(ReuseBag bag) throws IgniteCheckedException { * @throws IgniteCheckedException If failed. */ protected long allocatePageNoReuse() throws IgniteCheckedException { - return pageMem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, FLAG_IDX); + return pageMem.allocatePage(grpId, PageIdAllocator.INDEX_PARTITION, FLAG_IDX); } /** @@ -126,7 +126,7 @@ protected final long acquirePage(long pageId) throws IgniteCheckedException { assert PageIdUtils.flag(pageId) == FLAG_IDX && PageIdUtils.partId(pageId) == INDEX_PARTITION || PageIdUtils.flag(pageId) == FLAG_DATA && PageIdUtils.partId(pageId) <= MAX_PARTITION_ID : U.hexLong(pageId); - return pageMem.acquirePage(cacheId, pageId); + return pageMem.acquirePage(grpId, pageId); } /** @@ -134,7 +134,7 @@ protected final long acquirePage(long pageId) throws IgniteCheckedException { * @param page Page pointer. */ protected final void releasePage(long pageId, long page) { - pageMem.releasePage(cacheId, pageId, page); + pageMem.releasePage(grpId, pageId, page); } /** @@ -143,7 +143,7 @@ protected final void releasePage(long pageId, long page) { * @return Page address or {@code 0} if failed to lock due to recycling. */ protected final long tryWriteLock(long pageId, long page) { - return PageHandler.writeLock(pageMem, cacheId, pageId, page, this, true); + return PageHandler.writeLock(pageMem, grpId, pageId, page, this, true); } /** @@ -152,7 +152,7 @@ protected final long tryWriteLock(long pageId, long page) { * @return Page address. */ protected final long writeLock(long pageId, long page) { - return PageHandler.writeLock(pageMem, cacheId, pageId, page, this, false); + return PageHandler.writeLock(pageMem, grpId, pageId, page, this, false); } /** @@ -174,7 +174,7 @@ protected final void writeUnlock(long pageId, long page, long pageAddr, boolean * @return Page address. */ protected final long readLock(long pageId, long page) { - return PageHandler.readLock(pageMem, cacheId, pageId, page, this); + return PageHandler.readLock(pageMem, grpId, pageId, page, this); } /** @@ -183,7 +183,7 @@ protected final long readLock(long pageId, long page) { * @param pageAddr Page address. */ protected final void readUnlock(long pageId, long page, long pageAddr) { - PageHandler.readUnlock(pageMem, cacheId, pageId, page, pageAddr, this); + PageHandler.readUnlock(pageMem, grpId, pageId, page, pageAddr, this); } /** @@ -194,7 +194,7 @@ protected final void readUnlock(long pageId, long page, long pageAddr) { * @param dirty Dirty flag. */ protected final void writeUnlock(long pageId, long page, long pageAddr, Boolean walPlc, boolean dirty) { - PageHandler.writeUnlock(pageMem, cacheId, pageId, page, pageAddr, this, walPlc, dirty); + PageHandler.writeUnlock(pageMem, grpId, pageId, page, pageAddr, this, walPlc, dirty); } /** @@ -204,7 +204,7 @@ protected final void writeUnlock(long pageId, long page, long pageAddr, Boolean * @return {@code true} If we need to make a delta WAL record for the change in this page. */ protected final boolean needWalDeltaRecord(long pageId, long page, Boolean walPlc) { - return PageHandler.isWalDeltaRecordNeeded(pageMem, cacheId, pageId, page, wal, walPlc); + return PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, pageId, page, wal, walPlc); } /** @@ -220,7 +220,7 @@ protected final R write( PageHandler h, int intArg, R lockFailed) throws IgniteCheckedException { - return PageHandler.writePage(pageMem, cacheId, pageId, this, h, null, null, null, null, intArg, lockFailed); + return PageHandler.writePage(pageMem, grpId, pageId, this, h, null, null, null, null, intArg, lockFailed); } /** @@ -238,7 +238,7 @@ protected final R write( X arg, int intArg, R lockFailed) throws IgniteCheckedException { - return PageHandler.writePage(pageMem, cacheId, pageId, this, h, null, null, null, arg, intArg, lockFailed); + return PageHandler.writePage(pageMem, grpId, pageId, this, h, null, null, null, arg, intArg, lockFailed); } /** @@ -258,7 +258,7 @@ protected final R write( X arg, int intArg, R lockFailed) throws IgniteCheckedException { - return PageHandler.writePage(pageMem, cacheId, pageId, page, this, h, null, null, null, arg, intArg, lockFailed); + return PageHandler.writePage(pageMem, grpId, pageId, page, this, h, null, null, null, arg, intArg, lockFailed); } /** @@ -278,7 +278,7 @@ protected final R write( X arg, int intArg, R lockFailed) throws IgniteCheckedException { - return PageHandler.writePage(pageMem, cacheId, pageId, this, h, init, wal, null, arg, intArg, lockFailed); + return PageHandler.writePage(pageMem, grpId, pageId, this, h, init, wal, null, arg, intArg, lockFailed); } /** @@ -296,7 +296,7 @@ protected final R read( X arg, int intArg, R lockFailed) throws IgniteCheckedException { - return PageHandler.readPage(pageMem, cacheId, pageId, this, h, arg, intArg, lockFailed); + return PageHandler.readPage(pageMem, grpId, pageId, this, h, arg, intArg, lockFailed); } /** @@ -316,7 +316,7 @@ protected final R read( X arg, int intArg, R lockFailed) throws IgniteCheckedException { - return PageHandler.readPage(pageMem, cacheId, pageId, page, this, h, arg, intArg, lockFailed); + return PageHandler.readPage(pageMem, grpId, pageId, page, this, h, arg, intArg, lockFailed); } /** @@ -325,7 +325,7 @@ protected final R read( * @throws IgniteCheckedException if failed. */ protected final void init(long pageId, PageIO init) throws IgniteCheckedException { - PageHandler.initPage(pageMem, cacheId, pageId, init, wal, this); + PageHandler.initPage(pageMem, grpId, pageId, init, wal, this); } /** @@ -346,7 +346,7 @@ protected final long recyclePage( PageIO.setPageId(pageAddr, rotated); if (needWalDeltaRecord(pageId, page, walPlc)) - wal.log(new RecycleRecord(cacheId, pageId, rotated)); + wal.log(new RecycleRecord(grpId, pageId, rotated)); return rotated; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 8fe9377cfe79c..51367316ba8c5 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -623,7 +623,7 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize) { Integer tag ) throws IgniteCheckedException { // First of all, write page to disk. - storeMgr.write(fullId.cacheId(), fullId.pageId(), pageBuf, tag); + storeMgr.write(fullId.groupId(), fullId.pageId(), pageBuf, tag); // Only after write we can write page into snapshot. snapshotMgr.flushDirtyPageHandler(fullId, pageBuf, tag); @@ -1336,25 +1336,25 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId)) // Here we do not require tag check because we may be applying memory changes after // several repetitive restarts and the same pages may have changed several times. - int cacheId = pageRec.fullPageId().cacheId(); + int grpId = pageRec.fullPageId().groupId(); long pageId = pageRec.fullPageId().pageId(); - PageMemoryEx pageMem = getPageMemoryForCacheGroup(cacheId); + PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId); - long page = pageMem.acquirePage(cacheId, pageId, true); + long page = pageMem.acquirePage(grpId, pageId, true); try { - long pageAddr = pageMem.writeLock(cacheId, pageId, page); + long pageAddr = pageMem.writeLock(grpId, pageId, page); try { PageUtils.putBytes(pageAddr, 0, pageRec.pageData()); } finally { - pageMem.writeUnlock(cacheId, pageId, page, null, true, true); + pageMem.writeUnlock(grpId, pageId, page, null, true, true); } } finally { - pageMem.releasePage(cacheId, pageId, page); + pageMem.releasePage(grpId, pageId, page); } applied++; @@ -1366,14 +1366,14 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId)) if (apply) { PartitionDestroyRecord destroyRec = (PartitionDestroyRecord)rec; - final int cId = destroyRec.cacheId(); + final int gId = destroyRec.groupId(); final int pId = destroyRec.partitionId(); - PageMemoryEx pageMem = getPageMemoryForCacheGroup(cId); + PageMemoryEx pageMem = getPageMemoryForCacheGroup(gId); pageMem.clearAsync(new P3() { @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) { - return cacheId == cId && PageIdUtils.partId(pageId) == pId; + return cacheId == gId && PageIdUtils.partId(pageId) == pId; } }, true).get(); } @@ -1384,27 +1384,27 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId)) if (apply && rec instanceof PageDeltaRecord) { PageDeltaRecord r = (PageDeltaRecord)rec; - int cacheId = r.cacheId(); + int grpId = r.groupId(); long pageId = r.pageId(); - PageMemoryEx pageMem = getPageMemoryForCacheGroup(cacheId); + PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId); // Here we do not require tag check because we may be applying memory changes after // several repetitive restarts and the same pages may have changed several times. - long page = pageMem.acquirePage(cacheId, pageId, true); + long page = pageMem.acquirePage(grpId, pageId, true); try { - long pageAddr = pageMem.writeLock(cacheId, pageId, page); + long pageAddr = pageMem.writeLock(grpId, pageId, page); try { r.applyDelta(pageMem, pageAddr); } finally { - pageMem.writeUnlock(cacheId, pageId, page, null, true, true); + pageMem.writeUnlock(grpId, pageId, page, null, true, true); } } finally { - pageMem.releasePage(cacheId, pageId, page); + pageMem.releasePage(grpId, pageId, page); } applied++; @@ -1493,7 +1493,7 @@ private void applyLastUpdates(CheckpointStatus status) throws IgniteCheckedExcep case PART_META_UPDATE_STATE: PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec; - partStates.put(new T2<>(metaStateRecord.cacheId(), metaStateRecord.partitionId()), + partStates.put(new T2<>(metaStateRecord.groupId(), metaStateRecord.partitionId()), new T2<>((int)metaStateRecord.state(), metaStateRecord.updateCounter())); break; @@ -1686,7 +1686,7 @@ private void finalizeCheckpointOnRecovery(long cpTs, UUID cpId, WALPointer walPt if (tag != null) { tmpWriteBuf.rewind(); - PageStore store = storeMgr.writeInternal(fullId.cacheId(), fullId.pageId(), tmpWriteBuf, tag); + PageStore store = storeMgr.writeInternal(fullId.groupId(), fullId.pageId(), tmpWriteBuf, tag); tmpWriteBuf.rewind(); @@ -2326,7 +2326,7 @@ private WriteCheckpointPages( snapshotMgr.beforePageWrite(fullId); - int grpId = fullId.cacheId(); + int grpId = fullId.groupId(); CacheGroupContext grp = context().cache().cacheGroup(grpId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 4d30eb48ea830..bd902fb36f7da 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -888,7 +888,7 @@ private CacheDataStore init0(boolean checkExists) throws IgniteCheckedException reuseRoot.pageId().pageId(), reuseRoot.isAllocated()) { @Override protected long allocatePageNoReuse() throws IgniteCheckedException { - return pageMem.allocatePage(cacheId, partId, PageIdAllocator.FLAG_DATA); + return pageMem.allocatePage(grpId, partId, PageIdAllocator.FLAG_DATA); } }; @@ -904,7 +904,7 @@ private CacheDataStore init0(boolean checkExists) throws IgniteCheckedException treeRoot.pageId().pageId(), treeRoot.isAllocated()) { @Override protected long allocatePageNoReuse() throws IgniteCheckedException { - return pageMem.allocatePage(cacheId, partId, PageIdAllocator.FLAG_DATA); + return pageMem.allocatePage(grpId, partId, PageIdAllocator.FLAG_DATA); } }; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java index 806afb8d136f8..743f3b9099aee 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java @@ -186,7 +186,7 @@ private MetaTree( /** {@inheritDoc} */ @Override protected long allocatePageNoReuse() throws IgniteCheckedException { - return pageMem.allocatePage(getCacheId(), allocPartId, allocSpace); + return pageMem.allocatePage(groupId(), allocPartId, allocSpace); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index 4a56ec7e22dee..af20136b5be09 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -269,8 +269,8 @@ public FilePageStoreManager(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void read(int cacheId, long pageId, ByteBuffer pageBuf) throws IgniteCheckedException { - read(cacheId, pageId, pageBuf, false); + @Override public void read(int grpId, long pageId, ByteBuffer pageBuf) throws IgniteCheckedException { + read(grpId, pageId, pageBuf, false); } /** @@ -296,20 +296,20 @@ public void read(int cacheId, long pageId, ByteBuffer pageBuf, boolean keepCrc) } /** {@inheritDoc} */ - @Override public void readHeader(int cacheId, int partId, ByteBuffer buf) throws IgniteCheckedException { - PageStore store = getStore(cacheId, partId); + @Override public void readHeader(int grpId, int partId, ByteBuffer buf) throws IgniteCheckedException { + PageStore store = getStore(grpId, partId); store.readHeader(buf); } /** {@inheritDoc} */ - @Override public void write(int cacheId, long pageId, ByteBuffer pageBuf,int tag) throws IgniteCheckedException { - writeInternal(cacheId, pageId, pageBuf, tag); + @Override public void write(int grpId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException { + writeInternal(grpId, pageId, pageBuf, tag); } /** {@inheritDoc} */ - @Override public long pageOffset(int cacheId, long pageId) throws IgniteCheckedException { - PageStore store = getStore(cacheId, PageIdUtils.partId(pageId)); + @Override public long pageOffset(int grpId, long pageId) throws IgniteCheckedException { + PageStore store = getStore(grpId, PageIdUtils.partId(pageId)); return store.pageOffset(pageId); } @@ -447,20 +447,20 @@ private boolean checkAndInitCacheWorkDir(File cacheWorkDir) throws IgniteChecked } /** {@inheritDoc} */ - @Override public void sync(int cacheId, int partId) throws IgniteCheckedException { - getStore(cacheId, partId).sync(); + @Override public void sync(int grpId, int partId) throws IgniteCheckedException { + getStore(grpId, partId).sync(); } /** {@inheritDoc} */ - @Override public void ensure(int cacheId, int partId) throws IgniteCheckedException { - getStore(cacheId, partId).ensure(); + @Override public void ensure(int grpId, int partId) throws IgniteCheckedException { + getStore(grpId, partId).ensure(); } /** {@inheritDoc} */ - @Override public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException { + @Override public long allocatePage(int grpId, int partId, byte flags) throws IgniteCheckedException { assert partId <= PageIdAllocator.MAX_PARTITION_ID || partId == PageIdAllocator.INDEX_PARTITION; - PageStore store = getStore(cacheId, partId); + PageStore store = getStore(grpId, partId); long pageIdx = store.allocatePage(); @@ -468,13 +468,13 @@ private boolean checkAndInitCacheWorkDir(File cacheWorkDir) throws IgniteChecked } /** {@inheritDoc} */ - @Override public long metaPageId(final int cacheId) { + @Override public long metaPageId(final int grpId) { return metaPageId; } /** {@inheritDoc} */ - @Override public int pages(int cacheId, int partId) throws IgniteCheckedException { - PageStore store = getStore(cacheId, partId); + @Override public int pages(int grpId, int partId) throws IgniteCheckedException { + PageStore store = getStore(grpId, partId); return store.pages(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java index 844bc0227d332..e99a5eecc1b70 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java @@ -203,7 +203,7 @@ private int addRow( PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize); wal.log(new DataPageInsertRecord( - cacheId, + grpId, pageId, payload)); } @@ -246,7 +246,7 @@ private int addRowFragment( PageUtils.getBytes(pageAddr, data.offset(), payload, 0, payloadSize); - wal.log(new DataPageInsertFragmentRecord(cacheId, pageId, payload, lastLink)); + wal.log(new DataPageInsertFragmentRecord(grpId, pageId, payload, lastLink)); } return written + payloadSize; @@ -448,7 +448,7 @@ private long allocateDataPage(int part) throws IgniteCheckedException { assert part <= PageIdAllocator.MAX_PARTITION_ID; assert part != PageIdAllocator.INDEX_PARTITION; - return pageMem.allocatePage(cacheId, part, PageIdAllocator.FLAG_DATA); + return pageMem.allocatePage(grpId, part, PageIdAllocator.FLAG_DATA); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java index eb5df6bb0a34c..39a6865a89019 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java @@ -333,7 +333,7 @@ public void saveMetadata() throws IgniteCheckedException { io.resetCount(pageAddr); if (needWalDeltaRecord(pageId, page, null)) - wal.log(new PageListMetaResetCountRecord(cacheId, pageId)); + wal.log(new PageListMetaResetCountRecord(grpId, pageId)); nextPageId = io.getNextMetaPageId(pageAddr); } @@ -682,13 +682,13 @@ private boolean putDataPage( incrementBucketSize(bucket); if (needWalDeltaRecord(pageId, page, null)) - wal.log(new PagesListAddPageRecord(cacheId, pageId, dataId)); + wal.log(new PagesListAddPageRecord(grpId, pageId, dataId)); DataPageIO dataIO = DataPageIO.VERSIONS.forPage(dataAddr); dataIO.setFreeListPageId(dataAddr, pageId); if (needWalDeltaRecord(dataId, dataPage, null)) - wal.log(new DataPageSetFreeListPageRecord(cacheId, dataId, pageId)); + wal.log(new DataPageSetFreeListPageRecord(grpId, dataId, pageId)); } return true; @@ -728,11 +728,11 @@ private void handlePageFull( setupNextPage(io, pageId, pageAddr, newDataId, dataAddr); if (needWalDeltaRecord(pageId, page, null)) - wal.log(new PagesListSetNextRecord(cacheId, pageId, newDataId)); + wal.log(new PagesListSetNextRecord(grpId, pageId, newDataId)); if (needWalDeltaRecord(dataId, data, null)) wal.log(new PagesListInitNewPageRecord( - cacheId, + grpId, dataId, io.getType(), io.getVersion(), @@ -761,13 +761,13 @@ private void handlePageFull( setupNextPage(io, pageId, pageAddr, nextId, nextPageAddr); if (needWalDeltaRecord(pageId, page, null)) - wal.log(new PagesListSetNextRecord(cacheId, pageId, nextId)); + wal.log(new PagesListSetNextRecord(grpId, pageId, nextId)); int idx = io.addPage(nextPageAddr, dataId, pageSize()); if (needWalDeltaRecord(nextId, nextPage, nextWalPlc)) wal.log(new PagesListInitNewPageRecord( - cacheId, + grpId, nextId, io.getType(), io.getVersion(), @@ -781,7 +781,7 @@ private void handlePageFull( dataIO.setFreeListPageId(dataAddr, nextId); if (needWalDeltaRecord(dataId, data, null)) - wal.log(new DataPageSetFreeListPageRecord(cacheId, dataId, nextId)); + wal.log(new DataPageSetFreeListPageRecord(grpId, dataId, nextId)); incrementBucketSize(bucket); @@ -853,12 +853,12 @@ private boolean putReuseBag( setupNextPage(io, prevId, prevAddr, nextId, nextPageAddr); if (needWalDeltaRecord(prevId, prevPage, walPlc)) - wal.log(new PagesListSetNextRecord(cacheId, prevId, nextId)); + wal.log(new PagesListSetNextRecord(grpId, prevId, nextId)); // Here we should never write full page, because it is known to be new. if (needWalDeltaRecord(nextId, nextPage, FALSE)) wal.log(new PagesListInitNewPageRecord( - cacheId, + grpId, nextId, io.getType(), io.getVersion(), @@ -887,7 +887,7 @@ private boolean putReuseBag( else { // TODO: use single WAL record for bag? if (needWalDeltaRecord(prevId, prevPage, walPlc)) - wal.log(new PagesListAddPageRecord(cacheId, prevId, nextId)); + wal.log(new PagesListAddPageRecord(grpId, prevId, nextId)); incrementBucketSize(bucket); } @@ -1035,7 +1035,7 @@ protected final long takeEmptyPage(int bucket, @Nullable IOVersions initIoVers) decrementBucketSize(bucket); if (needWalDeltaRecord(tailId, tailPage, null)) - wal.log(new PagesListRemovePageRecord(cacheId, tailId, pageId)); + wal.log(new PagesListRemovePageRecord(grpId, tailId, pageId)); dirty = true; @@ -1085,7 +1085,7 @@ protected final long takeEmptyPage(int bucket, @Nullable IOVersions initIoVers) initIo.initNewPage(tailAddr, dataPageId, pageSize()); if (needWalDeltaRecord(tailId, tailPage, null)) { - wal.log(new InitNewPageRecord(cacheId, tailId, initIo.getType(), + wal.log(new InitNewPageRecord(grpId, tailId, initIo.getType(), initIo.getVersion(), dataPageId)); } } @@ -1163,13 +1163,13 @@ protected final boolean removeDataPage( decrementBucketSize(bucket); if (needWalDeltaRecord(pageId, page, null)) - wal.log(new PagesListRemovePageRecord(cacheId, pageId, dataId)); + wal.log(new PagesListRemovePageRecord(grpId, pageId, dataId)); // Reset free list page ID. dataIO.setFreeListPageId(dataAddr, 0L); if (needWalDeltaRecord(dataId, dataPage, null)) - wal.log(new DataPageSetFreeListPageRecord(cacheId, dataId, 0L)); + wal.log(new DataPageSetFreeListPageRecord(grpId, dataId, 0L)); if (!io.isEmpty(pageAddr)) return true; // In optimistic case we still have something in the page and can leave it as is. @@ -1343,7 +1343,7 @@ private long doMerge( nextIO.setPreviousId(nextAddr, 0); if (needWalDeltaRecord(nextId, nextPage, null)) - wal.log(new PagesListSetPreviousRecord(cacheId, nextId, 0L)); + wal.log(new PagesListSetPreviousRecord(grpId, nextId, 0L)); } else // Do a fair merge: link previous and next to each other. fairMerge(prevId, pageId, nextId, nextPage, nextAddr); @@ -1384,12 +1384,12 @@ private void fairMerge( prevIO.setNextId(prevAddr, nextId); if (needWalDeltaRecord(prevId, prevPage, null)) - wal.log(new PagesListSetNextRecord(cacheId, prevId, nextId)); + wal.log(new PagesListSetNextRecord(grpId, prevId, nextId)); nextIO.setPreviousId(nextAddr, prevId); if (needWalDeltaRecord(nextId, nextPage, null)) - wal.log(new PagesListSetPreviousRecord(cacheId, nextId, prevId)); + wal.log(new PagesListSetPreviousRecord(grpId, nextId, prevId)); } finally { writeUnlock(prevId, prevPage, prevAddr, true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java index 78d83b3faac36..19d26ffdd14db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java @@ -45,13 +45,13 @@ public class FullPageIdTable { private static final long EMPTY_PAGE_ID = EMPTY_FULL_PAGE_ID.pageId(); /** */ - private static final int EMPTY_CACHE_ID = EMPTY_FULL_PAGE_ID.cacheId(); + private static final int EMPTY_CACHE_GRP_ID = EMPTY_FULL_PAGE_ID.groupId(); /** */ private static final long REMOVED_PAGE_ID = 0x8000000000000000L; /** */ - private static final int REMOVED_CACHE_ID = 0; + private static final int REMOVED_CACHE_GRP_ID = 0; /** */ private static final int EQUAL = 0; @@ -178,13 +178,13 @@ public void put(int cacheId, long pageId, long value, int tag) { /** * Removes key-value association for the given key. * - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. */ - public void remove(int cacheId, long pageId, int tag) { - assert assertKey(cacheId, pageId); + public void remove(int grpId, long pageId, int tag) { + assert assertKey(grpId, pageId); - int index = removeKey(cacheId, pageId, tag); + int index = removeKey(grpId, pageId, tag); if (index >= 0) setValueAt(index, 0); @@ -225,18 +225,18 @@ public EvictCandidate getNearestAt(final int idx, final long absent) { public long clearAt(int idx, GridPredicate3 pred, long absent) { long base = entryBase(idx); - int cacheId = GridUnsafe.getInt(base); + int grpId = GridUnsafe.getInt(base); int tag = GridUnsafe.getInt(base + 4); long pageId = GridUnsafe.getLong(base + 8); - if ((pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID) - || (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID)) + if ((pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID) + || (pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID)) return absent; - if (pred.apply(cacheId, pageId, tag)) { + if (pred.apply(grpId, pageId, tag)) { long res = valueAt(idx); - setKeyAt(idx, REMOVED_CACHE_ID, REMOVED_PAGE_ID); + setKeyAt(idx, REMOVED_CACHE_GRP_ID, REMOVED_PAGE_ID); setValueAt(idx, 0); return res; @@ -367,7 +367,7 @@ else if (res == EMPTY) while (++step <= maxSteps); if (foundIndex != -1) { - setKeyAt(foundIndex, REMOVED_CACHE_ID, REMOVED_PAGE_ID); + setKeyAt(foundIndex, REMOVED_CACHE_GRP_ID, REMOVED_PAGE_ID); decrementSize(); } @@ -382,17 +382,17 @@ else if (res == EMPTY) private int testKeyAt(int index, int testCacheId, long testPageId, int testTag) { long base = entryBase(index); - int cacheId = GridUnsafe.getInt(base); + int grpId = GridUnsafe.getInt(base); int tag = GridUnsafe.getInt(base + 4); long pageId = GridUnsafe.getLong(base + 8); - if (pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID) + if (pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID) return REMOVED; - else if (pageId == testPageId && cacheId == testCacheId && tag >= testTag) + else if (pageId == testPageId && grpId == testCacheId && tag >= testTag) return EQUAL; - else if (pageId == testPageId && cacheId == testCacheId && tag < testTag) + else if (pageId == testPageId && grpId == testCacheId && tag < testTag) return OUTDATED; - else if (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID) + else if (pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID) return EMPTY; else return NOT_EQUAL; @@ -405,34 +405,34 @@ else if (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID) private boolean isValuePresentAt(final int idx) { long base = entryBase(idx); - int cacheId = GridUnsafe.getInt(base); + int grpId = GridUnsafe.getInt(base); long pageId = GridUnsafe.getLong(base + 8); - return !((pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID) - || (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID)); + return !((pageId == REMOVED_PAGE_ID && grpId == REMOVED_CACHE_GRP_ID) + || (pageId == EMPTY_PAGE_ID && grpId == EMPTY_CACHE_GRP_ID)); } /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @return {@code True} if checks succeeded. */ - private boolean assertKey(int cacheId, long pageId) { - assert cacheId != EMPTY_CACHE_ID && PageIdUtils.isEffectivePageId(pageId): - "cacheId=" + cacheId + ", pageId=" + U.hexLong(pageId); + private boolean assertKey(int grpId, long pageId) { + assert grpId != EMPTY_CACHE_GRP_ID && PageIdUtils.isEffectivePageId(pageId): + "grpId=" + grpId + ", pageId=" + U.hexLong(pageId); return true; } /** * @param index Entry index. - * @param cacheId Cache ID to write. + * @param grpId Cache group ID to write. * @param pageId Page ID to write. */ - private void setKeyAt(int index, int cacheId, long pageId) { + private void setKeyAt(int index, int grpId, long pageId) { long base = entryBase(index); - GridUnsafe.putInt(base, cacheId); + GridUnsafe.putInt(base, grpId); GridUnsafe.putLong(base + 8, pageId); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java index 3246f218ee3a0..7c63d419b2c4a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java @@ -77,13 +77,13 @@ void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc, * @see #acquirePage(int, long) * Will not read page from file if it is not present in memory * - * @param cacheId Cache id. + * @param grpId Cache group ID. * @param pageId Page id. * @param restore Get page for restore * @throws IgniteCheckedException If failed. * @return Page. */ - public long acquirePage(int cacheId, long pageId, boolean restore) throws IgniteCheckedException; + public long acquirePage(int grpId, long pageId, boolean restore) throws IgniteCheckedException; /** * Heuristic method which allows a thread to check if it safe to start memory struture modifications diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index 6bb5c337db0f9..ab6619dd064bd 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -714,7 +714,7 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck if (curPage != null && deltaRecord.pageId() == fullId.pageId() - && deltaRecord.cacheId() == fullId.cacheId()) { + && deltaRecord.groupId() == fullId.groupId()) { assert tmpAddr != null; deltaRecord.applyDelta(this, tmpAddr); @@ -727,8 +727,8 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck if (restored == null) throw new AssertionError(String.format( - "Page is broken. Can't restore it from WAL. (cacheId = %d, pageId = %X).", - fullId.cacheId(), fullId.pageId() + "Page is broken. Can't restore it from WAL. (grpId = %d, pageId = %X).", + fullId.groupId(), fullId.pageId() )); long pageAddr = writeLockPage(absPtr, fullId, false); @@ -796,7 +796,7 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck @Override public Integer getForCheckpoint(FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) { assert tmpBuf.remaining() == pageSize(); - Segment seg = segment(fullId.cacheId(), fullId.pageId()); + Segment seg = segment(fullId.groupId(), fullId.pageId()); long absPtr = 0; @@ -807,10 +807,10 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck seg.readLock().lock(); try { - tag = seg.partTag(fullId.cacheId(), PageIdUtils.partId(fullId.pageId())); + tag = seg.partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId())); relPtr = seg.loadedPages.get( - fullId.cacheId(), + fullId.groupId(), PageIdUtils.effectivePageId(fullId.pageId()), tag, INVALID_REL_PTR, @@ -839,10 +839,10 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck try { // Double-check. relPtr = seg.loadedPages.get( - fullId.cacheId(), + fullId.groupId(), PageIdUtils.effectivePageId(fullId.pageId()), seg.partTag( - fullId.cacheId(), + fullId.groupId(), PageIdUtils.partId(fullId.pageId()) ), INVALID_REL_PTR, @@ -855,7 +855,7 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck if (relPtr == OUTDATED_REL_PTR) { relPtr = refreshOutdatedPage( seg, - fullId.cacheId(), + fullId.groupId(), PageIdUtils.effectivePageId(fullId.pageId()), true ); @@ -1245,7 +1245,7 @@ ByteBuffer wrapPointer(long ptr, int len) { * @return {@code True} if it was added to the checkpoint list. */ boolean isInCheckpoint(FullPageId pageId) { - Segment seg = segment(pageId.cacheId(), pageId.pageId()); + Segment seg = segment(pageId.groupId(), pageId.pageId()); Collection pages0 = seg.segCheckpointPages; @@ -1256,7 +1256,7 @@ boolean isInCheckpoint(FullPageId pageId) { * @param fullPageId Page ID to clear. */ void clearCheckpoint(FullPageId fullPageId) { - Segment seg = segment(fullPageId.cacheId(), fullPageId.pageId()); + Segment seg = segment(fullPageId.groupId(), fullPageId.pageId()); Collection pages0 = seg.segCheckpointPages; @@ -1300,14 +1300,14 @@ void setDirty(FullPageId pageId, long absPtr, boolean dirty, boolean forceAdd) { if (dirty) { if (!wasDirty || forceAdd) { - boolean added = segment(pageId.cacheId(), pageId.pageId()).dirtyPages.add(pageId); + boolean added = segment(pageId.groupId(), pageId.pageId()).dirtyPages.add(pageId); if (added) memMetrics.incrementDirtyPages(); } } else { - boolean rmv = segment(pageId.cacheId(), pageId.pageId()).dirtyPages.remove(pageId); + boolean rmv = segment(pageId.groupId(), pageId.pageId()).dirtyPages.remove(pageId); if (rmv) memMetrics.decrementDirtyPages(); @@ -1330,12 +1330,12 @@ void beforeReleaseWrite(FullPageId pageId, long ptr, boolean pageWalRec) { } /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param pageId Page ID. * @return Segment. */ - private Segment segment(int cacheId, long pageId) { - int idx = segmentIndex(cacheId, pageId, segments.length); + private Segment segment(int grpId, long pageId) { + int idx = segmentIndex(grpId, pageId, segments.length); return segments[idx]; } @@ -1344,11 +1344,11 @@ private Segment segment(int cacheId, long pageId) { * @param pageId Page ID. * @return Segment index. */ - public static int segmentIndex(int cacheId, long pageId, int segments) { + public static int segmentIndex(int grpId, long pageId, int segments) { pageId = PageIdUtils.effectivePageId(pageId); // Take a prime number larger than total number of partitions. - int hash = U.hash(pageId * 65537 + cacheId); + int hash = U.hash(pageId * 65537 + grpId); return U.safeAbs(hash) % segments; } @@ -1645,7 +1645,7 @@ private boolean prepareEvict(FullPageId fullPageId, long absPtr) throws IgniteCh assert writeLock().isHeldByCurrentThread(); // Do not evict cache meta pages. - if (fullPageId.pageId() == storeMgr.metaPageId(fullPageId.cacheId())) + if (fullPageId.pageId() == storeMgr.metaPageId(fullPageId.groupId())) return false; if (PageHeader.isAcquired(absPtr)) @@ -1662,7 +1662,7 @@ private boolean prepareEvict(FullPageId fullPageId, long absPtr) throws IgniteCh fullPageId, wrapPointer(absPtr + PAGE_OVERHEAD, pageSize()), partTag( - fullPageId.cacheId(), + fullPageId.groupId(), PageIdUtils.partId(fullPageId.pageId()) ) ); @@ -1743,10 +1743,10 @@ private long evictPage() throws IgniteCheckedException { assert fullId.equals(nearest.fullId()) : "Invalid page mapping [tableId=" + nearest.fullId() + ", actual=" + fullId + ", nearest=" + nearest; - boolean outdated = partTag < partTag(fullId.cacheId(), PageIdUtils.partId(fullId.pageId())); + boolean outdated = partTag < partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId())); if (outdated) - return refreshOutdatedPage(this, fullId.cacheId(), fullId.pageId(), true); + return refreshOutdatedPage(this, fullId.groupId(), fullId.pageId(), true); boolean pinned = PageHeader.isAcquired(absPageAddr); @@ -1797,10 +1797,10 @@ else if (pageTs < dirtyTs && dirty) { } loadedPages.remove( - fullPageId.cacheId(), + fullPageId.groupId(), PageIdUtils.effectivePageId(fullPageId.pageId()), partTag( - fullPageId.cacheId(), + fullPageId.groupId(), PageIdUtils.partId(fullPageId.pageId()) ) ); @@ -1834,8 +1834,8 @@ private long tryToFindSequentially(int cap) throws IgniteCheckedException { FullPageId fullId = PageHeader.fullPageId(absPageAddr); - if (partTag < partTag(fullId.cacheId(), PageIdUtils.partId(fullId.pageId()))) - return refreshOutdatedPage(this, fullId.cacheId(), fullId.pageId(), true); + if (partTag < partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId()))) + return refreshOutdatedPage(this, fullId.groupId(), fullId.pageId(), true); boolean pinned = PageHeader.isAcquired(absPageAddr); @@ -1851,10 +1851,10 @@ private long tryToFindSequentially(int cap) throws IgniteCheckedException { if (prepareEvict(fullPageId, absEvictAddr)) { loadedPages.remove( - fullPageId.cacheId(), + fullPageId.groupId(), PageIdUtils.effectivePageId(fullPageId.pageId()), partTag( - fullPageId.cacheId(), + fullPageId.groupId(), PageIdUtils.partId(fullPageId.pageId()) ) ); @@ -1888,14 +1888,14 @@ private long absolute(long relPtr) { } /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param partId Partition ID. * @return Partition tag. */ - private int partTag(int cacheId, int partId) { + private int partTag(int grpId, int partId) { assert getReadHoldCount() > 0 || getWriteHoldCount() > 0; - Integer tag = partTagMap.get(new T2<>(cacheId, partId)); + Integer tag = partTagMap.get(new T2<>(grpId, partId)); if (tag == null) return 1; @@ -1904,14 +1904,14 @@ private int partTag(int cacheId, int partId) { } /** - * @param cacheId Cache ID. + * @param grpId Cache group ID. * @param partId Partition ID. * @return New partition tag. */ - private int incrementPartTag(int cacheId, int partId) { + private int incrementPartTag(int grpId, int partId) { assert getWriteHoldCount() > 0; - T2 t = new T2<>(cacheId, partId); + T2 t = new T2<>(grpId, partId); Integer tag = partTagMap.get(t); @@ -1921,7 +1921,7 @@ private int incrementPartTag(int cacheId, int partId) { return 2; } else if (tag == Integer.MAX_VALUE) { - U.warn(log, "Partition tag overflow [cacheId=" + cacheId + ", partId=" + partId + "]"); + U.warn(log, "Partition tag overflow [grpId=" + grpId + ", partId=" + partId + "]"); partTagMap.put(t, 0); @@ -2224,7 +2224,7 @@ private static FullPageId fullPageId(final long absPtr) { private static void fullPageId(final long absPtr, final FullPageId fullPageId) { pageId(absPtr, fullPageId.pageId()); - pageCacheId(absPtr, fullPageId.cacheId()); + pageCacheId(absPtr, fullPageId.groupId()); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java index 74d251a4c4761..c73b4c75cc51b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java @@ -377,7 +377,7 @@ private class Replace extends GetPageHandler { byte[] newRowBytes = io.store(pageAddr, idx, newRow, null, needWal); if (needWal) - wal.log(new ReplaceRecord<>(cacheId, pageId, io, newRowBytes, idx)); + wal.log(new ReplaceRecord<>(grpId, pageId, io, newRowBytes, idx)); return FOUND; } @@ -2200,7 +2200,7 @@ private boolean splitPage( io.splitExistingPage(pageAddr, mid, fwdId); if (needWalDeltaRecord(pageId, page, null)) - wal.log(new SplitExistingPageRecord(cacheId, pageId, mid, fwdId)); + wal.log(new SplitExistingPageRecord(grpId, pageId, mid, fwdId)); return res; } @@ -2685,7 +2685,7 @@ private void insertSimple(long pageId, long page, long pageAddr, BPlusIO io, byte[] rowBytes = io.insert(pageAddr, idx, row, null, rightId, needWal); if (needWal) - wal.log(new InsertRecord<>(cacheId, pageId, io, idx, rowBytes, rightId)); + wal.log(new InsertRecord<>(grpId, pageId, io, idx, rowBytes, rightId)); } /** @@ -2728,7 +2728,7 @@ private L insertWithSplit(long pageId, long page, long pageAddr, BPlusIO io, inner(io).setLeft(fwdPageAddr, 0, rightId); if (needWalDeltaRecord(fwdId, fwdPage, fwdPageWalPlc)) // Rare case, we can afford separate WAL record to avoid complexity. - wal.log(new FixLeftmostChildRecord(cacheId, fwdId, rightId)); + wal.log(new FixLeftmostChildRecord(grpId, fwdId, rightId)); } } else // Insert into newly allocated forward page. @@ -2744,7 +2744,7 @@ private L insertWithSplit(long pageId, long page, long pageAddr, BPlusIO io, io.setCount(pageAddr, cnt - 1); if (needWalDeltaRecord(pageId, page, null)) // Rare case, we can afford separate WAL record to avoid complexity. - wal.log(new FixCountRecord(cacheId, pageId, cnt - 1)); + wal.log(new FixCountRecord(grpId, pageId, cnt - 1)); } if (!hadFwd && lvl == getRootLevel()) { // We are splitting root. @@ -2775,7 +2775,7 @@ private L insertWithSplit(long pageId, long page, long pageAddr, BPlusIO io, needWal); if (needWal) - wal.log(new NewRootInitRecord<>(cacheId, newRootId, newRootId, + wal.log(new NewRootInitRecord<>(grpId, newRootId, newRootId, inner(io), pageId, moveUpRowBytes, fwdId)); } finally { @@ -3656,7 +3656,7 @@ private void doRemove(long pageId, long page, long pageAddr, Boolean walPlc, BPl io.remove(pageAddr, idx, cnt); if (needWalDeltaRecord(pageId, page, walPlc)) - wal.log(new RemoveRecord(cacheId, pageId, idx, cnt)); + wal.log(new RemoveRecord(grpId, pageId, idx, cnt)); } /** @@ -3909,7 +3909,7 @@ private void replaceInner() throws IgniteCheckedException { leaf.io.setRemoveId(leaf.buf, rmvId); if (needWalDeltaRecord(leaf.pageId, leaf.page, leaf.walPlc)) - wal.log(new FixRemoveId(cacheId, leaf.pageId, rmvId)); + wal.log(new FixRemoveId(grpId, leaf.pageId, rmvId)); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index 0a7b3dd57fd28..663fe0c0bb560 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -166,7 +166,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PAGE_RECORD: PageSnapshot snap = (PageSnapshot)record; - buf.putInt(snap.fullPageId().cacheId()); + buf.putInt(snap.fullPageId().groupId()); buf.putLong(snap.fullPageId().pageId()); buf.put(snap.pageData()); @@ -182,7 +182,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PARTITION_DESTROY: PartitionDestroyRecord partDestroy = (PartitionDestroyRecord)record; - buf.putInt(partDestroy.cacheId()); + buf.putInt(partDestroy.groupId()); buf.putInt(partDestroy.partitionId()); break; @@ -190,7 +190,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case META_PAGE_INIT: MetaPageInitRecord updRootsRec = (MetaPageInitRecord)record; - buf.putInt(updRootsRec.cacheId()); + buf.putInt(updRootsRec.groupId()); buf.putLong(updRootsRec.pageId()); buf.putShort((short)updRootsRec.ioType()); @@ -203,7 +203,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PARTITION_META_PAGE_UPDATE_COUNTERS: MetaPageUpdatePartitionDataRecord partDataRec = (MetaPageUpdatePartitionDataRecord)record; - buf.putInt(partDataRec.cacheId()); + buf.putInt(partDataRec.groupId()); buf.putLong(partDataRec.pageId()); buf.putLong(partDataRec.updateCounter()); @@ -261,7 +261,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case DATA_PAGE_INSERT_RECORD: DataPageInsertRecord diRec = (DataPageInsertRecord)record; - buf.putInt(diRec.cacheId()); + buf.putInt(diRec.groupId()); buf.putLong(diRec.pageId()); buf.putShort((short)diRec.payload().length); @@ -273,7 +273,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case DATA_PAGE_UPDATE_RECORD: DataPageUpdateRecord uRec = (DataPageUpdateRecord)record; - buf.putInt(uRec.cacheId()); + buf.putInt(uRec.groupId()); buf.putLong(uRec.pageId()); buf.putInt(uRec.itemId()); @@ -286,7 +286,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case DATA_PAGE_INSERT_FRAGMENT_RECORD: final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord)record; - buf.putInt(difRec.cacheId()); + buf.putInt(difRec.groupId()); buf.putLong(difRec.pageId()); buf.putLong(difRec.lastLink()); @@ -298,7 +298,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case DATA_PAGE_REMOVE_RECORD: DataPageRemoveRecord drRec = (DataPageRemoveRecord)record; - buf.putInt(drRec.cacheId()); + buf.putInt(drRec.groupId()); buf.putLong(drRec.pageId()); buf.put((byte)drRec.itemId()); @@ -308,7 +308,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case DATA_PAGE_SET_FREE_LIST_PAGE: DataPageSetFreeListPageRecord freeListRec = (DataPageSetFreeListPageRecord)record; - buf.putInt(freeListRec.cacheId()); + buf.putInt(freeListRec.groupId()); buf.putLong(freeListRec.pageId()); buf.putLong(freeListRec.freeListPage()); @@ -318,7 +318,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case INIT_NEW_PAGE_RECORD: InitNewPageRecord inpRec = (InitNewPageRecord)record; - buf.putInt(inpRec.cacheId()); + buf.putInt(inpRec.groupId()); buf.putLong(inpRec.pageId()); buf.putShort((short)inpRec.ioType()); @@ -330,7 +330,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_META_PAGE_INIT_ROOT: MetaPageInitRootRecord imRec = (MetaPageInitRootRecord)record; - buf.putInt(imRec.cacheId()); + buf.putInt(imRec.groupId()); buf.putLong(imRec.pageId()); buf.putLong(imRec.rootId()); @@ -340,7 +340,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_META_PAGE_INIT_ROOT2: MetaPageInitRootInlineRecord imRec2 = (MetaPageInitRootInlineRecord)record; - buf.putInt(imRec2.cacheId()); + buf.putInt(imRec2.groupId()); buf.putLong(imRec2.pageId()); buf.putLong(imRec2.rootId()); @@ -351,7 +351,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_META_PAGE_ADD_ROOT: MetaPageAddRootRecord arRec = (MetaPageAddRootRecord)record; - buf.putInt(arRec.cacheId()); + buf.putInt(arRec.groupId()); buf.putLong(arRec.pageId()); buf.putLong(arRec.rootId()); @@ -361,7 +361,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_META_PAGE_CUT_ROOT: MetaPageCutRootRecord crRec = (MetaPageCutRootRecord)record; - buf.putInt(crRec.cacheId()); + buf.putInt(crRec.groupId()); buf.putLong(crRec.pageId()); break; @@ -369,7 +369,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_INIT_NEW_ROOT: NewRootInitRecord riRec = (NewRootInitRecord)record; - buf.putInt(riRec.cacheId()); + buf.putInt(riRec.groupId()); buf.putLong(riRec.pageId()); buf.putLong(riRec.rootId()); @@ -385,7 +385,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_PAGE_RECYCLE: RecycleRecord recRec = (RecycleRecord)record; - buf.putInt(recRec.cacheId()); + buf.putInt(recRec.groupId()); buf.putLong(recRec.pageId()); buf.putLong(recRec.newPageId()); @@ -395,7 +395,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_PAGE_INSERT: InsertRecord inRec = (InsertRecord)record; - buf.putInt(inRec.cacheId()); + buf.putInt(inRec.groupId()); buf.putLong(inRec.pageId()); buf.putShort((short)inRec.io().getType()); @@ -410,7 +410,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_FIX_LEFTMOST_CHILD: FixLeftmostChildRecord flRec = (FixLeftmostChildRecord)record; - buf.putInt(flRec.cacheId()); + buf.putInt(flRec.groupId()); buf.putLong(flRec.pageId()); buf.putLong(flRec.rightId()); @@ -420,7 +420,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_FIX_COUNT: FixCountRecord fcRec = (FixCountRecord)record; - buf.putInt(fcRec.cacheId()); + buf.putInt(fcRec.groupId()); buf.putLong(fcRec.pageId()); buf.putShort((short)fcRec.count()); @@ -430,7 +430,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_PAGE_REPLACE: ReplaceRecord rRec = (ReplaceRecord)record; - buf.putInt(rRec.cacheId()); + buf.putInt(rRec.groupId()); buf.putLong(rRec.pageId()); buf.putShort((short)rRec.io().getType()); @@ -444,7 +444,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_PAGE_REMOVE: RemoveRecord rmRec = (RemoveRecord)record; - buf.putInt(rmRec.cacheId()); + buf.putInt(rmRec.groupId()); buf.putLong(rmRec.pageId()); buf.putShort((short)rmRec.index()); @@ -455,7 +455,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_PAGE_INNER_REPLACE: InnerReplaceRecord irRec = (InnerReplaceRecord)record; - buf.putInt(irRec.cacheId()); + buf.putInt(irRec.groupId()); buf.putLong(irRec.pageId()); buf.putShort((short)irRec.destinationIndex()); @@ -468,7 +468,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_FORWARD_PAGE_SPLIT: SplitForwardPageRecord sfRec = (SplitForwardPageRecord)record; - buf.putInt(sfRec.cacheId()); + buf.putInt(sfRec.groupId()); buf.putLong(sfRec.pageId()); buf.putLong(sfRec.forwardId()); @@ -483,7 +483,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_EXISTING_PAGE_SPLIT: SplitExistingPageRecord seRec = (SplitExistingPageRecord)record; - buf.putInt(seRec.cacheId()); + buf.putInt(seRec.groupId()); buf.putLong(seRec.pageId()); buf.putShort((short)seRec.middleIndex()); @@ -494,7 +494,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_PAGE_MERGE: MergeRecord mRec = (MergeRecord)record; - buf.putInt(mRec.cacheId()); + buf.putInt(mRec.groupId()); buf.putLong(mRec.pageId()); buf.putLong(mRec.parentId()); @@ -507,7 +507,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PAGES_LIST_SET_NEXT: PagesListSetNextRecord plNextRec = (PagesListSetNextRecord)record; - buf.putInt(plNextRec.cacheId()); + buf.putInt(plNextRec.groupId()); buf.putLong(plNextRec.pageId()); buf.putLong(plNextRec.nextPageId()); @@ -517,7 +517,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PAGES_LIST_SET_PREVIOUS: PagesListSetPreviousRecord plPrevRec = (PagesListSetPreviousRecord)record; - buf.putInt(plPrevRec.cacheId()); + buf.putInt(plPrevRec.groupId()); buf.putLong(plPrevRec.pageId()); buf.putLong(plPrevRec.previousPageId()); @@ -527,7 +527,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PAGES_LIST_INIT_NEW_PAGE: PagesListInitNewPageRecord plNewRec = (PagesListInitNewPageRecord)record; - buf.putInt(plNewRec.cacheId()); + buf.putInt(plNewRec.groupId()); buf.putLong(plNewRec.pageId()); buf.putInt(plNewRec.ioType()); buf.putInt(plNewRec.ioVersion()); @@ -541,7 +541,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PAGES_LIST_ADD_PAGE: PagesListAddPageRecord plAddRec = (PagesListAddPageRecord)record; - buf.putInt(plAddRec.cacheId()); + buf.putInt(plAddRec.groupId()); buf.putLong(plAddRec.pageId()); buf.putLong(plAddRec.dataPageId()); @@ -551,7 +551,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PAGES_LIST_REMOVE_PAGE: PagesListRemovePageRecord plRmvRec = (PagesListRemovePageRecord)record; - buf.putInt(plRmvRec.cacheId()); + buf.putInt(plRmvRec.groupId()); buf.putLong(plRmvRec.pageId()); buf.putLong(plRmvRec.removedPageId()); @@ -561,7 +561,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case BTREE_FIX_REMOVE_ID: FixRemoveId frRec = (FixRemoveId)record; - buf.putInt(frRec.cacheId()); + buf.putInt(frRec.groupId()); buf.putLong(frRec.pageId()); buf.putLong(frRec.removeId()); @@ -571,7 +571,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case TRACKING_PAGE_DELTA: TrackingPageDeltaRecord tpDelta = (TrackingPageDeltaRecord)record; - buf.putInt(tpDelta.cacheId()); + buf.putInt(tpDelta.groupId()); buf.putLong(tpDelta.pageId()); buf.putLong(tpDelta.pageIdToMark()); @@ -583,7 +583,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID: MetaPageUpdateNextSnapshotId mpUpdateNextSnapshotId = (MetaPageUpdateNextSnapshotId)record; - buf.putInt(mpUpdateNextSnapshotId.cacheId()); + buf.putInt(mpUpdateNextSnapshotId.groupId()); buf.putLong(mpUpdateNextSnapshotId.pageId()); buf.putLong(mpUpdateNextSnapshotId.nextSnapshotId()); @@ -594,7 +594,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { MetaPageUpdateLastSuccessfulFullSnapshotId mpUpdateLastSuccFullSnapshotId = (MetaPageUpdateLastSuccessfulFullSnapshotId)record; - buf.putInt(mpUpdateLastSuccFullSnapshotId.cacheId()); + buf.putInt(mpUpdateLastSuccFullSnapshotId.groupId()); buf.putLong(mpUpdateLastSuccFullSnapshotId.pageId()); buf.putLong(mpUpdateLastSuccFullSnapshotId.lastSuccessfulFullSnapshotId()); @@ -605,7 +605,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { MetaPageUpdateLastSuccessfulSnapshotId mpUpdateLastSuccSnapshotId = (MetaPageUpdateLastSuccessfulSnapshotId)record; - buf.putInt(mpUpdateLastSuccSnapshotId.cacheId()); + buf.putInt(mpUpdateLastSuccSnapshotId.groupId()); buf.putLong(mpUpdateLastSuccSnapshotId.pageId()); buf.putLong(mpUpdateLastSuccSnapshotId.lastSuccessfulSnapshotId()); @@ -617,7 +617,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { MetaPageUpdateLastAllocatedIndex mpUpdateLastAllocatedIdx = (MetaPageUpdateLastAllocatedIndex) record; - buf.putInt(mpUpdateLastAllocatedIdx.cacheId()); + buf.putInt(mpUpdateLastAllocatedIdx.groupId()); buf.putLong(mpUpdateLastAllocatedIdx.pageId()); buf.putInt(mpUpdateLastAllocatedIdx.lastAllocatedIndex()); @@ -627,7 +627,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PART_META_UPDATE_STATE: PartitionMetaStateRecord partMetaStateRecord = (PartitionMetaStateRecord) record; - buf.putInt(partMetaStateRecord.cacheId()); + buf.putInt(partMetaStateRecord.groupId()); buf.putInt(partMetaStateRecord.partitionId()); @@ -640,7 +640,7 @@ public RecordV1Serializer(GridCacheSharedContext cctx) { case PAGE_LIST_META_RESET_COUNT_RECORD: PageListMetaResetCountRecord pageListMetaResetCntRecord = (PageListMetaResetCountRecord) record; - buf.putInt(pageListMetaResetCntRecord.cacheId()); + buf.putInt(pageListMetaResetCntRecord.groupId()); buf.putLong(pageListMetaResetCntRecord.pageId()); break; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java index 02c521ee2606f..5bef37207beb0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java @@ -64,10 +64,10 @@ public void testPageTearingInner() throws Exception { FullPageId fullId1 = allocatePage(mem); FullPageId fullId2 = allocatePage(mem); - long page1 = mem.acquirePage(fullId1.cacheId(), fullId1.pageId()); + long page1 = mem.acquirePage(fullId1.groupId(), fullId1.pageId()); try { - long page2 = mem.acquirePage(fullId2.cacheId(), fullId2.pageId()); + long page2 = mem.acquirePage(fullId2.groupId(), fullId2.pageId()); info("Allocated pages [page1Id=" + fullId1.pageId() + ", page1=" + page1 + ", page2Id=" + fullId2.pageId() + ", page2=" + page2 + ']'); @@ -84,11 +84,11 @@ public void testPageTearingInner() throws Exception { readPage(mem, fullId2.pageId(), page2, 2); } finally { - mem.releasePage(fullId2.cacheId(), fullId2.pageId(), page2); + mem.releasePage(fullId2.groupId(), fullId2.pageId(), page2); } } finally { - mem.releasePage(fullId1.cacheId(), fullId1.pageId(), page1); + mem.releasePage(fullId1.groupId(), fullId1.pageId(), page1); } } finally { @@ -139,7 +139,7 @@ public void testPageTearingSequential() throws Exception { pages.add(fullId); - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { if (i % 64 == 0) @@ -148,14 +148,14 @@ public void testPageTearingSequential() throws Exception { writePage(mem, fullId.pageId(), page, i + 1); } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } for (int i = 0; i < pagesCnt; i++) { FullPageId fullId = pages.get(i); - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { if (i % 64 == 0) @@ -164,7 +164,7 @@ public void testPageTearingSequential() throws Exception { readPage(mem, fullId.pageId(), page, i + 1); } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } } @@ -190,7 +190,7 @@ public void testPageHandleDeallocation() throws Exception { handles.add(allocatePage(mem)); for (FullPageId fullId : handles) - mem.freePage(fullId.cacheId(), fullId.pageId()); + mem.freePage(fullId.groupId(), fullId.pageId()); for (int i = 0; i < pages; i++) assertFalse(handles.add(allocatePage(mem))); @@ -219,9 +219,9 @@ public void testPageIdRotation() throws Exception { // Check that initial pages are accessible. for (FullPageId id : old) { - long pageApsPtr = mem.acquirePage(id.cacheId(), id.pageId()); + long pageApsPtr = mem.acquirePage(id.groupId(), id.pageId()); try { - long pageAddr = mem.writeLock(id.cacheId(), id.pageId(), pageApsPtr); + long pageAddr = mem.writeLock(id.groupId(), id.pageId(), pageApsPtr); assertNotNull(pageAddr); @@ -230,47 +230,47 @@ public void testPageIdRotation() throws Exception { PageIO.setPageId(pageAddr, updId); - updated.add(new FullPageId(updId, id.cacheId())); + updated.add(new FullPageId(updId, id.groupId())); } finally { - mem.writeUnlock(id.cacheId(), id.pageId(), pageApsPtr, null, true); + mem.writeUnlock(id.groupId(), id.pageId(), pageApsPtr, null, true); } } finally { - mem.releasePage(id.cacheId(), id.pageId(), pageApsPtr); + mem.releasePage(id.groupId(), id.pageId(), pageApsPtr); } } // Check that updated pages are inaccessible using old IDs. for (FullPageId id : old) { - long pageApsPtr = mem.acquirePage(id.cacheId(), id.pageId()); + long pageApsPtr = mem.acquirePage(id.groupId(), id.pageId()); try { - long pageAddr = mem.writeLock(id.cacheId(), id.pageId(), pageApsPtr); + long pageAddr = mem.writeLock(id.groupId(), id.pageId(), pageApsPtr); if (pageAddr != 0L) { - mem.writeUnlock(id.cacheId(), id.pageId(), pageApsPtr, null, false); + mem.writeUnlock(id.groupId(), id.pageId(), pageApsPtr, null, false); fail("Was able to acquire page write lock."); } - mem.readLock(id.cacheId(), id.pageId(), pageApsPtr); + mem.readLock(id.groupId(), id.pageId(), pageApsPtr); if (pageAddr != 0) { - mem.readUnlock(id.cacheId(), id.pageId(), pageApsPtr); + mem.readUnlock(id.groupId(), id.pageId(), pageApsPtr); fail("Was able to acquire page read lock."); } } finally { - mem.releasePage(id.cacheId(), id.pageId(), pageApsPtr); + mem.releasePage(id.groupId(), id.pageId(), pageApsPtr); } } // Check that updated pages are accessible using new IDs. for (FullPageId id : updated) { - long pageApsPtr = mem.acquirePage(id.cacheId(), id.pageId()); + long pageApsPtr = mem.acquirePage(id.groupId(), id.pageId()); try { - long pageAddr = mem.writeLock(id.cacheId(), id.pageId(), pageApsPtr); + long pageAddr = mem.writeLock(id.groupId(), id.pageId(), pageApsPtr); assertNotSame(0L, pageAddr); @@ -278,10 +278,10 @@ public void testPageIdRotation() throws Exception { assertEquals(id.pageId(), PageIO.getPageId(pageAddr)); } finally { - mem.writeUnlock(id.cacheId(), id.pageId(), pageApsPtr, null, false); + mem.writeUnlock(id.groupId(), id.pageId(), pageApsPtr, null, false); } - pageAddr = mem.readLock(id.cacheId(), id.pageId(), pageApsPtr); + pageAddr = mem.readLock(id.groupId(), id.pageId(), pageApsPtr); assertNotSame(0L, pageAddr); @@ -289,11 +289,11 @@ public void testPageIdRotation() throws Exception { assertEquals(id.pageId(), PageIO.getPageId(pageAddr)); } finally { - mem.readUnlock(id.cacheId(), id.pageId(), pageApsPtr); + mem.readUnlock(id.groupId(), id.pageId(), pageApsPtr); } } finally { - mem.releasePage(id.cacheId(), id.pageId(), pageApsPtr); + mem.releasePage(id.groupId(), id.pageId(), pageApsPtr); } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java index e086258613085..c248c35d19020 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java @@ -213,18 +213,18 @@ private void checkRestore(IgniteEx ig, FullPageId[] pages) throws IgniteCheckedE PageMemory mem = shared.database().memoryPolicy(null).pageMemory(); for (FullPageId fullId : pages) { - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { - long pageAddr = mem.readLock(fullId.cacheId(), fullId.pageId(), page); + long pageAddr = mem.readLock(fullId.groupId(), fullId.pageId(), page); for (int j = PageIO.COMMON_HEADER_END; j < mem.pageSize(); j += 4) assertEquals(j + (int)fullId.pageId(), PageUtils.getInt(pageAddr, j)); - mem.readUnlock(fullId.cacheId(), fullId.pageId(), page); + mem.readUnlock(fullId.groupId(), fullId.pageId(), page); } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } } @@ -252,10 +252,10 @@ private void generateWal( for (int i = 0; i < totalPages; i++) { FullPageId fullId = pages[i]; - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { - long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page); + long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page); PageIO.setPageId(pageAddr, fullId.pageId()); @@ -264,11 +264,11 @@ private void generateWal( PageUtils.putInt(pageAddr, j, j + (int)fullId.pageId()); } finally { - mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true); + mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true); } } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } @@ -340,17 +340,17 @@ private void generateWal( wal.fsync(wal.log(new CheckpointRecord(null))); for (FullPageId fullId : pages) { - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { assertFalse("Page has a temp heap copy after the last checkpoint: [cacheId=" + - fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.hasTempCopy(page)); + fullId.groupId() + ", pageId=" + fullId.pageId() + "]", mem.hasTempCopy(page)); assertFalse("Page is dirty after the last checkpoint: [cacheId=" + - fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); + fullId.groupId() + ", pageId=" + fullId.pageId() + "]", mem.isDirty(fullId.groupId(), fullId.pageId(), page)); } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java index e70c56e47d6d3..297e69f1baead 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java @@ -228,9 +228,9 @@ public void testGetForInitialWrite() throws Exception { // Check getForInitialWrite methods. for (FullPageId fullId : initWrites) { - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { - long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page); + long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page); try { DataPageIO.VERSIONS.latest().initNewPage(pageAddr, fullId.pageId(), mem.pageSize()); @@ -239,11 +239,11 @@ public void testGetForInitialWrite() throws Exception { PageUtils.putByte(pageAddr, i, (byte)0xAB); } finally { - mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true); + mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true); } } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } @@ -527,26 +527,26 @@ public void testDirtyFlag() throws Exception { pageIds[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId); for (FullPageId fullId : pageIds) { - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { - assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); //page is dirty right after allocation + assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page)); //page is dirty right after allocation - long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page); + long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page); PageIO.setPageId(pageAddr, fullId.pageId()); try { - assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); + assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page)); } finally { - mem.writeUnlock(fullId.cacheId(), fullId.pageId(),page, null,true); + mem.writeUnlock(fullId.groupId(), fullId.pageId(),page, null,true); } - assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); + assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page)); } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } } @@ -564,24 +564,24 @@ public void testDirtyFlag() throws Exception { ByteBuffer buf = ByteBuffer.allocate(mem.pageSize()); - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { - assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); + assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page)); - long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page); + long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page); try { - assertFalse(mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); + assertFalse(mem.isDirty(fullId.groupId(), fullId.pageId(), page)); for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) PageUtils.putByte(pageAddr, i, (byte)1); } finally { - mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true); + mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true); } - assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); + assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page)); buf.rewind(); @@ -593,7 +593,7 @@ public void testDirtyFlag() throws Exception { assertEquals((byte)0, buf.get()); } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } } @@ -604,12 +604,12 @@ public void testDirtyFlag() throws Exception { mem.finishCheckpoint(); for (FullPageId fullId : pageIds) { - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { - assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); + assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page)); } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } } @@ -618,9 +618,9 @@ public void testDirtyFlag() throws Exception { * @throws Exception if failed. */ private void writePageData(FullPageId fullId, PageMemory mem) throws Exception { - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { - long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page); + long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page); try { DataPageIO.VERSIONS.latest().initNewPage(pageAddr, fullId.pageId(), mem.pageSize()); @@ -631,11 +631,11 @@ private void writePageData(FullPageId fullId, PageMemory mem) throws Exception { PageUtils.putByte(pageAddr, i, (byte)rnd.nextInt(255)); } finally { - mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true); + mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true); } } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } @@ -693,9 +693,9 @@ else if (rec instanceof PageSnapshot) { assertNotNull("Missing WAL record for a written page: " + fullId, walData); - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { - long pageAddr = mem.readLock(fullId.cacheId(), fullId.pageId(), page); + long pageAddr = mem.readLock(fullId.groupId(), fullId.pageId(), page); try { for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) { @@ -707,11 +707,11 @@ else if (rec instanceof PageSnapshot) { } } finally { - mem.readUnlock(fullId.cacheId(), fullId.pageId(), page); + mem.readUnlock(fullId.groupId(), fullId.pageId(), page); } } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } } @@ -772,10 +772,10 @@ private IgniteBiTuple, WALPointer> runCheckpointing( ig.context().cache().context().database().checkpointReadLock(); try { - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { - long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page); + long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page); PageIO.setPageId(pageAddr, fullId.pageId()); @@ -813,11 +813,11 @@ private IgniteBiTuple, WALPointer> runCheckpointing( resMap.put(fullId, state); } finally { - mem.writeUnlock(fullId.cacheId(), fullId.pageId(),page, null,true); + mem.writeUnlock(fullId.groupId(), fullId.pageId(),page, null,true); } } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(),page);} + mem.releasePage(fullId.groupId(), fullId.pageId(),page);} } finally { ig.context().cache().context().database().checkpointReadUnlock(); @@ -958,17 +958,17 @@ private IgniteBiTuple, WALPointer> runCheckpointing( for (FullPageId fullId : pages) { - long page = mem.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = mem.acquirePage(fullId.groupId(), fullId.pageId()); try { assertFalse("Page has a temp heap copy after the last checkpoint: [cacheId=" + - fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.hasTempCopy(page)); + fullId.groupId() + ", pageId=" + fullId.pageId() + "]", mem.hasTempCopy(page)); assertFalse("Page is dirty after the last checkpoint: [cacheId=" + - fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); + fullId.groupId() + ", pageId=" + fullId.pageId() + "]", mem.isDirty(fullId.groupId(), fullId.pageId(), page)); } finally { - mem.releasePage(fullId.cacheId(), fullId.pageId(), page); + mem.releasePage(fullId.groupId(), fullId.pageId(), page); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java index ea4a0e9426b77..d7790c36e9362 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java @@ -203,10 +203,10 @@ private IgniteInternalFuture runWriteInThread( try { FullPageId fullId = pageIds.get(i); - long page = memory.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = memory.acquirePage(fullId.groupId(), fullId.pageId()); try { - final long pageAddr = memory.writeLock(fullId.cacheId(), fullId.pageId(), page); + final long pageAddr = memory.writeLock(fullId.groupId(), fullId.pageId(), page); try { PageIO.setPageId(pageAddr, fullId.pageId()); @@ -214,11 +214,11 @@ private IgniteInternalFuture runWriteInThread( PageUtils.putLong(pageAddr, PageIO.COMMON_HEADER_END, i * 2); } finally { - memory.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true); + memory.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true); } } finally { - memory.releasePage(fullId.cacheId(), fullId.pageId(), page); + memory.releasePage(fullId.groupId(), fullId.pageId(), page); } } finally { @@ -252,19 +252,19 @@ private IgniteInternalFuture runReadInThread(final IgniteEx ignite, final int st try { final FullPageId fullId = pageIds.get(i); - long page = memory.acquirePage(fullId.cacheId(), fullId.pageId()); + long page = memory.acquirePage(fullId.groupId(), fullId.pageId()); try { - final long pageAddr = memory.readLock(fullId.cacheId(), fullId.pageId(), page); + final long pageAddr = memory.readLock(fullId.groupId(), fullId.pageId(), page); try { assertEquals(i * 2, PageUtils.getLong(pageAddr, PageIO.COMMON_HEADER_END)); } finally { - memory.readUnlock(fullId.cacheId(), fullId.pageId(), page); + memory.readUnlock(fullId.groupId(), fullId.pageId(), page); } } finally { - memory.releasePage(fullId.cacheId(), fullId.pageId(), page); + memory.releasePage(fullId.groupId(), fullId.pageId(), page); } } finally { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java index 843fb5bf8c8b0..c5d6a8b16d7c3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java @@ -934,7 +934,7 @@ public void testApplyDeltaRecords() throws Exception { else if (rec instanceof PageDeltaRecord) { PageDeltaRecord delta = (PageDeltaRecord)rec; - FullPageId fullId = new FullPageId(delta.pageId(), delta.cacheId()); + FullPageId fullId = new FullPageId(delta.pageId(), delta.groupId()); byte[] pageData = rolledPages.get(fullId); @@ -976,10 +976,10 @@ else if (rec instanceof PageDeltaRecord) { ignite0.context().cache().context().database().checkpointReadLock(); try { - long page = pageMem.acquirePage(fullId.cacheId(), fullId.pageId(), true); + long page = pageMem.acquirePage(fullId.groupId(), fullId.pageId(), true); try { - long buf = pageMem.writeLock(fullId.cacheId(), fullId.pageId(), page, true); + long buf = pageMem.writeLock(fullId.groupId(), fullId.pageId(), page, true); try { byte[] data = entry.getValue(); @@ -992,11 +992,11 @@ else if (rec instanceof PageDeltaRecord) { } } finally { - pageMem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, false, true); + pageMem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, false, true); } } finally { - pageMem.releasePage(fullId.cacheId(), fullId.pageId(), page); + pageMem.releasePage(fullId.groupId(), fullId.pageId(), page); } } finally { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java index 665bb566598ad..8fc2bdb98f77d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java @@ -74,7 +74,7 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { } /** {@inheritDoc} */ - @Override public void read(int cacheId, long pageId, ByteBuffer pageBuf) throws IgniteCheckedException { + @Override public void read(int grpId, long pageId, ByteBuffer pageBuf) throws IgniteCheckedException { } @@ -84,35 +84,35 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { } /** {@inheritDoc} */ - @Override public void readHeader(int cacheId, int partId, ByteBuffer buf) throws IgniteCheckedException { + @Override public void readHeader(int grpId, int partId, ByteBuffer buf) throws IgniteCheckedException { // No-op. } /** {@inheritDoc} */ - @Override public void write(int cacheId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException { + @Override public void write(int grpId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException { // No-op. } /** {@inheritDoc} */ - @Override public void sync(int cacheId, int partId) throws IgniteCheckedException { + @Override public void sync(int grpId, int partId) throws IgniteCheckedException { // No-op. } /** {@inheritDoc} */ - @Override public void ensure(int cacheId, int partId) throws IgniteCheckedException { + @Override public void ensure(int grpId, int partId) throws IgniteCheckedException { // No-op. } /** {@inheritDoc} */ - @Override public long pageOffset(int cacheId, long pageId) throws IgniteCheckedException { + @Override public long pageOffset(int grpId, long pageId) throws IgniteCheckedException { return 0; } /** {@inheritDoc} */ - @Override public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException { + @Override public long allocatePage(int grpId, int partId, byte flags) throws IgniteCheckedException { long root = PageIdUtils.pageId(partId, flags, 0); - FullPageId fullId = new FullPageId(root, cacheId); + FullPageId fullId = new FullPageId(root, grpId); AtomicInteger allocator = allocators.get(fullId); @@ -123,10 +123,10 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { } /** {@inheritDoc} */ - @Override public int pages(int cacheId, int partId) throws IgniteCheckedException { + @Override public int pages(int grpId, int partId) throws IgniteCheckedException { long root = PageIdUtils.pageId(partId, (byte)0, 0); - FullPageId fullId = new FullPageId(root, cacheId); + FullPageId fullId = new FullPageId(root, grpId); AtomicInteger allocator = allocators.get(fullId); @@ -137,7 +137,7 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { } /** {@inheritDoc} */ - @Override public long metaPageId(int cacheId) { + @Override public long metaPageId(int grpId) { return 1; } From c52d2bf312b3c950a187db74fff5a7348e3709c1 Mon Sep 17 00:00:00 2001 From: tledkov-gridgain Date: Mon, 10 Jul 2017 15:19:58 +0300 Subject: [PATCH 121/155] IGNITE-5679: Example for thin JDBC driver. This closes #2232. --- .../datagrid/CacheQueryDdlExample.java | 26 ++-- .../ignite/examples/datagrid/JdbcExample.java | 135 ++++++++++++++++++ .../CacheExamplesMultiNodeSelfTest.java | 9 ++ .../examples/CacheExamplesSelfTest.java | 2 +- 4 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 examples/src/main/java/org/apache/ignite/examples/datagrid/JdbcExample.java diff --git a/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java b/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java index 84a67cd0c18ea..201dda10bd8ac 100644 --- a/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java +++ b/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java @@ -56,33 +56,33 @@ public static void main(String[] args) throws Exception { try ( IgniteCache cache = ignite.getOrCreateCache(cacheCfg) ) { - // Create table based on PARTITIONED template with one backup. - cache.query(new SqlFieldsQuery( - "CREATE TABLE person (id LONG PRIMARY KEY, name VARCHAR, city_id LONG) " + - "WITH \"backups=1\"")).getAll(); - // Create reference City table based on REPLICATED template. cache.query(new SqlFieldsQuery( "CREATE TABLE city (id LONG PRIMARY KEY, name VARCHAR) WITH \"template=replicated\"")).getAll(); + // Create table based on PARTITIONED template with one backup. + cache.query(new SqlFieldsQuery( + "CREATE TABLE person (id LONG, name VARCHAR, city_id LONG, PRIMARY KEY (id, city_id)) " + + "WITH \"backups=1, affinityKey=city_id\"")).getAll(); + // Create an index. cache.query(new SqlFieldsQuery("CREATE INDEX on Person (city_id)")).getAll(); print("Created database objects."); - SqlFieldsQuery qry = new SqlFieldsQuery("INSERT INTO person (id, name, city_id) values (?, ?, ?)"); + SqlFieldsQuery qry = new SqlFieldsQuery("INSERT INTO city (id, name) VALUES (?, ?)"); + + cache.query(qry.setArgs(1L, "Forest Hill")).getAll(); + cache.query(qry.setArgs(2L, "Denver")).getAll(); + cache.query(qry.setArgs(3L, "St. Petersburg")).getAll(); + + qry = new SqlFieldsQuery("INSERT INTO person (id, name, city_id) values (?, ?, ?)"); cache.query(qry.setArgs(1L, "John Doe", 3L)).getAll(); cache.query(qry.setArgs(2L, "Jane Roe", 2L)).getAll(); cache.query(qry.setArgs(3L, "Mary Major", 1L)).getAll(); cache.query(qry.setArgs(4L, "Richard Miles", 2L)).getAll(); - qry = new SqlFieldsQuery("INSERT INTO city (id, name) VALUES (?, ?)"); - - cache.query(qry.setArgs(1L, "Forest Hill")).getAll(); - cache.query(qry.setArgs(2L, "Denver")).getAll(); - cache.query(qry.setArgs(3L, "St. Petersburg")).getAll(); - print("Populated data."); List> res = cache.query(new SqlFieldsQuery( @@ -91,7 +91,7 @@ public static void main(String[] args) throws Exception { print("Query results:"); for (Object next : res) - System.out.println(">>> " + next); + System.out.println(">>> " + next); cache.query(new SqlFieldsQuery("drop table Person")).getAll(); cache.query(new SqlFieldsQuery("drop table City")).getAll(); diff --git a/examples/src/main/java/org/apache/ignite/examples/datagrid/JdbcExample.java b/examples/src/main/java/org/apache/ignite/examples/datagrid/JdbcExample.java new file mode 100644 index 0000000000000..bc96e427e7ebb --- /dev/null +++ b/examples/src/main/java/org/apache/ignite/examples/datagrid/JdbcExample.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.examples.datagrid; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import org.apache.ignite.examples.ExampleNodeStartup; + +/** + * This example demonstrates usage of Ignite JDBC driver. + *

+ * Ignite nodes must be started in separate process using {@link ExampleNodeStartup} before running this example. + */ +public class JdbcExample { + /** + * Executes example. + * + * @param args Command line arguments, none required. + * @throws Exception If example execution failed. + */ + public static void main(String[] args) throws Exception { + print("JDBC example started."); + + // Open JDBC connection + try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1/")) { + print("Connected to server."); + + // Create database objects. + try (Statement stmt = conn.createStatement()) { + // Create reference City table based on REPLICATED template. + stmt.executeUpdate("CREATE TABLE city (id LONG PRIMARY KEY, name VARCHAR) " + + "WITH \"template=replicated\""); + + // Create table based on PARTITIONED template with one backup. + stmt.executeUpdate("CREATE TABLE person (id LONG, name VARCHAR, city_id LONG, " + + "PRIMARY KEY (id, city_id)) WITH \"backups=1, affinityKey=city_id\""); + + // Create an index. + stmt.executeUpdate("CREATE INDEX on Person (city_id)"); + } + + print("Created database objects."); + + // Populate City table with PreparedStatement. + try (PreparedStatement stmt = conn.prepareStatement("INSERT INTO city (id, name) VALUES (?, ?)")) { + stmt.setLong(1, 1L); + stmt.setString(2, "Forest Hill"); + stmt.executeUpdate(); + + stmt.setLong(1, 2L); + stmt.setString(2, "Denver"); + stmt.executeUpdate(); + + stmt.setLong(1, 3L); + stmt.setString(2, "St. Petersburg"); + stmt.executeUpdate(); + } + + // Populate Person table with PreparedStatement. + try (PreparedStatement stmt = + conn.prepareStatement("INSERT INTO person (id, name, city_id) values (?, ?, ?)")) { + stmt.setLong(1, 1L); + stmt.setString(2, "John Doe"); + stmt.setLong(3, 3L); + stmt.executeUpdate(); + + stmt.setLong(1, 2L); + stmt.setString(2, "Jane Roe"); + stmt.setLong(3, 2L); + stmt.executeUpdate(); + + stmt.setLong(1, 3L); + stmt.setString(2, "Mary Major"); + stmt.setLong(3, 1L); + stmt.executeUpdate(); + + stmt.setLong(1, 4L); + stmt.setString(2, "Richard Miles"); + stmt.setLong(3, 2L); + stmt.executeUpdate(); + } + + print("Populated data."); + + // Get data. + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = + stmt.executeQuery("SELECT p.name, c.name FROM Person p INNER JOIN City c on c.id = p.city_id")) { + print("Query results:"); + + while (rs.next()) + System.out.println(">>> " + rs.getString(1) + ", " + rs.getString(2)); + } + } + + // Drop database objects. + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE Person"); + stmt.executeUpdate("DROP TABLE City"); + } + + print("Dropped database objects."); + } + + print("JDBC example finished."); + } + + /** + * Prints message. + * + * @param msg Message to print before all objects are printed. + */ + private static void print(String msg) { + System.out.println(); + System.out.println(">>> " + msg); + } +} \ No newline at end of file diff --git a/examples/src/test/java/org/apache/ignite/examples/CacheExamplesMultiNodeSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/CacheExamplesMultiNodeSelfTest.java index 6de0273aa5ca0..f940ff7c16146 100644 --- a/examples/src/test/java/org/apache/ignite/examples/CacheExamplesMultiNodeSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/CacheExamplesMultiNodeSelfTest.java @@ -17,6 +17,8 @@ package org.apache.ignite.examples; +import org.apache.ignite.examples.datagrid.JdbcExample; + /** * Cache examples multi-node self test. */ @@ -42,4 +44,11 @@ public class CacheExamplesMultiNodeSelfTest extends CacheExamplesSelfTest { super.testCacheLockExample(); } + + /** + * @throws Exception If failed. + */ + public void testJdbcThinExample() throws Exception { + JdbcExample.main(EMPTY_ARGS); + } } \ No newline at end of file diff --git a/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java b/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java index f65d97cacfabe..30f0763f686dc 100644 --- a/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java +++ b/examples/src/test/java/org/apache/ignite/examples/CacheExamplesSelfTest.java @@ -139,7 +139,7 @@ public void testCacheQueryDmlExample() throws Exception { /** * @throws Exception If failed. */ - public void testCacheQUeryDdlExample() throws Exception { + public void testCacheQueryDdlExample() throws Exception { CacheQueryDdlExample.main(EMPTY_ARGS); } From 92becc8918f0cfc0a0c0d94c19a93d50d5ffb10e Mon Sep 17 00:00:00 2001 From: sboikov Date: Mon, 10 Jul 2017 16:59:48 +0300 Subject: [PATCH 122/155] Removed unused method. --- .../GridCachePartitionExchangeManager.java | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index f1db79a8adb4f..de0adc791c50c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -89,7 +89,6 @@ import org.apache.ignite.internal.util.GridPartitionStateMap; import org.apache.ignite.internal.util.future.GridCompoundFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; -import org.apache.ignite.internal.util.lang.IgnitePair; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.CI1; @@ -101,7 +100,6 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; import org.apache.ignite.lang.IgniteBiInClosure; -import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.thread.IgniteThread; import org.jetbrains.annotations.Nullable; @@ -159,10 +157,6 @@ public class GridCachePartitionExchangeManager extends GridCacheSharedMana private final ConcurrentMap readyFuts = new ConcurrentSkipListMap<>(); - /** */ - private final ConcurrentSkipListMap> nodeVers = - new ConcurrentSkipListMap<>(); - /** */ private final AtomicReference readyTopVer = new AtomicReference<>(AffinityTopologyVersion.NONE); @@ -805,18 +799,6 @@ else if (stopErr != null) return fut; } - /** - * Gets minimum node version for the given topology version. - * - * @param topVer Topology version to get minimum node version for. - * @return Minimum node version. - */ - public IgniteProductVersion minimumNodeVersion(AffinityTopologyVersion topVer) { - IgnitePair vers = nodeVers.get(topVer); - - return vers == null ? cctx.localNode().version() : vers.get1(); - } - /** * @return {@code true} if entered to busy state. */ @@ -1257,30 +1239,6 @@ public void onExchangeDone(GridDhtPartitionsExchangeFuture exchFut, @Nullable Th if (log.isDebugEnabled()) log.debug("Exchange done [topVer=" + topVer + ", fut=" + exchFut + ", err=" + err + ']'); - IgniteProductVersion minVer = cctx.localNode().version(); - IgniteProductVersion maxVer = cctx.localNode().version(); - - if (err == null) { - if (!F.isEmpty(exchFut.discoveryEvent().topologyNodes())) { - for (ClusterNode node : exchFut.discoveryEvent().topologyNodes()) { - IgniteProductVersion ver = node.version(); - - if (ver.compareTo(minVer) < 0) - minVer = ver; - - if (ver.compareTo(maxVer) > 0) - maxVer = ver; - } - } - } - - nodeVers.put(topVer, new IgnitePair<>(minVer, maxVer)); - - AffinityTopologyVersion histVer = new AffinityTopologyVersion(topVer.topologyVersion() - 10, 0); - - for (AffinityTopologyVersion oldVer : nodeVers.headMap(histVer).keySet()) - nodeVers.remove(oldVer); - if (err == null) { while (true) { AffinityTopologyVersion readyVer = readyTopVer.get(); From 2ac39082c6aa660085ea94df121166920821b427 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Mon, 10 Jul 2017 17:05:08 +0300 Subject: [PATCH 123/155] IGNITE-2492 .NET: Peer assembly loading - add TestCacheGetOnRemoteNode --- .../Apache.Ignite.Core.Tests.csproj | 1 + .../Deployment/CacheGetFunc.cs | 50 +++++++++++++++++++ .../Deployment/PeerAssemblyLoadingTest.cs | 24 +++++++++ 3 files changed, 75 insertions(+) create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/Deployment/CacheGetFunc.cs diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj index d91f0e6b0e57f..4787c4d6c7403 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj @@ -78,6 +78,7 @@ + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Deployment/CacheGetFunc.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Deployment/CacheGetFunc.cs new file mode 100644 index 0000000000000..2679130527bc7 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Deployment/CacheGetFunc.cs @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.Deployment +{ + using Apache.Ignite.Core.Compute; + using Apache.Ignite.Core.Resource; + + ///

+ /// Function to get a value from cache. + /// + public class CacheGetFunc : IComputeFunc + { + /// + /// Gets or sets the cache key. + /// + public TKey Key { get; set; } + + /// + /// Gets or sets the name of the cache. + /// + public string CacheName { get; set; } + + /// + /// Gets or sets the ignite. + /// + [InstanceResource] + public IIgnite Ignite { get;set; } + + /** */ + public TValue Invoke() + { + return Ignite.GetCache(CacheName).Get(Key); + } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Deployment/PeerAssemblyLoadingTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Deployment/PeerAssemblyLoadingTest.cs index 8245333dd269d..da896ddfbc49f 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Deployment/PeerAssemblyLoadingTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Deployment/PeerAssemblyLoadingTest.cs @@ -130,6 +130,30 @@ public void TestRuntimeDependency() }); } + /// + /// Tests the cache get operation on remote node. + /// + [Test] + public void TestCacheGetOnRemoteNode() + { + TestDeployment(remoteCompute => + { + var cache = remoteCompute.ClusterGroup.Ignite.GetOrCreateCache("addr"); + cache[1] = new Address("street", 123); + + // This will fail for func, because cache operations are not p2p-enabled. + // However, generic nature of the func causes Address to be peer-deployed before cache.Get call. + var func = new CacheGetFunc + { + CacheName = cache.Name, + Key = 1 + }; + + var res = remoteCompute.Call(func); + Assert.AreEqual("street", res.Street); + }); + } + /// /// Tests the peer deployment. /// From 33ae472fc47c8983e48ac0febe22f11457c89ba1 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Mon, 10 Jul 2017 17:18:18 +0300 Subject: [PATCH 124/155] Added partition release future timing --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 21a3a1373c488..5760f875f8425 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -1042,6 +1042,8 @@ private void waitPartitionRelease() throws IgniteCheckedException { int dumpCnt = 0; + long waitStart = U.currentTimeMillis(); + long nextDumpTime = 0; long futTimeout = 2 * cctx.gridConfig().getNetworkTimeout(); @@ -1062,8 +1064,11 @@ private void waitPartitionRelease() throws IgniteCheckedException { } } - if (log.isDebugEnabled()) - log.debug("After waiting for partition release future: " + this); + long waitEnd = U.currentTimeMillis(); + + if (log.isInfoEnabled()) + log.info("Finished waiting for partition release future [topVer=" + exchangeId().topologyVersion() + + ", waitTime=" + (waitEnd - waitStart) + "ms]"); IgniteInternalFuture locksFut = cctx.mvcc().finishLocks(exchId.topologyVersion()); From d2fe437426399e2cf36b00ce6f527731da88613e Mon Sep 17 00:00:00 2001 From: artemmalykh Date: Mon, 10 Jul 2017 18:16:49 +0300 Subject: [PATCH 125/155] IGNITE-5688: Fix SparseDistributedMatrixTest::testCacheBehaviour crash. This closes #2272 --- modules/ml/pom.xml | 3 +- .../main/java/org/apache/ignite/ml/Model.java | 1 - .../ml/clustering/BaseKMeansClusterer.java | 2 - .../ignite/ml/clustering/Clusterer.java | 3 +- .../KMeansDistributedClusterer.java | 46 +++++++++---------- .../ml/clustering/KMeansLocalClusterer.java | 11 +++-- .../ignite/ml/math/DistanceMeasure.java | 1 - .../ignite/ml/math/EuclideanDistance.java | 2 +- .../org/apache/ignite/ml/math/Matrix.java | 2 - .../org/apache/ignite/ml/math/Tracer.java | 1 - .../apache/ignite/ml/math/VectorUtils.java | 4 +- .../math/exceptions/ConvergenceException.java | 5 +- .../exceptions/MathIllegalStateException.java | 4 +- .../ignite/ml/math/functions/Functions.java | 8 +--- .../ignite/ml/math/impls/CacheUtils.java | 23 ++++++---- .../ml/math/impls/matrix/AbstractMatrix.java | 2 - .../impls/matrix/SparseDistributedMatrix.java | 4 +- .../storage/matrix/MapWrapperStorage.java | 7 ++- .../SparseDistributedMatrixStorage.java | 7 ++- .../SparseLocalOnHeapVectorStorage.java | 8 ++-- .../math/impls/vector/MapWrapperVector.java | 3 +- .../ignite/ml/math/util/MatrixUtil.java | 1 - .../ignite/ml/math/d3-matrix-template.html | 2 + .../ignite/ml/math/d3-vector-template.html | 2 + .../apache/ignite/ml/IgniteMLTestSuite.java | 4 +- ...setSuite.java => ClusteringTestSuite.java} | 6 +-- .../KMeansDistributedClustererTest.java | 28 +++++++---- .../ml/math/MathImplLocalTestSuite.java | 1 - .../org/apache/ignite/ml/math/TracerTest.java | 3 ++ .../matrix/SparseDistributedMatrixTest.java | 24 ++++++---- .../OLSMultipleLinearRegressionTest.java | 2 +- 31 files changed, 115 insertions(+), 105 deletions(-) rename modules/ml/src/test/java/org/apache/ignite/ml/clustering/{ClusteringTesetSuite.java => ClusteringTestSuite.java} (89%) diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml index 75ebf1b4e0834..d6197199fd6be 100644 --- a/modules/ml/pom.xml +++ b/modules/ml/pom.xml @@ -19,7 +19,8 @@ - + 4.0.0 diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java index e4d2117640faa..4dc1aca8b9213 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/Model.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/Model.java @@ -19,7 +19,6 @@ import java.util.function.BiFunction; - /** Basic interface for all models. */ @FunctionalInterface public interface Model { diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java index a6acb8eb9564d..570ea7e7a081d 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java @@ -18,14 +18,12 @@ package org.apache.ignite.ml.clustering; import java.util.List; - import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.ml.math.DistanceMeasure; import org.apache.ignite.ml.math.Matrix; import org.apache.ignite.ml.math.Vector; import org.apache.ignite.ml.math.exceptions.ConvergenceException; import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException; -import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; /** * This class is partly based on the corresponding class from Apache Common Math lib. diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java index f03dc951b9cea..204e28a7ab8f5 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java @@ -23,7 +23,8 @@ * Base interface for clusterers. */ public interface Clusterer { - /** Cluster given points set into k clusters. + /** + * Cluster given points set into k clusters. * * @param points Points set. * @param k Clusters count. diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java index 09317d60acd2d..5f702d2f38078 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java @@ -22,12 +22,13 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; - +import javax.cache.Cache; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteUuid; -import org.apache.ignite.ml.math.*; +import org.apache.ignite.ml.math.DistanceMeasure; +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.VectorUtils; import org.apache.ignite.ml.math.exceptions.ConvergenceException; import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException; import org.apache.ignite.ml.math.functions.Functions; @@ -39,9 +40,7 @@ import org.apache.ignite.ml.math.util.MapUtil; import org.apache.ignite.ml.math.util.MatrixUtil; -import javax.cache.Cache; - -import static org.apache.ignite.ml.math.impls.CacheUtils.*; +import static org.apache.ignite.ml.math.impls.CacheUtils.distributedFold; import static org.apache.ignite.ml.math.util.MatrixUtil.localCopyOf; /** @@ -78,8 +77,8 @@ public KMeansDistributedClusterer(DistanceMeasure measure, int initSteps, int ma /** */ @Override public KMeansModel cluster(SparseDistributedMatrix points, int k) throws - MathIllegalArgumentException, ConvergenceException { - SparseDistributedMatrix pointsCp = (SparseDistributedMatrix) points.like(points.rowSize(), points.columnSize()); + MathIllegalArgumentException, ConvergenceException { + SparseDistributedMatrix pointsCp = (SparseDistributedMatrix)points.like(points.rowSize(), points.columnSize()); // TODO: this copy is very ineffective, just for POC. Immutability of data should be guaranteed by other methods // such as logical locks for example. @@ -174,7 +173,8 @@ private Vector[] initClusterCenters(SparseDistributedMatrix points, int k) { } /** */ - private List getNewCenters(int k, ConcurrentHashMap costs, IgniteUuid uid, double sumCosts) { + private List getNewCenters(int k, ConcurrentHashMap costs, IgniteUuid uid, + double sumCosts) { return distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (IgniteBiFunction, Map>, List, @@ -253,25 +253,25 @@ private double distance(Map vecMap, Vector vector) { /** */ private SumsAndCounts getSumsAndCounts(Vector[] centers, int dim, IgniteUuid uid) { return CacheUtils.distributedFold(SparseDistributedMatrixStorage.ML_CACHE_NAME, - (IgniteBiFunction, Map>, SumsAndCounts, SumsAndCounts>)(entry, counts) -> { - Map vec = entry.getValue(); + (IgniteBiFunction, Map>, SumsAndCounts, SumsAndCounts>)(entry, counts) -> { + Map vec = entry.getValue(); - IgniteBiTuple closest = findClosest(centers, VectorUtils.fromMap(vec, false)); - int bestCenterIdx = closest.get1(); + IgniteBiTuple closest = findClosest(centers, VectorUtils.fromMap(vec, false)); + int bestCenterIdx = closest.get1(); - counts.totalCost += closest.get2(); - counts.sums.putIfAbsent(bestCenterIdx, VectorUtils.zeroes(dim)); + counts.totalCost += closest.get2(); + counts.sums.putIfAbsent(bestCenterIdx, VectorUtils.zeroes(dim)); - counts.sums.compute(bestCenterIdx, - (IgniteBiFunction)(ind, v) -> v.plus(VectorUtils.fromMap(vec, false))); + counts.sums.compute(bestCenterIdx, + (IgniteBiFunction)(ind, v) -> v.plus(VectorUtils.fromMap(vec, false))); - counts.counts.merge(bestCenterIdx, 1, - (IgniteBiFunction)(i1, i2) -> i1 + i2); + counts.counts.merge(bestCenterIdx, 1, + (IgniteBiFunction)(i1, i2) -> i1 + i2); - return counts; - }, - key -> key.get2().equals(uid), - SumsAndCounts::merge, new SumsAndCounts() + return counts; + }, + key -> key.get2().equals(uid), + SumsAndCounts::merge, new SumsAndCounts() ); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java index c98b818744cc0..9f2aefbeaf69f 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java @@ -22,9 +22,11 @@ import java.util.Collections; import java.util.List; import java.util.Random; - import org.apache.ignite.internal.util.GridArgumentCheck; -import org.apache.ignite.ml.math.*; +import org.apache.ignite.ml.math.DistanceMeasure; +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.VectorUtils; import org.apache.ignite.ml.math.exceptions.ConvergenceException; import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException; import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; @@ -97,7 +99,7 @@ public KMeansLocalClusterer(DistanceMeasure measure, int maxIterations, Long see for (int p = 0; p < points.rowSize(); p++) costs.setX(p, Math.min(getDistanceMeasure().compute(localCopyOf(points.viewRow(p)), centers[i]), - costs.get(p))); + costs.get(p))); } int[] oldClosest = new int[points.rowSize()]; @@ -135,7 +137,8 @@ public KMeansLocalClusterer(DistanceMeasure measure, int maxIterations, Long see if (counts[j] == 0.0) { // Assign center to a random point centers[j] = points.viewRow(rand.nextInt(points.rowSize())); - } else { + } + else { sums[j] = sums[j].times(1.0 / counts[j]); centers[j] = sums[j]; } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java index 0fd74ac31f8be..09be0c3258766 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java @@ -17,7 +17,6 @@ package org.apache.ignite.ml.math; import java.io.Externalizable; - import org.apache.ignite.ml.math.exceptions.CardinalityException; /** diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java index b748ac55faae5..5f962ce1ec4ca 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java @@ -32,7 +32,7 @@ public class EuclideanDistance implements DistanceMeasure { /** {@inheritDoc} */ @Override public double compute(Vector a, Vector b) - throws CardinalityException { + throws CardinalityException { return MatrixUtil.localCopyOf(a).minus(b).kNorm(2.0); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java index db822e6e555c4..2cf4e6381268e 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/Matrix.java @@ -18,8 +18,6 @@ package org.apache.ignite.ml.math; import java.io.Externalizable; -import java.util.function.BiFunction; -import java.util.function.BinaryOperator; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.ml.math.exceptions.CardinalityException; import org.apache.ignite.ml.math.exceptions.IndexException; diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/Tracer.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/Tracer.java index d343ce80b3aa4..1928f46f0b42f 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/Tracer.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/Tracer.java @@ -32,7 +32,6 @@ import java.util.Locale; import java.util.function.Function; import java.util.stream.Collectors; - import org.apache.ignite.IgniteLogger; import org.apache.ignite.lang.IgniteUuid; diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java index f41a5fed71661..061d5ae163f1c 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/VectorUtils.java @@ -18,10 +18,8 @@ package org.apache.ignite.ml.math; import java.util.Map; -import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; import org.apache.ignite.ml.math.impls.vector.MapWrapperVector; -import org.apache.ignite.ml.math.impls.vector.SparseLocalVector; public class VectorUtils { /** Create new vector like given vector initialized by zeroes. */ @@ -31,7 +29,7 @@ public static Vector zeroesLike(Vector v) { /** Create new */ public static DenseLocalOnHeapVector zeroes(int n) { - return (DenseLocalOnHeapVector) new DenseLocalOnHeapVector(n).assign(0.0); + return (DenseLocalOnHeapVector)new DenseLocalOnHeapVector(n).assign(0.0); } /** */ diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java index 2cf0bcf579609..64687b6326d3e 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/ConvergenceException.java @@ -38,11 +38,10 @@ public ConvergenceException() { /** * Construct the exception with a specific context and arguments. * - * @param msg Message pattern providing the specific context of - * the error. + * @param msg Message pattern providing the specific context of the error. * @param args Arguments. */ - public ConvergenceException(String msg, Object ... args) { + public ConvergenceException(String msg, Object... args) { super(msg, args); } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java index 13ef5ca630a1a..6c63086bbec67 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/MathIllegalStateException.java @@ -27,7 +27,7 @@ public class MathIllegalStateException extends MathRuntimeException { private static final long serialVersionUID = -6024911025449780478L; /** */ - private static final String ILLEGAL_STATE= "Illegal state."; + private static final String ILLEGAL_STATE = "Illegal state."; /** * Simple constructor. @@ -35,7 +35,7 @@ public class MathIllegalStateException extends MathRuntimeException { * @param msg Message pattern explaining the cause of the error. * @param args Arguments. */ - public MathIllegalStateException(String msg, Object ... args) { + public MathIllegalStateException(String msg, Object... args) { super(msg, args); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java index 22a453d706a74..022dd04852388 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/Functions.java @@ -17,9 +17,8 @@ package org.apache.ignite.ml.math.functions; -import org.apache.ignite.lang.IgniteBiTuple; - import java.util.List; +import org.apache.ignite.lang.IgniteBiTuple; /** * Compatibility with Apache Mahout. @@ -88,11 +87,6 @@ public final class Functions { /** Function that returns {@code a < b ? -1 : a > b ? 1 : 0}. */ public static final IgniteBiFunction COMPARE = (a, b) -> a < b ? -1.0 : a > b ? 1.0 : 0.0; - /** */ - public static IgniteFunction curry(IgniteBiFunction f, A a) { - return (IgniteFunction)b -> f.apply(a, b); - } - /** */ public static > IgniteBiTuple argmin(List args, IgniteFunction f) { A res = null; diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java index 836789b31284f..1bda5e6f2b3e1 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java @@ -40,7 +40,6 @@ import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.functions.IgniteConsumer; import org.apache.ignite.ml.math.functions.IgniteFunction; -import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix; import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage; /** @@ -137,7 +136,8 @@ public static double sparseSum(IgniteUuid matrixUuid) { double sum = sum(map.values()); return acc == null ? sum : acc + sum; - } else + } + else return acc; }, key -> key.get2().equals(matrixUuid)); @@ -199,7 +199,8 @@ public static double sparseMin(IgniteUuid matrixUuid) { return min; else return Math.min(acc, min); - } else + } + else return acc; }, key -> key.get2().equals(matrixUuid)); @@ -222,7 +223,8 @@ public static double sparseMax(IgniteUuid matrixUuid) { return max; else return Math.max(acc, max); - } else + } + else return acc; }, key -> key.get2().equals(matrixUuid)); @@ -307,7 +309,8 @@ public static void foreach(String cacheName, IgniteConsumer Cache key object type. * @param Cache value object type. */ - public static void foreach(String cacheName, IgniteConsumer> fun, IgnitePredicate keyFilter) { + public static void foreach(String cacheName, IgniteConsumer> fun, + IgnitePredicate keyFilter) { bcast(cacheName, () -> { Ignite ignite = Ignition.localIgnite(); IgniteCache cache = ignite.getOrCreateCache(cacheName); @@ -355,7 +358,8 @@ public static Collection fold(String cacheName, IgniteBiFunction Fold result type. * @return Fold operation result. */ - public static Collection fold(String cacheName, IgniteBiFunction, A, A> folder, IgnitePredicate keyFilter) { + public static Collection fold(String cacheName, IgniteBiFunction, A, A> folder, + IgnitePredicate keyFilter) { return bcast(cacheName, () -> { Ignite ignite = Ignition.localIgnite(); IgniteCache cache = ignite.getOrCreateCache(cacheName); @@ -385,12 +389,13 @@ public static Collection fold(String cacheName, IgniteBiFunction A distributedFold(String cacheName, IgniteBiFunction, A, A> folder, IgnitePredicate keyFilter, BinaryOperator accumulator, A zeroVal) { - return sparseFold(cacheName, folder, keyFilter, accumulator, zeroVal, null, null, 0 , + return sparseFold(cacheName, folder, keyFilter, accumulator, zeroVal, null, null, 0, false); } private static A sparseFold(String cacheName, IgniteBiFunction, A, A> folder, - IgnitePredicate keyFilter, BinaryOperator accumulator, A zeroVal, V defVal, K defKey, long defValCnt, boolean isNilpotent) { + IgnitePredicate keyFilter, BinaryOperator accumulator, A zeroVal, V defVal, K defKey, long defValCnt, + boolean isNilpotent) { A defRes = zeroVal; @@ -417,7 +422,7 @@ private static A sparseFold(String cacheName, IgniteBiFunction entry : cache.query(new ScanQuery(part, - (k, v) -> affinity.mapPartitionToNode(p) == localNode && (keyFilter == null || keyFilter.apply(k))))) + (k, v) -> affinity.mapPartitionToNode(p) == localNode && (keyFilter == null || keyFilter.apply(k))))) a = folder.apply(entry, a); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java index 106a425fd2252..d1d3904f0be2f 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java @@ -24,8 +24,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Random; -import java.util.function.BiFunction; -import java.util.function.BinaryOperator; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.ml.math.Matrix; import org.apache.ignite.ml.math.MatrixStorage; diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java index 50c7da83112c6..df2ddc4b1c28c 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java @@ -147,7 +147,7 @@ private Matrix mapOverValues(IgniteFunction mapper) { } /** */ - public IgniteUuid getUUID(){ - return ((SparseDistributedMatrixStorage) getStorage()).getUUID(); + public IgniteUuid getUUID() { + return ((SparseDistributedMatrixStorage)getStorage()).getUUID(); } } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java index 6cefb255653c8..c0bcde0887aae 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/MapWrapperStorage.java @@ -17,14 +17,13 @@ package org.apache.ignite.ml.math.impls.storage.matrix; -import org.apache.ignite.internal.util.GridArgumentCheck; -import org.apache.ignite.ml.math.VectorStorage; - import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.Map; import java.util.Set; +import org.apache.ignite.internal.util.GridArgumentCheck; +import org.apache.ignite.ml.math.VectorStorage; /** * Storage for wrapping given map. @@ -80,7 +79,7 @@ else if (data.containsKey(i)) /** {@inheritDoc} */ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - data = (Map) in.readObject(); + data = (Map)in.readObject(); } /** {@inheritDoc} */ diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java index 816bf447ef937..1513502d98ea6 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/SparseDistributedMatrixStorage.java @@ -44,7 +44,7 @@ * {@link MatrixStorage} implementation for {@link SparseDistributedMatrix}. */ public class SparseDistributedMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants { - /** Cache name used for all instances of {@link SparseDistributedMatrixStorage}.*/ + /** Cache name used for all instances of {@link SparseDistributedMatrixStorage}. */ public static final String ML_CACHE_NAME = "ML_SPARSE_MATRICES_CONTAINER"; /** Amount of rows in the matrix. */ private int rows; @@ -93,7 +93,7 @@ public SparseDistributedMatrixStorage(int rows, int cols, int stoMode, int acsMo } /** - * Create new ML cache if needed. + * Create new ML cache if needed. */ private IgniteCache, Map> newCache() { CacheConfiguration, Map> cfg = new CacheConfiguration<>(); @@ -198,7 +198,6 @@ private void matrixSet(int a, int b, double v) { // Local get. Map map = cache.localPeek(getCacheKey(a), CachePeekMode.PRIMARY); - if (map == null) { map = cache.get(getCacheKey(a)); //Remote entry get. @@ -217,7 +216,7 @@ else if (map.containsKey(b)) } /** Build cache key for row/column. */ - private IgniteBiTuple getCacheKey(int idx){ + private IgniteBiTuple getCacheKey(int idx) { return new IgniteBiTuple<>(idx, uuid); } diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java index f07a16e9b0250..f2efe74121348 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java @@ -47,7 +47,6 @@ public SparseLocalOnHeapVectorStorage() { } /** - * * @param map */ public SparseLocalOnHeapVectorStorage(Map map, boolean copy) { @@ -57,11 +56,10 @@ public SparseLocalOnHeapVectorStorage(Map map, boolean copy) { if (map instanceof Int2DoubleRBTreeMap) acsMode = SEQUENTIAL_ACCESS_MODE; + else if (map instanceof Int2DoubleOpenHashMap) + acsMode = RANDOM_ACCESS_MODE; else - if (map instanceof Int2DoubleOpenHashMap) - acsMode = RANDOM_ACCESS_MODE; - else - acsMode = UNKNOWN_STORAGE_MODE; + acsMode = UNKNOWN_STORAGE_MODE; if (copy) switch (acsMode) { diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java index c04cc4562e20c..a093d0e68793e 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/vector/MapWrapperVector.java @@ -17,12 +17,11 @@ package org.apache.ignite.ml.math.impls.vector; +import java.util.Map; import org.apache.ignite.ml.math.Matrix; import org.apache.ignite.ml.math.Vector; import org.apache.ignite.ml.math.impls.storage.matrix.MapWrapperStorage; -import java.util.Map; - /** * Vector wrapping a given map. */ diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java index 5ef717692323a..7a90ba153e276 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MatrixUtil.java @@ -18,7 +18,6 @@ package org.apache.ignite.ml.math.util; import java.util.List; - import org.apache.ignite.internal.util.GridArgumentCheck; import org.apache.ignite.ml.math.Matrix; import org.apache.ignite.ml.math.Vector; diff --git a/modules/ml/src/main/resources/org/apache/ignite/ml/math/d3-matrix-template.html b/modules/ml/src/main/resources/org/apache/ignite/ml/math/d3-matrix-template.html index 0639f077c47df..055344b90c60f 100644 --- a/modules/ml/src/main/resources/org/apache/ignite/ml/math/d3-matrix-template.html +++ b/modules/ml/src/main/resources/org/apache/ignite/ml/math/d3-matrix-template.html @@ -39,6 +39,7 @@ margin-left: 5px; vertical-align: bottom; } + @@ -116,4 +117,5 @@ }) .attr("width", rw) .attr("height", rh); + \ No newline at end of file diff --git a/modules/ml/src/main/resources/org/apache/ignite/ml/math/d3-vector-template.html b/modules/ml/src/main/resources/org/apache/ignite/ml/math/d3-vector-template.html index 9593e86ccfb8a..a615b7c5fceb8 100644 --- a/modules/ml/src/main/resources/org/apache/ignite/ml/math/d3-vector-template.html +++ b/modules/ml/src/main/resources/org/apache/ignite/ml/math/d3-vector-template.html @@ -39,6 +39,7 @@ margin-left: 5px; vertical-align: bottom; } + @@ -101,4 +102,5 @@ }) .attr("width", rw) .attr("height", rh); + \ No newline at end of file diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java index dea3edfafdcd0..5ac744383a663 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java @@ -17,7 +17,7 @@ package org.apache.ignite.ml; -import org.apache.ignite.ml.clustering.ClusteringTesetSuite; +import org.apache.ignite.ml.clustering.ClusteringTestSuite; import org.apache.ignite.ml.math.MathImplMainTestSuite; import org.apache.ignite.ml.regressions.RegressionsTestSuite; import org.junit.runner.RunWith; @@ -30,7 +30,7 @@ @Suite.SuiteClasses({ MathImplMainTestSuite.class, RegressionsTestSuite.class, - ClusteringTesetSuite.class + ClusteringTestSuite.class }) public class IgniteMLTestSuite { // No-op. diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTestSuite.java similarity index 89% rename from modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java rename to modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTestSuite.java index fb756ba8c531b..c39eeefed10ce 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTesetSuite.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTestSuite.java @@ -25,8 +25,8 @@ */ @RunWith(Suite.class) @Suite.SuiteClasses({ - KMeansDistributedClustererTest.class, - KMeansLocalClustererTest.class + KMeansDistributedClustererTest.class, + KMeansLocalClustererTest.class }) -public class ClusteringTesetSuite { +public class ClusteringTestSuite { } diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java index 7fb3534dcd7e8..cdc2651393f9f 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTest.java @@ -18,13 +18,23 @@ package org.apache.ignite.ml.clustering; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.ignite.Ignite; import org.apache.ignite.internal.util.IgniteUtils; -import org.apache.ignite.ml.math.*; +import org.apache.ignite.ml.math.DistanceMeasure; +import org.apache.ignite.ml.math.EuclideanDistance; +import org.apache.ignite.ml.math.StorageConstants; import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.VectorUtils; import org.apache.ignite.ml.math.functions.Functions; import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix; import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; @@ -36,8 +46,10 @@ /** */ public class KMeansDistributedClustererTest extends GridCommonAbstractTest { - /** Number of nodes in grid. We should use 1 in this test because otherwise algorithm will be unstable - * (We cannot guarantee the order in which results are returned from each node). */ + /** + * Number of nodes in grid. We should use 1 in this test because otherwise algorithm will be unstable + * (We cannot guarantee the order in which results are returned from each node). + */ private static final int NODE_COUNT = 1; /** Grid instance. */ @@ -79,7 +91,7 @@ public void testPerformClusterAnalysisDegenerate() { double[] v2 = new double[] {1960, 373200}; SparseDistributedMatrix points = new SparseDistributedMatrix(2, 2, StorageConstants.ROW_STORAGE_MODE, - StorageConstants.RANDOM_ACCESS_MODE); + StorageConstants.RANDOM_ACCESS_MODE); points.setRow(0, v1); points.setRow(1, v2); @@ -110,7 +122,7 @@ public void testClusterizationOnDatasetWithObviousStructure() throws IOException int centersCnt = centers.size(); SparseDistributedMatrix points = new SparseDistributedMatrix(ptsCnt, 2, StorageConstants.ROW_STORAGE_MODE, - StorageConstants.RANDOM_ACCESS_MODE); + StorageConstants.RANDOM_ACCESS_MODE); List permutation = IntStream.range(0, ptsCnt).boxed().collect(Collectors.toList()); Collections.shuffle(permutation, rnd); @@ -146,7 +158,7 @@ public void testClusterizationOnDatasetWithObviousStructure() throws IOException Vector[] resCenters = mdl.centers(); Arrays.sort(resCenters, comp); - checkIsInEpsilonNeighbourhood(resCenters, massCenters.toArray(new Vector[]{}), 30.0); + checkIsInEpsilonNeighbourhood(resCenters, massCenters.toArray(new Vector[] {}), 30.0); } /** */ @@ -178,7 +190,7 @@ private int findClosestNodeIndex(Vector v) { return signum; return (int)Math.signum(orderedNodes.get(ind1).minus(v1).kNorm(2) - - orderedNodes.get(ind2).minus(v2).kNorm(2)); + orderedNodes.get(ind2).minus(v2).kNorm(2)); } } } \ No newline at end of file diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java index d164be9ce7772..216fd7b9f8599 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplLocalTestSuite.java @@ -59,7 +59,6 @@ import org.apache.ignite.ml.math.impls.vector.VectorNormTest; import org.apache.ignite.ml.math.impls.vector.VectorToMatrixTest; import org.apache.ignite.ml.math.impls.vector.VectorViewTest; -import org.apache.ignite.ml.regressions.OLSMultipleLinearRegressionTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/TracerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/TracerTest.java index 0bc393f682d3a..407f3a051f150 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/TracerTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/TracerTest.java @@ -26,6 +26,7 @@ import org.apache.ignite.ml.math.impls.MathTestConstants; import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix; import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector; +import org.junit.Ignore; import org.junit.Test; import static java.nio.file.Files.createTempFile; @@ -108,6 +109,7 @@ public void testAsciiMatrixTracer() { * */ @Test + @Ignore("Can not run on TeamCity yet, see IGNITE-5725") public void testHtmlVectorTracer() throws IOException { Vector vec1 = makeRandomVector(1000); @@ -125,6 +127,7 @@ public void testHtmlVectorTracer() throws IOException { * */ @Test + @Ignore("Can not run on TeamCity yet, see IGNITE-5725") public void testHtmlMatrixTracer() throws IOException { Matrix mtx1 = makeRandomMatrix(100, 100); diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java index 54975f0d905bf..a7cd6b5ac3e7e 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java @@ -24,6 +24,7 @@ import java.io.ObjectOutputStream; import java.util.Collection; import java.util.HashSet; +import java.util.Map; import java.util.Set; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -108,7 +109,7 @@ public void testGetSet() throws Exception { double v = Math.random(); cacheMatrix.set(i, j, v); - assertEquals("Unexpected value for matrix element["+ i +" " + j + "]", v, cacheMatrix.get(i, j), PRECISION); + assertEquals("Unexpected value for matrix element[" + i + " " + j + "]", v, cacheMatrix.get(i, j), PRECISION); } } } @@ -223,7 +224,7 @@ public void testCopy() { } /** */ - public void testCacheBehaviour(){ + public void testCacheBehaviour() { IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); SparseDistributedMatrix cacheMatrix1 = new SparseDistributedMatrix(rows, cols, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE); @@ -236,22 +237,27 @@ public void testCacheBehaviour(){ assert cacheNames.contains(SparseDistributedMatrixStorage.ML_CACHE_NAME); - IgniteCache, Object> cache = ignite.getOrCreateCache(SparseDistributedMatrixStorage.ML_CACHE_NAME); + IgniteCache, Map> cache = ignite.getOrCreateCache(SparseDistributedMatrixStorage.ML_CACHE_NAME); Set> keySet1 = buildKeySet(cacheMatrix1); Set> keySet2 = buildKeySet(cacheMatrix2); - assert cache.containsKeys(keySet1); - assert cache.containsKeys(keySet2); + assert cache.containsKeys(keySet1) || + keySet1.stream().allMatch(k -> cache.invoke(k, (entry, arguments) -> entry.getKey().equals(k) && entry.getValue().size() == 100)); + assert cache.containsKeys(keySet2) || + keySet2.stream().allMatch(k -> cache.invoke(k, (entry, arguments) -> entry.getKey().equals(k) && entry.getValue().size() == 100)); cacheMatrix2.destroy(); - assert cache.containsKeys(keySet1); - assert !cache.containsKeys(keySet2); + assert cache.containsKeys(keySet1) || + keySet1.stream().allMatch(k -> cache.invoke(k, (entry, arguments) -> entry.getKey().equals(k) && entry.getValue().size() == 100)); + assert !cache.containsKeys(keySet2) && + keySet2.stream().allMatch(k -> cache.invoke(k, (entry, arguments) -> entry.getKey().equals(k) && entry.getValue() == null)); cacheMatrix1.destroy(); - assert !cache.containsKeys(keySet1); + assert !cache.containsKeys(keySet1) && + keySet1.stream().allMatch(k -> cache.invoke(k, (entry, arguments) -> entry.getKey().equals(k) && entry.getValue() == null)); } /** */ @@ -287,7 +293,7 @@ private void initMtx(Matrix m) { } /** Build key set for SparseDistributedMatrix. */ - private Set> buildKeySet(SparseDistributedMatrix m){ + private Set> buildKeySet(SparseDistributedMatrix m) { Set> set = new HashSet<>(); SparseDistributedMatrixStorage storage = (SparseDistributedMatrixStorage)m.getStorage(); diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegressionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegressionTest.java index e9921639ee1db..7f76750a61647 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegressionTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/OLSMultipleLinearRegressionTest.java @@ -812,7 +812,7 @@ public void testNoDataNPESSTO() { /** */ @Test(expected = MathIllegalArgumentException.class) - public void testMathIllegalArgumentException(){ + public void testMathIllegalArgumentException() { OLSMultipleLinearRegression mdl = new OLSMultipleLinearRegression(); mdl.validateSampleData(new DenseLocalOnHeapMatrix(1, 2), new DenseLocalOnHeapVector(1)); } From b3e6298a099f684a293a61b97e7623f5400cc5d0 Mon Sep 17 00:00:00 2001 From: Eduard Shangareev Date: Mon, 10 Jul 2017 20:44:29 +0300 Subject: [PATCH 126/155] IGNITE-5662 Primary index name should contain type ID or name - Fixes #2275. Signed-off-by: Alexey Goncharuk --- .../internal/processors/query/QueryUtils.java | 8 +- ...eQueryWithMultipleClassesPerCacheTest.java | 185 ++++++++++++++++++ .../IgnitePdsWithIndexingTestSuite.java | 2 + 3 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java index 022d121169f17..320b25a2f770a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java @@ -421,10 +421,12 @@ public static QueryTypeCandidate typeForQueryEntity(String cacheName, GridCacheC QueryTypeIdKey typeId; QueryTypeIdKey altTypeId = null; + int valTypeId = ctx.cacheObjects().typeId(qryEntity.findValueType()); + if (valCls == null || (binaryEnabled && !keyOrValMustDeserialize)) { processBinaryMeta(ctx, qryEntity, desc); - typeId = new QueryTypeIdKey(cacheName, ctx.cacheObjects().typeId(qryEntity.findValueType())); + typeId = new QueryTypeIdKey(cacheName, valTypeId); if (valCls != null) altTypeId = new QueryTypeIdKey(cacheName, valCls); @@ -471,9 +473,11 @@ public static QueryTypeCandidate typeForQueryEntity(String cacheName, GridCacheC } typeId = new QueryTypeIdKey(cacheName, valCls); - altTypeId = new QueryTypeIdKey(cacheName, ctx.cacheObjects().typeId(qryEntity.findValueType())); + altTypeId = new QueryTypeIdKey(cacheName, valTypeId); } + desc.typeId(valTypeId); + return new QueryTypeCandidate(typeId, altTypeId, desc); } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.java new file mode 100644 index 0000000000000..432f41f583973 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.database; + +import java.io.Serializable; +import java.util.List; +import java.util.UUID; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheRebalanceMode; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.PersistentStoreConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK; + +/** + * + */ +public class IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest extends GridCommonAbstractTest { + /** */ + private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + /** Cache name. */ + private static final String CACHE_NAME = "test_cache"; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder); + + cfg.setCacheConfiguration(cacheCfg(CACHE_NAME)); + + PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration(); + + pCfg.setCheckpointingFrequency(1000); + + cfg.setPersistentStoreConfiguration(pCfg); + + return cfg; + } + + /** + * @param name Cache name. + * @return Cache configuration. + */ + private CacheConfiguration cacheCfg(String name) { + CacheConfiguration cfg = new CacheConfiguration<>(); + + cfg.setName(name); + + cfg.setRebalanceMode(CacheRebalanceMode.NONE); + + cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC); + + cfg.setIndexedTypes( + Integer.class, Country.class, + Integer.class, Person.class + ); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + System.setProperty(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK, "true"); + + stopAllGrids(); + + deleteWorkFiles(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + deleteWorkFiles(); + + System.clearProperty(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK); + } + + /** + * @throws Exception If failed. + */ + public void testSimple() throws Exception { + IgniteEx ig0 = startGrid(0); + + ig0.active(true); + + ig0.cache(CACHE_NAME).put(0, new Person(0)); + + ig0.cache(CACHE_NAME).put(1, new Country()); + + List> all = ig0.cache(CACHE_NAME) + .query(new SqlFieldsQuery("select depId FROM \"" + CACHE_NAME + "\".Person")).getAll(); + + assert all.size() == 1; + + } + + /** + * @throws IgniteCheckedException On error. + */ + private void deleteWorkFiles() throws IgniteCheckedException { + deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); + } + + /** + * + */ + public static class Person implements Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + @QuerySqlField + protected int depId; + + /** */ + @QuerySqlField + protected String name; + + /** */ + @QuerySqlField + protected UUID id = UUID.randomUUID(); + + + /** */ + @SuppressWarnings("unused") + private Person() { + // No-op. + } + + /** + * @param depId Department ID. + */ + private Person(int depId) { + this.depId = depId; + + name = "Name-" + id + " " + depId; + } + } + + + /** + * + */ + public static class Country { + /** */ + @QuerySqlField(index = true) + private int id; + + /** */ + @QuerySqlField + private String name; + + /** */ + @QuerySqlField + private UUID uuid = UUID.randomUUID(); + } +} \ No newline at end of file diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java index cb3cd1b3e5e24..15ea594c1d3c5 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest; import org.apache.ignite.internal.processors.database.IgniteDbMultiNodeWithIndexingPutGetTest; import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest; +import org.apache.ignite.internal.processors.database.IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest; import org.apache.ignite.internal.processors.database.IgnitePersistentStoreSchemaLoadTest; /** @@ -40,6 +41,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.class); suite.addTestSuite(IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.class); suite.addTestSuite(IgnitePersistentStoreSchemaLoadTest.class); + suite.addTestSuite(IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.class); return suite; } From 2a2c803039965e52ac5fc33be6cbccc5a27abd84 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Tue, 11 Jul 2017 09:50:04 +0700 Subject: [PATCH 127/155] Web Console: Fixed logic on cluster lost. --- modules/web-console/frontend/app/app.js | 2 + .../cluster-select.controller.js | 72 +++++++++---------- .../cluster-select/cluster-select.pug | 10 +-- .../frontend/app/filters/id8.filter.js | 20 ++++++ .../app/modules/agent/AgentManager.service.js | 4 +- .../views/templates/agent-download.tpl.pug | 4 +- 6 files changed, 68 insertions(+), 44 deletions(-) create mode 100644 modules/web-console/frontend/app/filters/id8.filter.js diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js index 7b8196eaca802..c707810424769 100644 --- a/modules/web-console/frontend/app/app.js +++ b/modules/web-console/frontend/app/app.js @@ -104,6 +104,7 @@ import domainsValidation from './filters/domainsValidation.filter'; import duration from './filters/duration.filter'; import hasPojo from './filters/hasPojo.filter'; import uiGridSubcategories from './filters/uiGridSubcategories.filter'; +import id8 from './filters/id8.filter'; // Controllers import profile from 'Controllers/profile-controller'; @@ -254,6 +255,7 @@ angular .filter('duration', duration) .filter('hasPojo', hasPojo) .filter('uiGridSubcategories', uiGridSubcategories) +.filter('id8', id8) .config(['$translateProvider', '$stateProvider', '$locationProvider', '$urlRouterProvider', ($translateProvider, $stateProvider, $locationProvider, $urlRouterProvider) => { $translateProvider.translations('en', i18n); $translateProvider.preferredLanguage('en'); diff --git a/modules/web-console/frontend/app/components/cluster-select/cluster-select.controller.js b/modules/web-console/frontend/app/components/cluster-select/cluster-select.controller.js index 6d30e941f7786..a2d8e1e664edd 100644 --- a/modules/web-console/frontend/app/components/cluster-select/cluster-select.controller.js +++ b/modules/web-console/frontend/app/components/cluster-select/cluster-select.controller.js @@ -15,52 +15,50 @@ * limitations under the License. */ -export default ['$scope', 'AgentManager', function($scope, agentMgr) { - const ctrl = this; +export default class { + static $inject = ['AgentManager']; - ctrl.counter = 1; + constructor(agentMgr) { + const ctrl = this; - ctrl.cluster = null; - ctrl.clusters = []; + ctrl.counter = 1; - agentMgr.connectionSbj.subscribe({ - next: ({cluster, clusters}) => { - if (_.isEmpty(clusters)) - return ctrl.clusters.length = 0; + ctrl.cluster = null; + ctrl.clusters = []; - const removed = _.differenceBy(ctrl.clusters, clusters, 'id'); + agentMgr.connectionSbj.subscribe({ + next: ({cluster, clusters}) => { + if (_.isEmpty(clusters)) + return ctrl.clusters.length = 0; - if (_.nonEmpty(removed)) - _.pullAll(ctrl.clusters, removed); + const removed = _.differenceBy(ctrl.clusters, clusters, 'id'); - const added = _.differenceBy(clusters, ctrl.clusters, 'id'); + if (_.nonEmpty(removed)) + _.pullAll(ctrl.clusters, removed); - _.forEach(added, (cluster) => { - ctrl.clusters.push({ - id: cluster.id, - name: `Cluster ${cluster.id.substring(0, 8).toUpperCase()}`, - connected: true, - click: () => { - if (cluster.id === _.get(ctrl, 'cluster.id')) - return; + const added = _.differenceBy(clusters, ctrl.clusters, 'id'); - if (_.get(ctrl, 'cluster.connected')) { - agentMgr.saveToStorage(cluster); + _.forEach(added, (cluster) => { + ctrl.clusters.push({ + id: cluster.id, + connected: true, + click: () => { + if (cluster.id === _.get(ctrl, 'cluster.id')) + return; - window.open(window.location.href, '_blank'); + if (_.get(ctrl, 'cluster.connected')) { + agentMgr.saveToStorage(cluster); + + window.open(window.location.href, '_blank'); + } + else + ctrl.cluster = _.find(ctrl.clusters, {id: cluster.id}); } - else - ctrl.cluster = _.find(ctrl.clusters, {id: cluster.id}); - } + }); }); - }); - - const item = _.find(ctrl.clusters, {id: cluster.id}); - if (_.isNil(ctrl.cluster)) - ctrl.cluster = item; - else - ctrl.cluster.connected = !!item; - } - }); -}]; + ctrl.cluster = cluster; + } + }); + } +} diff --git a/modules/web-console/frontend/app/components/cluster-select/cluster-select.pug b/modules/web-console/frontend/app/components/cluster-select/cluster-select.pug index bb8141500e84b..eb46e2625bdb4 100644 --- a/modules/web-console/frontend/app/components/cluster-select/cluster-select.pug +++ b/modules/web-console/frontend/app/components/cluster-select/cluster-select.pug @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. +-var clusterName = 'Cluster {{ ctrl.cluster.id | id8 }}' + ul.nav li.disabled(ng-if='ctrl.clusters.length === 0') a(ng-if='!ctrl.cluster') @@ -21,24 +23,24 @@ ul.nav label.padding-left-dflt(bs-tooltip='' data-placement='bottom' data-title='Check that Web Agent(s) started and connected to cluster(s)') No clusters available a(ng-if='ctrl.cluster') i.icon-danger - label.padding-left-dflt(bs-tooltip='' data-placement='bottom' data-title='Connection to cluster was lost') {{ctrl.cluster.name}} + label.padding-left-dflt(bs-tooltip='' data-placement='bottom' data-title='Connection to cluster was lost') #{clusterName} li(ng-if='ctrl.clusters.length === 1 && ctrl.cluster.connected') a i.icon-cluster - label.padding-left-dflt {{ctrl.cluster.name}} + label.padding-left-dflt #{clusterName} li(ng-if='ctrl.clusters.length > 1 || ctrl.clusters.length === 1 && !ctrl.cluster.connected') a.dropdown-toggle(bs-dropdown='' data-placement='bottom-left' data-trigger='hover focus' data-container='self' ng-click='$event.stopPropagation()' aria-haspopup='true' aria-expanded='expanded') i(ng-class='{"icon-cluster": ctrl.cluster.connected, "icon-danger": !ctrl.cluster.connected}') - label.padding-left-dflt {{ctrl.cluster.name}} + label.padding-left-dflt #{clusterName} span.caret ul.dropdown-menu(role='menu') li(ng-repeat='item in ctrl.clusters' ng-class='{active: ctrl.cluster === item}') div(ng-click='item.click()') i.icon-cluster.pull-left(style='margin: 0; padding-left: 10px;') - div: a {{item.name}} + div: a Cluster {{ item.id | id8 }} i.icon-help(bs-tooltip='' data-placement='bottom' data-html=true data-title='Multi-Cluster Support
\ diff --git a/modules/web-console/frontend/app/filters/id8.filter.js b/modules/web-console/frontend/app/filters/id8.filter.js new file mode 100644 index 0000000000000..ab4e92011cfa7 --- /dev/null +++ b/modules/web-console/frontend/app/filters/id8.filter.js @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default [() => { + return _.id8; +}]; diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js index 387356732dd4b..bdd1fda4639cf 100644 --- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js +++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js @@ -55,6 +55,8 @@ class ConnectionState { if (_.nonEmpty(this.clusters) && !this.cluster.connected) { this.cluster = _.head(this.clusters); + this.cluster.connected = true; + this.state = State.CONNECTED; } } @@ -368,7 +370,7 @@ export default class IgniteAgentManager { * @private */ _rest(event, ...args) { - return this._emit(event, _.get(this, 'cluster.id'), ...args); + return this._emit(event, _.get(this.connectionSbj.getValue(), 'cluster.id'), ...args); } /** diff --git a/modules/web-console/frontend/views/templates/agent-download.tpl.pug b/modules/web-console/frontend/views/templates/agent-download.tpl.pug index 0aec7e57f1bd3..782958101651f 100644 --- a/modules/web-console/frontend/views/templates/agent-download.tpl.pug +++ b/modules/web-console/frontend/views/templates/agent-download.tpl.pug @@ -45,10 +45,10 @@ .modal-header.header h4.modal-title i.fa.fa-download - span Connection to Ignite Node is not established + span Connection to cluster was lost or can't be established .modal-body.agent-download - p Connection to Ignite Web Agent is established, but agent failed to connect to Ignite Node + p Connection to Ignite Web Agent is established, but agent failed to connect to cluster p Please check the following: p ul From 4f3b69c7018913618e7bbc724ab83ac6c274bc1f Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Tue, 11 Jul 2017 11:27:57 +0300 Subject: [PATCH 128/155] IGNITE-5684 - Fixed update sequence handling in GridDhtPartitionFullMap --- .../dht/GridClientPartitionTopology.java | 71 +++++++++++++------ .../dht/GridDhtPartitionTopologyImpl.java | 27 ++++--- .../preloader/GridDhtPartitionFullMap.java | 24 +------ .../CacheLateAffinityAssignmentTest.java | 2 +- 4 files changed, 67 insertions(+), 57 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java index e7519610f360c..f4ed517975155 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java @@ -572,56 +572,66 @@ public long lastUpdateSequence() { } } + /** + * Checks should current partition map overwritten by new partition map + * Method returns true if topology version or update sequence of new map are greater than of current map. + * + * @param currentMap Current partition map. + * @param newMap New partition map. + * @return True if current partition map should be overwritten by new partition map, false in other case. + */ + private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridDhtPartitionMap newMap) { + return newMap != null && + (newMap.topologyVersion().compareTo(currentMap.topologyVersion()) > 0 || + newMap.topologyVersion().compareTo(currentMap.topologyVersion()) == 0 && newMap.updateSequence() > currentMap.updateSequence()); + } + /** {@inheritDoc} */ @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"}) @Override public boolean update( - @Nullable AffinityTopologyVersion exchVer, + @Nullable AffinityTopologyVersion exchangeVer, GridDhtPartitionFullMap partMap, Map> cntrMap, Set partsToReload ) { if (log.isDebugEnabled()) - log.debug("Updating full partition map [exchVer=" + exchVer + ", parts=" + fullMapString() + ']'); + log.debug("Updating full partition map [exchVer=" + exchangeVer + ", parts=" + fullMapString() + ']'); lock.writeLock().lock(); try { - if (exchVer != null && lastExchangeVer != null && lastExchangeVer.compareTo(exchVer) >= 0) { + if (exchangeVer != null && lastExchangeVer != null && lastExchangeVer.compareTo(exchangeVer) >= 0) { if (log.isDebugEnabled()) log.debug("Stale exchange id for full partition map update (will ignore) [lastExchId=" + - lastExchangeVer + ", exchVer=" + exchVer + ']'); - - return false; - } - - if (node2part != null && node2part.compareTo(partMap) >= 0) { - if (log.isDebugEnabled()) - log.debug("Stale partition map for full partition map update (will ignore) [lastExchId=" + - lastExchangeVer + ", exchVer=" + exchVer + ", curMap=" + node2part + ", newMap=" + partMap + ']'); + lastExchangeVer + ", exchVer=" + exchangeVer + ']'); return false; } - updateSeq.incrementAndGet(); - - if (exchVer != null) - lastExchangeVer = exchVer; + boolean fullMapUpdated = (node2part == null); if (node2part != null) { for (GridDhtPartitionMap part : node2part.values()) { GridDhtPartitionMap newPart = partMap.get(part.nodeId()); - // If for some nodes current partition has a newer map, - // then we keep the newer value. - if (newPart != null && newPart.updateSequence() < part.updateSequence()) { - if (log.isDebugEnabled()) - log.debug("Overriding partition map in full update map [exchVer=" + exchVer + ", curPart=" + - mapString(part) + ", newPart=" + mapString(newPart) + ']'); + if (shouldOverridePartitionMap(part, newPart)) { + fullMapUpdated = true; + if (log.isDebugEnabled()) + log.debug("Overriding partition map in full update map [exchId=" + exchangeVer + ", curPart=" + + mapString(part) + ", newPart=" + mapString(newPart) + ']'); + } + else { + // If for some nodes current partition has a newer map, + // then we keep the newer value. partMap.put(part.nodeId(), part); } } + for (GridDhtPartitionMap part : partMap.values()) + fullMapUpdated |= !node2part.containsKey(part); + + // Remove entry if node left. for (Iterator it = partMap.keySet().iterator(); it.hasNext(); ) { UUID nodeId = it.next(); @@ -635,9 +645,24 @@ public long lastUpdateSequence() { } } + if (!fullMapUpdated) { + if (log.isDebugEnabled()) + log.debug("No updates for full partition map (will ignore) [lastExch=" + + lastExchangeVer + ", exch=" + exchangeVer + ", curMap=" + node2part + ", newMap=" + partMap + ']'); + + return false; + } + + if (exchangeVer != null) + lastExchangeVer = exchangeVer; + + node2part = partMap; + + updateSeq.incrementAndGet(); + part2node.clear(); - for (Map.Entry e : partMap.entrySet()) { + for (Map.Entry e : node2part.entrySet()) { for (Map.Entry e0 : e.getValue().entrySet()) { if (e0.getValue() != MOVING && e0.getValue() != OWNING) continue; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java index fe0a0c669edb9..601da1b4ea2f4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java @@ -1134,22 +1134,15 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD return false; } - if (node2part != null && node2part.compareTo(partMap) > 0) { - if (log.isDebugEnabled()) - log.debug("Stale partition map for full partition map update (will ignore) [lastExch=" + - lastExchangeVer + ", exch=" + exchangeVer + ", curMap=" + node2part + ", newMap=" + partMap + ']'); - - return false; - } - - if (exchangeVer != null) - lastExchangeVer = exchangeVer; + boolean fullMapUpdated = (node2part == null); if (node2part != null) { for (GridDhtPartitionMap part : node2part.values()) { GridDhtPartitionMap newPart = partMap.get(part.nodeId()); if (shouldOverridePartitionMap(part, newPart)) { + fullMapUpdated = true; + if (log.isDebugEnabled()) log.debug("Overriding partition map in full update map [exchId=" + exchangeVer + ", curPart=" + mapString(part) + ", newPart=" + mapString(newPart) + ']'); @@ -1161,6 +1154,9 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD } } + for (GridDhtPartitionMap part : partMap.values()) + fullMapUpdated |= !node2part.containsKey(part); + // Remove entry if node left. for (Iterator it = partMap.keySet().iterator(); it.hasNext(); ) { UUID nodeId = it.next(); @@ -1175,6 +1171,17 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD } } + if (!fullMapUpdated) { + if (log.isDebugEnabled()) + log.debug("No updates for full partition map (will ignore) [lastExch=" + + lastExchangeVer + ", exch=" + exchangeVer + ", curMap=" + node2part + ", newMap=" + partMap + ']'); + + return false; + } + + if (exchangeVer != null) + lastExchangeVer = exchangeVer; + node2part = partMap; AffinityTopologyVersion affVer = grp.affinity().lastVersion(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java index 27e6777af5ae7..73b7714b8f8e3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java @@ -31,8 +31,7 @@ /** * Full partition map. */ -public class GridDhtPartitionFullMap extends HashMap - implements Comparable, Externalizable { +public class GridDhtPartitionFullMap extends HashMap implements Externalizable { /** */ private static final long serialVersionUID = 0L; @@ -177,27 +176,6 @@ public long updateSequence(long updateSeq) { return old; } - /** {@inheritDoc} */ - @Override public int compareTo(GridDhtPartitionFullMap o) { - assert nodeId == null || (nodeOrder != o.nodeOrder && !nodeId.equals(o.nodeId)) || - (nodeOrder == o.nodeOrder && nodeId.equals(o.nodeId)): "Inconsistent node order and ID [id1=" + nodeId + - ", order1=" + nodeOrder + ", id2=" + o.nodeId + ", order2=" + o.nodeOrder + ']'; - - if (nodeId == null && o.nodeId != null) - return -1; - else if (nodeId != null && o.nodeId == null) - return 1; - else if (nodeId == null && o.nodeId == null) - return 0; - - int res = Long.compare(nodeOrder, o.nodeOrder); - - if (res == 0) - res = Long.compare(updateSeq, o.updateSeq); - - return res; - } - /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { U.writeUuid(out, nodeId); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java index 6174209d74792..23043d10f773f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java @@ -181,7 +181,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest { MemoryConfiguration cfg1 = new MemoryConfiguration(); - cfg1.setDefaultMemoryPolicySize(50 * 1024 * 1024L); + cfg1.setDefaultMemoryPolicySize(100 * 1024 * 1024L); cfg.setMemoryConfiguration(cfg1); From a668a224a8c69afd618667fa19309f1b7a0c8ca3 Mon Sep 17 00:00:00 2001 From: Igor Seliverstov Date: Tue, 11 Jul 2017 10:40:56 +0200 Subject: [PATCH 129/155] IGNITE-5686 Endless partition eviction during node shutdown --- .../src/main/scala/org/apache/ignite/spark/IgniteRDD.scala | 6 +++--- .../scala/org/apache/ignite/spark/impl/IgniteSqlRDD.scala | 6 +++++- .../apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java | 5 ----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala index 81509d06ed7e9..78e2223e7f059 100644 --- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala +++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala @@ -60,11 +60,11 @@ class IgniteRDD[K, V] ( val qry: ScanQuery[K, V] = new ScanQuery[K, V](part.index) - val partNodes = ic.ignite().affinity(cache.getName).mapPartitionToPrimaryAndBackups(part.index) + val cur = cache.query(qry) - val it: java.util.Iterator[Cache.Entry[K, V]] = cache.query(qry).iterator() + TaskContext.get().addTaskCompletionListener((_) ⇒ cur.close()) - new IgniteQueryIterator[Cache.Entry[K, V], (K, V)](it, entry ⇒ { + new IgniteQueryIterator[Cache.Entry[K, V], (K, V)](cur.iterator(), entry ⇒ { (entry.getKey, entry.getValue) }) } diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSqlRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSqlRDD.scala index f0745727a54d5..f386f26676b7f 100644 --- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSqlRDD.scala +++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSqlRDD.scala @@ -33,7 +33,11 @@ class IgniteSqlRDD[R: ClassTag, T, K, V]( keepBinary: Boolean ) extends IgniteAbstractRDD[R, K, V](ic, cacheName, cacheCfg, keepBinary) { override def compute(split: Partition, context: TaskContext): Iterator[R] = { - new IgniteQueryIterator[T, R](ensureCache().query(qry).iterator(), conv) + val cur = ensureCache().query(qry) + + TaskContext.get().addTaskCompletionListener((_) ⇒ cur.close()) + + new IgniteQueryIterator[T, R](cur.iterator(), conv) } override protected def getPartitions: Array[Partition] = { diff --git a/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java b/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java index 5477d4384246d..49bb1ac63ed61 100644 --- a/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java +++ b/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java @@ -99,11 +99,6 @@ public JavaEmbeddedIgniteRDDSelfTest() { super(false); } - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-5690"); - } - /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); From 704032d942bb0412607b7cfcb2c4378429e85649 Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Tue, 11 Jul 2017 11:56:22 +0300 Subject: [PATCH 130/155] GG-12449 - Fix eviction and checkpoint write race - Fixes #2277. Signed-off-by: Alexey Goncharuk --- .../persistence/pagemem/PageMemoryImpl.java | 86 +++++++------------ .../cache/persistence/wal/FileInput.java | 3 + 2 files changed, 35 insertions(+), 54 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index ab6619dd064bd..e4428a2ab3b41 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -646,7 +646,6 @@ private long refreshOutdatedPage(Segment seg, int cacheId, long pageId, boolean PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR); PageHeader.dirty(absPtr, false); - PageHeader.tempDirty(absPtr, false); // We pinned the page when allocated the temp buffer, release it now. PageHeader.releasePage(absPtr); @@ -868,13 +867,9 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck finally { seg.writeLock().unlock(); } - - } - else { - copyPageForCheckpoint(absPtr, fullId, tmpBuf, tracker); - - return tag; } + else + return copyPageForCheckpoint(absPtr, fullId, tmpBuf, tracker) ? tag : null; } /** @@ -882,51 +877,51 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck * @param fullId Full id. * @param tmpBuf Tmp buffer. */ - private void copyPageForCheckpoint(long absPtr, FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) { + private boolean copyPageForCheckpoint(long absPtr, FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) { assert absPtr != 0; - long tmpRelPtr; - rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS); try { - tmpRelPtr = PageHeader.tempBufferPointer(absPtr); + long tmpRelPtr = PageHeader.tempBufferPointer(absPtr); - clearCheckpoint(fullId); + if (!clearCheckpoint(fullId)){ + assert tmpRelPtr == INVALID_REL_PTR; - if (tmpRelPtr != INVALID_REL_PTR) + return false; + } + + if (tmpRelPtr != INVALID_REL_PTR){ PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR); + + long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr); + + copyInBuffer(tmpAbsPtr, tmpBuf); + + GridUnsafe.setMemory(tmpAbsPtr + PAGE_OVERHEAD, pageSize(), (byte)0); + + if (tracker != null) + tracker.onCowPageWritten(); + + checkpointPool.releaseFreePage(tmpRelPtr); + + // We pinned the page when allocated the temp buffer, release it now. + PageHeader.releasePage(absPtr); + } else { copyInBuffer(absPtr, tmpBuf); PageHeader.dirty(absPtr, false); + // We pinned the page when resolve abs pointer. PageHeader.releasePage(absPtr); - - return; } + + return true; } finally { rwLock.writeUnlock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS); } - - assert tmpRelPtr != 0; - - long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr); - - copyInBuffer(tmpAbsPtr, tmpBuf); - - GridUnsafe.setMemory(tmpAbsPtr + PAGE_OVERHEAD, pageSize(), (byte)0); - - PageHeader.dirty(tmpAbsPtr, false); - - if (tracker != null) - tracker.onCowPageWritten(); - - checkpointPool.releaseFreePage(tmpRelPtr); - - // We pinned the page when allocated the temp buffer, release it now. - PageHeader.releasePage(absPtr); } /** @@ -1146,8 +1141,6 @@ private long postWriteLockPage(long absPtr, FullPageId fullId) { long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr); - assert !PageHeader.tempDirty(tmpAbsPtr); - GridUnsafe.copyMemory( null, absPtr + PAGE_OVERHEAD, @@ -1254,15 +1247,16 @@ boolean isInCheckpoint(FullPageId pageId) { /** * @param fullPageId Page ID to clear. + * @return {@code True} if remove successfully. */ - void clearCheckpoint(FullPageId fullPageId) { + boolean clearCheckpoint(FullPageId fullPageId) { Segment seg = segment(fullPageId.groupId(), fullPageId.pageId()); Collection pages0 = seg.segCheckpointPages; assert pages0 != null; - pages0.remove(fullPageId); + return pages0.remove(fullPageId); } /** @@ -1655,6 +1649,7 @@ private boolean prepareEvict(FullPageId fullPageId, long absPtr) throws IgniteCh if (isDirty(absPtr)) { // Can evict a dirty page only if should be written by a checkpoint. + // These pages does not have tmp buffer. if (cpPages != null && cpPages.contains(fullPageId)) { assert storeMgr != null; @@ -2026,23 +2021,6 @@ private static boolean dirty(long absPtr, boolean dirty) { return flag(absPtr, DIRTY_FLAG, dirty); } - /** - * @param absPtr Absolute pointer. - * @return Dirty flag. - */ - private static boolean tempDirty(long absPtr) { - return flag(absPtr, TMP_DIRTY_FLAG); - } - - /** - * @param absPtr Absolute pointer. - * @param tmpDirty Temp dirty flag. - * @return Previous value of temp dirty flag. - */ - private static boolean tempDirty(long absPtr, boolean tmpDirty) { - return flag(absPtr, TMP_DIRTY_FLAG, tmpDirty); - } - /** * @param absPtr Absolute pointer. * @param flag Flag mask. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java index 6443a7cab4dca..74edbfa3f9944 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java @@ -102,6 +102,9 @@ public void seek(long pos) throws IOException { return; if (buf.capacity() < requested) { + if (expBuf == null) + throw new IOException("Requested size is greater than buffer: " + requested); + buf = expBuf.expand(requested); assert available == buf.remaining(); From e786beb64a72906c4e004ce0bed1fa2a9c028a9d Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 11 Jul 2017 12:28:43 +0300 Subject: [PATCH 131/155] IGNITE-5446 Always use late affinity assignment mode - update .NET part --- .../platform/utils/PlatformConfigurationUtils.java | 4 ---- .../Cache/Query/CacheQueriesTest.cs | 6 ++++++ .../Apache.Ignite.Core.Tests/Config/cache-query.xml | 1 - .../IgniteConfigurationSerializerTest.cs | 2 -- .../IgniteConfigurationTest.cs | 3 ++- .../dotnet/Apache.Ignite.Core/IgniteConfiguration.cs | 11 ++++------- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java index d513071b09d8c..03b30dbf2ab8d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java @@ -560,8 +560,6 @@ public static void readIgniteConfiguration(BinaryRawReaderEx in, IgniteConfigura cfg.setLocalHost(localHost); if (in.readBoolean()) cfg.setDaemon(in.readBoolean()); - if (in.readBoolean()) - cfg.setLateAffinityAssignment(in.readBoolean()); if (in.readBoolean()) cfg.setFailureDetectionTimeout(in.readLong()); if (in.readBoolean()) @@ -1011,8 +1009,6 @@ public static void writeIgniteConfiguration(BinaryRawWriter w, IgniteConfigurati w.writeBoolean(true); w.writeBoolean(cfg.isDaemon()); w.writeBoolean(true); - w.writeBoolean(cfg.isLateAffinityAssignment()); - w.writeBoolean(true); w.writeLong(cfg.getFailureDetectionTimeout()); w.writeBoolean(true); w.writeLong(cfg.getClientFailureDetectionTimeout()); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs index ee2806fcce1b3..4df12a44d7686 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs @@ -907,6 +907,12 @@ public void TestSqlFieldsQueryTimeout() exp.Add(val); } + if (loc) + { + Assert.AreEqual(exp.Count, + cache.GetLocalEntries(CachePeekMode.Primary).Count(x => expectedEntryFilter(x.Key))); + } + return exp; } } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/cache-query.xml b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/cache-query.xml index 4f92c7290bb7e..2b41efa672445 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/cache-query.xml +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/cache-query.xml @@ -28,7 +28,6 @@ - diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs index 7e0d94104541e..125902fcc376b 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs @@ -151,7 +151,6 @@ public void TestPredefinedXml() Assert.AreEqual("c:", cfg.WorkDirectory); Assert.AreEqual("127.1.1.1", cfg.Localhost); Assert.IsTrue(cfg.IsDaemon); - Assert.IsFalse(cfg.IsLateAffinityAssignment); Assert.AreEqual(1024, cfg.JvmMaxMemoryMb); Assert.AreEqual(TimeSpan.FromSeconds(10), cfg.MetricsLogFrequency); Assert.AreEqual(TimeSpan.FromMinutes(1), ((TcpDiscoverySpi)cfg.DiscoverySpi).JoinTimeout); @@ -836,7 +835,6 @@ private static IgniteConfiguration GetTestConfig() SocketSendBufferSize = 2045, UnacknowledgedMessagesBufferSize = 3450 }, - IsLateAffinityAssignment = false, SpringConfigUrl = "test", Logger = new IgniteNLogLogger(), FailureDetectionTimeout = TimeSpan.FromMinutes(2), diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs index 4902118eec2df..81fd226ef6ee2 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs @@ -15,6 +15,7 @@ * limitations under the License. */ +#pragma warning disable 618 // Ignore obsolete, we still need to test them. namespace Apache.Ignite.Core.Tests { using System; @@ -138,7 +139,7 @@ public void TestAllConfigurationProperties() Assert.IsTrue(File.Exists(resCfg.JvmDllPath)); Assert.AreEqual(cfg.Localhost, resCfg.Localhost); Assert.AreEqual(cfg.IsDaemon, resCfg.IsDaemon); - Assert.AreEqual(cfg.IsLateAffinityAssignment, resCfg.IsLateAffinityAssignment); + Assert.AreEqual(IgniteConfiguration.DefaultIsLateAffinityAssignment, resCfg.IsLateAffinityAssignment); Assert.AreEqual(cfg.UserAttributes, resCfg.UserAttributes); var atm = cfg.AtomicConfiguration; diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs index 4419e2ecf2205..86155a6d5fd55 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs @@ -145,9 +145,6 @@ public class IgniteConfiguration /** */ private bool? _isDaemon; - /** */ - private bool? _isLateAffinityAssignment; - /** */ private bool? _clientMode; @@ -274,7 +271,6 @@ internal void Write(BinaryWriter writer) writer.WriteString(WorkDirectory); writer.WriteString(Localhost); writer.WriteBooleanNullable(_isDaemon); - writer.WriteBooleanNullable(_isLateAffinityAssignment); writer.WriteTimeSpanAsLongNullable(_failureDetectionTimeout); writer.WriteTimeSpanAsLongNullable(_clientFailureDetectionTimeout); writer.WriteTimeSpanAsLongNullable(_longQueryWarningTimeout); @@ -525,7 +521,6 @@ private void ReadCore(BinaryReader r) WorkDirectory = r.ReadString(); Localhost = r.ReadString(); _isDaemon = r.ReadBooleanNullable(); - _isLateAffinityAssignment = r.ReadBooleanNullable(); _failureDetectionTimeout = r.ReadTimeSpanNullable(); _clientFailureDetectionTimeout = r.ReadTimeSpanNullable(); _longQueryWarningTimeout = r.ReadTimeSpanNullable(); @@ -976,10 +971,12 @@ public bool IsDaemon /// If not provided, default value is . /// [DefaultValue(DefaultIsLateAffinityAssignment)] + [Obsolete("No longer supported, always true.")] public bool IsLateAffinityAssignment { - get { return _isLateAffinityAssignment ?? DefaultIsLateAffinityAssignment; } - set { _isLateAffinityAssignment = value; } + get { return DefaultIsLateAffinityAssignment; } + // ReSharper disable once ValueParameterNotUsed + set { /* No-op. */ } } /// From c5966dd33e12d9e1855c165854b965a8633b1869 Mon Sep 17 00:00:00 2001 From: Andrey Novikov Date: Tue, 11 Jul 2017 17:02:56 +0700 Subject: [PATCH 132/155] IGNITE-5726 Web Console: Fixed duplication of dependencies in maven project. --- .../modules/configuration/generator/Maven.service.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js index 700da4a6d3ac8..81d7d10541600 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/Maven.service.js @@ -39,7 +39,7 @@ export default class IgniteMavenGenerator { } addDependency(deps, groupId, artifactId, version, jar) { - deps.add({groupId, artifactId, version, jar}); + deps.push({groupId, artifactId, version, jar}); } pickDependency(deps, key, dfltVer, igniteVer) { @@ -78,7 +78,7 @@ export default class IgniteMavenGenerator { dependenciesSection(sb, deps) { sb.startBlock(''); - deps.forEach((dep) => { + _.forEach(deps, (dep) => { sb.startBlock(''); this.addProperty(sb, 'groupId', dep.groupId); @@ -154,8 +154,8 @@ export default class IgniteMavenGenerator { collectDependencies(cluster, targetVer) { const igniteVer = targetVer.ignite; - const deps = new Set(); - const storeDeps = new Set(); + const deps = []; + const storeDeps = []; this.addDependency(deps, 'org.apache.ignite', 'ignite-core', igniteVer); @@ -204,7 +204,7 @@ export default class IgniteMavenGenerator { if (cluster.logger && cluster.logger.kind) this.pickDependency(deps, cluster.logger.kind, igniteVer); - return new Set([...deps, ...storeDeps]); + return _.uniqWith(deps.concat(...storeDeps), _.isEqual); } /** From a9ae95342dd566b210fe835221214ecc67e6ed16 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 11 Jul 2017 14:13:14 +0300 Subject: [PATCH 133/155] IGNITE-5446 Always use late affinity assignment mode - fix .NET tests --- .../Cache/Query/CacheQueriesTest.cs | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs index 4df12a44d7686..62c82300f9c2b 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs @@ -379,12 +379,13 @@ private void CheckEnumeratorQuery(SqlQuery qry) var exp = PopulateCache(cache, loc, cnt, x => x < 50); // 2. Validate results. - var qry = new SqlFieldsQuery("SELECT name, age FROM QueryPerson WHERE age < 50", loc) + var qry = new SqlFieldsQuery("SELECT name, age FROM QueryPerson WHERE age < 50") { EnableDistributedJoins = distrJoin, EnforceJoinOrder = enforceJoinOrder, Colocated = !distrJoin, ReplicatedOnly = false, + Local = loc, Timeout = TimeSpan.FromSeconds(2) }; @@ -891,29 +892,18 @@ public void TestSqlFieldsQueryTimeout() { var rand = new Random(); - var exp = new HashSet(); - - var aff = cache.Ignite.GetAffinity(cache.Name); - - var localNode = cache.Ignite.GetCluster().GetLocalNode(); - for (var i = 0; i < cnt; i++) { var val = rand.Next(cnt); cache.Put(val, new QueryPerson(val.ToString(), val)); - - if (expectedEntryFilter(val) && (!loc || aff.IsPrimary(localNode, val))) - exp.Add(val); } - if (loc) - { - Assert.AreEqual(exp.Count, - cache.GetLocalEntries(CachePeekMode.Primary).Count(x => expectedEntryFilter(x.Key))); - } + var entries = loc + ? cache.GetLocalEntries(CachePeekMode.Primary) + : cache; - return exp; + return new HashSet(entries.Select(x => x.Key).Where(expectedEntryFilter)); } } From 1942db3af6538e6d1f0731b52eda94071f533ccb Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Tue, 11 Jul 2017 14:23:56 +0300 Subject: [PATCH 134/155] .NET: Fix code analysis warnings for deprecated IsLateAffinityAssignment --- .../platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs index 86155a6d5fd55..4d04348b71b4a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs @@ -970,6 +970,8 @@ public bool IsDaemon /// /// If not provided, default value is . /// + [SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic")] + [SuppressMessage("Microsoft.Usage", "CA1801:ReviewUnusedParameters", MessageId = "value")] [DefaultValue(DefaultIsLateAffinityAssignment)] [Obsolete("No longer supported, always true.")] public bool IsLateAffinityAssignment From 1b2f5052ed34776d14d7614812b78e0aa66c69b1 Mon Sep 17 00:00:00 2001 From: Ivan Rakov Date: Tue, 11 Jul 2017 14:19:14 +0300 Subject: [PATCH 135/155] Visor fails to connect to secure grid --- .../JettyRestProcessorAbstractSelfTest.java | 2 +- .../cluster/GridClusterStateProcessor.java | 4 ++ .../processors/igfs/IgfsNoopProcessor.java | 11 +++++ .../processors/igfs/IgfsProcessor.java | 12 ++++- .../processors/igfs/IgfsProcessorAdapter.java | 3 +- .../processors/rest/GridRestProcessor.java | 47 ++++++++++++++----- .../processors/task/GridTaskProcessor.java | 18 ++++++- 7 files changed, 80 insertions(+), 17 deletions(-) diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java index 2b3c8db8a6fcb..97321a713bab0 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java @@ -1976,7 +1976,7 @@ private boolean queryCursorFound() { /** * Init cache. */ - private void initCache() { + protected void initCache() { CacheConfiguration orgCacheCfg = new CacheConfiguration<>("organization"); orgCacheCfg.setIndexedTypes(Integer.class, Organization.class); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java index 8cea13f6dd801..6e94669f460b6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java @@ -577,6 +577,10 @@ private void onFinalActivate(final StateChangeRequest req) { ctx.dataStructures().onActivate(ctx); + ctx.igfs().onActivate(ctx); + + ctx.task().onActivate(ctx); + if (log.isInfoEnabled()) log.info("Successfully performed final activation steps [nodeId=" + ctx.localNodeId() + ", client=" + client + ", topVer=" + req.topologyVersion() + "]"); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java index 2dfac90635a97..6816b85f63c94 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java @@ -19,6 +19,7 @@ import java.util.Collection; import java.util.Collections; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteFileSystem; import org.apache.ignite.compute.ComputeJob; import org.apache.ignite.igfs.IgfsPath; @@ -69,4 +70,14 @@ public IgfsNoopProcessor(GridKernalContext ctx) { long start, long length, IgfsRecordResolver recRslv) { return null; } + + /** {@inheritDoc} */ + @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException { + // No-op + } + + /** {@inheritDoc} */ + @Override public void onDeActivate(GridKernalContext kctx) { + // No-op + } } \ No newline at end of file diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java index 244820ffef478..f18c4386e23fd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java @@ -178,7 +178,7 @@ public IgfsProcessor(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void onKernalStart(boolean active) throws IgniteCheckedException { - if (ctx.config().isDaemon()) + if (!active || ctx.config().isDaemon()) return; if (!getBoolean(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK)) { @@ -191,6 +191,16 @@ public IgfsProcessor(GridKernalContext ctx) { mgr.onKernalStart(); } + /** {@inheritDoc} */ + @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException { + onKernalStart(true); + } + + /** {@inheritDoc} */ + @Override public void onDeActivate(GridKernalContext kctx) { + onKernalStop(true); + } + /** {@inheritDoc} */ @Override public void stop(boolean cancel) { // Stop IGFS instances. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java index 8b7f662d34e37..c12b0a5b98e0e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java @@ -25,13 +25,14 @@ import org.apache.ignite.igfs.mapreduce.IgfsRecordResolver; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.GridProcessorAdapter; +import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; import org.apache.ignite.internal.util.ipc.IpcServerEndpoint; import org.jetbrains.annotations.Nullable; /** * Ignite file system processor adapter. */ -public abstract class IgfsProcessorAdapter extends GridProcessorAdapter { +public abstract class IgfsProcessorAdapter extends GridProcessorAdapter implements IgniteChangeGlobalStateSupport { /** * Constructor. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java index f5281844e2b28..fd5583d3edbe4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java @@ -67,6 +67,8 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; import org.apache.ignite.internal.util.worker.GridWorkerFuture; +import org.apache.ignite.internal.visor.compute.VisorGatewayTask; +import org.apache.ignite.internal.visor.misc.VisorChangeGridActiveStateTask; import org.apache.ignite.internal.visor.util.VisorClusterGroupEmptyException; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteInClosure; @@ -241,21 +243,23 @@ private IgniteInternalFuture handleRequest(final GridRestReque SecurityContext secCtx0 = ses.secCtx; - try { - if (secCtx0 == null) - ses.secCtx = secCtx0 = authenticate(req); + if (ctx.state().publicApiActiveState() || !isClusterActivateTaskRequest(req)) { + try { + if (secCtx0 == null) + ses.secCtx = secCtx0 = authenticate(req); - authorize(req, secCtx0); - } - catch (SecurityException e) { - assert secCtx0 != null; + authorize(req, secCtx0); + } + catch (SecurityException e) { + assert secCtx0 != null; - GridRestResponse res = new GridRestResponse(STATUS_SECURITY_CHECK_FAILED, e.getMessage()); + GridRestResponse res = new GridRestResponse(STATUS_SECURITY_CHECK_FAILED, e.getMessage()); - return new GridFinishedFuture<>(res); - } - catch (IgniteCheckedException e) { - return new GridFinishedFuture<>(new GridRestResponse(STATUS_AUTH_FAILED, e.getMessage())); + return new GridFinishedFuture<>(res); + } + catch (IgniteCheckedException e) { + return new GridFinishedFuture<>(new GridRestResponse(STATUS_AUTH_FAILED, e.getMessage())); + } } } @@ -316,6 +320,25 @@ private IgniteInternalFuture handleRequest(final GridRestReque }); } + /** + * We skip authentication for activate cluster request. + * It's necessary workaround to make possible cluster activation through Visor, + * as security checks require working caches. + * + * @param req Request. + */ + private boolean isClusterActivateTaskRequest(GridRestRequest req) { + if (req instanceof GridRestTaskRequest) { + GridRestTaskRequest taskReq = (GridRestTaskRequest)req; + + if (VisorGatewayTask.class.getCanonicalName().equals(taskReq.taskName()) && + taskReq.params().contains(VisorChangeGridActiveStateTask.class.getCanonicalName())) + return true; + } + + return false; + } + /** * @param req Request. * @return Not null session. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java index d0b88d80b5e24..4606b7c551419 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java @@ -61,6 +61,7 @@ import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; import org.apache.ignite.internal.processors.GridProcessorAdapter; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; +import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; import org.apache.ignite.internal.util.GridConcurrentFactory; import org.apache.ignite.internal.util.GridSpinReadWriteLock; import org.apache.ignite.internal.util.lang.GridPeerDeployAware; @@ -91,7 +92,7 @@ /** * This class defines task processor. */ -public class GridTaskProcessor extends GridProcessorAdapter { +public class GridTaskProcessor extends GridProcessorAdapter implements IgniteChangeGlobalStateSupport { /** Wait for 5 seconds to allow discovery to take effect (best effort). */ private static final long DISCO_TIMEOUT = 5000; @@ -154,6 +155,9 @@ public GridTaskProcessor(GridKernalContext ctx) { /** {@inheritDoc} */ @Override public void onKernalStart(boolean active) throws IgniteCheckedException { + if (!active) + return; + tasksMetaCache = ctx.security().enabled() && !ctx.isDaemon() ? ctx.cache().utilityCache() : null; @@ -694,7 +698,7 @@ else if (task != null) { IgniteCheckedException securityEx = null; - if (ctx.security().enabled() && deployEx == null) { + if (ctx.security().enabled() && deployEx == null && !dep.internalTask(task, taskCls)) { try { saveTaskMetadata(taskName); } @@ -1144,6 +1148,16 @@ public void onCancelled(IgniteUuid sesId) { } } + /** {@inheritDoc} */ + @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException { + onKernalStart(true); + } + + /** {@inheritDoc} */ + @Override public void onDeActivate(GridKernalContext kctx) { + onKernalStop(true); + } + /** * @return Number of executed tasks. */ From e9f194a328b1704a7881f1bf1a148a3426cd1b53 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 11 Jul 2017 19:16:06 +0700 Subject: [PATCH 136/155] ignite-2.1 Improved web-console primitives. --- .../helpers/jade/form/form-field-checkbox.pug | 2 +- .../helpers/jade/form/form-field-dropdown.pug | 1 - .../frontend/app/helpers/jade/mixins.pug | 1 + .../app/primitives/dropdown/index.scss | 4 ++ .../app/primitives/form-field/index.scss | 48 +++++++------- .../frontend/app/primitives/index.js | 2 + .../primitives/{switch => radio}/index.pug | 13 +++- .../primitives/{switch => radio}/index.scss | 65 ++++++++----------- .../app/primitives/switcher/index.scss | 2 +- .../app/primitives/tooltip/index.scss | 25 +++++++ .../app/primitives/typography/index.scss | 2 +- .../primitives/ui-grid-settings/index.scss | 47 +++++++------- .../app/primitives/ui-grid/index.scss | 13 ++-- .../frontend/app/services/Confirm.service.js | 2 +- .../frontend/app/services/Messages.service.js | 3 + 15 files changed, 135 insertions(+), 95 deletions(-) rename modules/web-console/frontend/app/primitives/{switch => radio}/index.pug (79%) rename modules/web-console/frontend/app/primitives/{switch => radio}/index.scss (59%) create mode 100644 modules/web-console/frontend/app/primitives/tooltip/index.scss diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug index 497680b9c25cf..fcd6f9dd51bdb 100644 --- a/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug +++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug @@ -15,7 +15,7 @@ limitations under the License. mixin form-field-checkbox(label, model, name, disabled, required, tip) - .checkbox + .checkbox(id=`{{ ${name} }}Field`) label(id=`{{ ${name} }}Label`) .input-tip if block diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug index d0f95d477973d..117568d24cbc5 100644 --- a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug +++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug @@ -25,7 +25,6 @@ mixin ignite-form-field-dropdown(label, model, name, disabled, required, multipl data-ng-model=model data-ng-required=required && `${required}` - data-ng-disabled=disabled && `${disabled}` || `!${options}.length` bs-select bs-options=`item.value as item.label for item in ${options}` diff --git a/modules/web-console/frontend/app/helpers/jade/mixins.pug b/modules/web-console/frontend/app/helpers/jade/mixins.pug index fc193eb502f4f..9ccbde2f7f7ab 100644 --- a/modules/web-console/frontend/app/helpers/jade/mixins.pug +++ b/modules/web-console/frontend/app/helpers/jade/mixins.pug @@ -20,6 +20,7 @@ include ../../primitives/datepicker/index include ../../primitives/timepicker/index include ../../primitives/dropdown/index include ../../primitives/tooltip/index +include ../../primitives/radio/index include ../../primitives/switcher/index //- Mixin for advanced options toggle. diff --git a/modules/web-console/frontend/app/primitives/dropdown/index.scss b/modules/web-console/frontend/app/primitives/dropdown/index.scss index d39306d2c269e..47a7e900aa648 100644 --- a/modules/web-console/frontend/app/primitives/dropdown/index.scss +++ b/modules/web-console/frontend/app/primitives/dropdown/index.scss @@ -56,6 +56,10 @@ ul.select.dropdown-menu { li { + hr { + display: none; + } + a { &:before { content: ''; diff --git a/modules/web-console/frontend/app/primitives/form-field/index.scss b/modules/web-console/frontend/app/primitives/form-field/index.scss index 42370b333f041..4bc42528dfe5f 100644 --- a/modules/web-console/frontend/app/primitives/form-field/index.scss +++ b/modules/web-console/frontend/app/primitives/form-field/index.scss @@ -29,7 +29,7 @@ .ignite-form-field__label { float: left; width: 100%; - margin: 4px 10px; + margin: 0 10px 4px; color: $gray-light; font-size: 12px; @@ -42,35 +42,35 @@ .input-tip { display: flex; overflow: visible; - } - .tipField { - line-height: 36px; - } + & > input, + & > button { + overflow: visible; - input, - button { - overflow: visible; + box-sizing: border-box; + width: 100%; + max-width: initial; + height: 36px; + padding: 0 10px; + margin-right: 0; - box-sizing: border-box; - width: 100%; - max-width: initial; - height: 36px; - padding: 0px 10px; - margin-right: 0; + border: solid 1px #c5c5c5; + border-radius: 4px; + background-color: #ffffff; + box-shadow: inset 0 1px 3px 0 rgba(0, 0, 0, 0.2); - border: solid 1px #c5c5c5; - border-radius: 4px; - background-color: #ffffff; - box-shadow: inset 0 1px 3px 0 rgba(0, 0, 0, 0.2); + color: $text-color; + line-height: 36px; + } - color: $text-color; - line-height: 36px; + & > input[type='number'] { + text-align: right; + } } - input[type='number'] { - text-align: right; + .tipField { + line-height: 36px; } - } + } } -} \ No newline at end of file +} diff --git a/modules/web-console/frontend/app/primitives/index.js b/modules/web-console/frontend/app/primitives/index.js index 219581c852b9a..5a2f45ceef415 100644 --- a/modules/web-console/frontend/app/primitives/index.js +++ b/modules/web-console/frontend/app/primitives/index.js @@ -28,7 +28,9 @@ import './ui-grid/index.scss'; import './ui-grid-header/index.scss'; import './ui-grid-settings/index.scss'; import './page/index.scss'; +import './radio/index.scss'; import './switcher/index.scss'; import './form-field/index.scss'; import './typography/index.scss'; import './grid/index.scss'; +import './tooltip/index.scss'; diff --git a/modules/web-console/frontend/app/primitives/switch/index.pug b/modules/web-console/frontend/app/primitives/radio/index.pug similarity index 79% rename from modules/web-console/frontend/app/primitives/switch/index.pug rename to modules/web-console/frontend/app/primitives/radio/index.pug index 02b9852922cc9..2b2223ae49434 100644 --- a/modules/web-console/frontend/app/primitives/switch/index.pug +++ b/modules/web-console/frontend/app/primitives/radio/index.pug @@ -14,8 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. -mixin form-field-switch(label, model, name, disabled, required) - .switch--ignite +mixin form-field-radio(label, model, name, value, disabled, required, tip) + .radio--ignite.ignite-form-field label(id=`{{ ${name} }}Label`) .input-tip if block @@ -24,11 +24,18 @@ mixin form-field-switch(label, model, name, disabled, required) input( id=`{{ ${name} }}Input` name=`{{ ${name} }}` - type='checkbox' + type='radio' data-ng-model=model + data-ng-value=value data-ng-required=required && `${required}` data-ng-disabled=disabled && `${disabled}` + + data-ng-focus='tableReset()' + + data-ignite-form-panel-field='' ) div span #{label} + + +tooltip(tip, tipOpts) diff --git a/modules/web-console/frontend/app/primitives/switch/index.scss b/modules/web-console/frontend/app/primitives/radio/index.scss similarity index 59% rename from modules/web-console/frontend/app/primitives/switch/index.scss rename to modules/web-console/frontend/app/primitives/radio/index.scss index 37b2f554cc38d..ff9b5b3a43a1a 100644 --- a/modules/web-console/frontend/app/primitives/switch/index.scss +++ b/modules/web-console/frontend/app/primitives/radio/index.scss @@ -17,14 +17,8 @@ @import '../../../public/stylesheets/variables'; -.switch--ignite { - width: 34px; - height: 20px; - +.radio--ignite { label { - width: 34px; - max-width: 34px !important; - height: 100%; padding-left: 20px; line-height: 20px; @@ -34,54 +28,51 @@ .input-tip { float: left; - - height: 100%; + width: 14px; margin-left: -20px; - input[type="checkbox"] { - position: absolute; + input[type="radio"] { + position: relative; left: -20px; & + div { - position: relative; + position: absolute; + top: 50%; - width: 34px; + width: 14px; height: 14px; - margin-top: 3px; - - border-radius: 8px; - background-color: #C5C5C5; - - &:before { - content: ''; - - position: absolute; - top: -3px; - left: 0; - - width: 20px; - height: 20px; + margin-top: -8px; - border: solid 1px #C5C5C5; - border-radius: 50%; - background-color: #FFF; - } + border-radius: 50%; + box-shadow: inset 0 1px 3px 0 rgba(0, 0, 0, 0.35); } &:checked + div { - background-color: #FF8485; + background-color: #0098ff; + box-shadow: none; &:before { content: ''; - left: initial; - right: 0; + position: absolute; + top: 50%; + left: 50%; + + width: 4px; + height: 4px; + margin-top: -2px; + margin-left: -2px; - border: 0; - background-color: #EE2B27; + border-radius: 50%; + background-color: #ffffff; + box-shadow: 0 1px 1px 0 rgba(12, 50, 76, 0.3); } } } } } -} + + & + .radio--ignite { + margin-left: 45px; + } +} \ No newline at end of file diff --git a/modules/web-console/frontend/app/primitives/switcher/index.scss b/modules/web-console/frontend/app/primitives/switcher/index.scss index cf8ed64462197..3e9cd49aa3563 100644 --- a/modules/web-console/frontend/app/primitives/switcher/index.scss +++ b/modules/web-console/frontend/app/primitives/switcher/index.scss @@ -30,7 +30,7 @@ label.switcher--ignite { input[type="checkbox"] { position: absolute; opacity: 0.0; - + & + div { position: relative; diff --git a/modules/web-console/frontend/app/primitives/tooltip/index.scss b/modules/web-console/frontend/app/primitives/tooltip/index.scss new file mode 100644 index 0000000000000..174d624dad133 --- /dev/null +++ b/modules/web-console/frontend/app/primitives/tooltip/index.scss @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.tooltip { + font-family: Roboto; + + & > .tooltip-inner { + padding: 12px; + background-color: #FFF; + } +} \ No newline at end of file diff --git a/modules/web-console/frontend/app/primitives/typography/index.scss b/modules/web-console/frontend/app/primitives/typography/index.scss index 7fee67438528b..9baa444c94b74 100644 --- a/modules/web-console/frontend/app/primitives/typography/index.scss +++ b/modules/web-console/frontend/app/primitives/typography/index.scss @@ -33,4 +33,4 @@ font-size: 16px; font-weight: normal; } -} \ No newline at end of file +} diff --git a/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss b/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss index e5d30e6679ef9..e0cf13907709a 100644 --- a/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss +++ b/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss @@ -186,30 +186,33 @@ .input-tip { overflow: visible; - } - button { - overflow: visible; + & > button { + overflow: visible; + + width: auto; + height: $height; + min-width: 70px; + max-width: 70px; + padding: 0 0 0 5px; + + cursor: pointer; + color: transparent; + font-size: inherit; + line-height: $height; + text-align: left; + text-shadow: 0 0 0 #ee2b27; - width: auto; - height: $height; - min-width: 70px; - max-width: 70px; - padding: 0; - padding-left: 5px; - - cursor: pointer; - color: transparent; - font-size: inherit; - line-height: $height; - text-align: left; - text-shadow: 0 0 0 #ee2b27; - - border: none; - box-shadow: none; - - &:hover, &:focus { - text-shadow: 0 0 0 #a8110f; + border: none; + box-shadow: none; + + &:hover, &:focus { + text-shadow: 0 0 0 #a8110f; + } + + button { + color: $text-color; + } } } diff --git a/modules/web-console/frontend/app/primitives/ui-grid/index.scss b/modules/web-console/frontend/app/primitives/ui-grid/index.scss index 8e1da49c4b83d..88bff694daa22 100644 --- a/modules/web-console/frontend/app/primitives/ui-grid/index.scss +++ b/modules/web-console/frontend/app/primitives/ui-grid/index.scss @@ -27,7 +27,7 @@ outline: none; } - sup { + sup, sub { color: $ignite-brand-success; } @@ -122,7 +122,6 @@ z-index: 1; position: fixed; - width: calc(100% - 40px); width: 100px; height: 20px; margin-top: -20px; @@ -314,7 +313,7 @@ .ui-grid-header, .ui-grid-viewport { .ui-grid-icon-cancel { - right: 20px; + right: 10px; } .ui-grid-tree-base-row-header-buttons { @@ -377,6 +376,12 @@ } } + .ui-grid-header--subcategories { + .ui-grid-icon-cancel { + right: 20px; + } + } + .ui-grid-pinned-container { .ui-grid-header { .ui-grid-header-cell-row { @@ -518,4 +523,4 @@ font-style: italic; line-height: 16px; -} \ No newline at end of file +} diff --git a/modules/web-console/frontend/app/services/Confirm.service.js b/modules/web-console/frontend/app/services/Confirm.service.js index 1638d7ed7f651..2429f4aad051b 100644 --- a/modules/web-console/frontend/app/services/Confirm.service.js +++ b/modules/web-console/frontend/app/services/Confirm.service.js @@ -46,7 +46,7 @@ export default ['IgniteConfirm', ['$rootScope', '$q', '$modal', '$animate', ($ro scope.confirmCancel = () => { _hide(); - deferred.reject('cancelled'); + deferred.reject({cancelled: true}); }; /** diff --git a/modules/web-console/frontend/app/services/Messages.service.js b/modules/web-console/frontend/app/services/Messages.service.js index fefdae9a4378c..5e691bc08f8ec 100644 --- a/modules/web-console/frontend/app/services/Messages.service.js +++ b/modules/web-console/frontend/app/services/Messages.service.js @@ -55,6 +55,9 @@ export default ['IgniteMessages', ['$alert', ($alert) => { errorMessage, hideAlert, showError(message, err) { + if (message && message.cancelled) + return false; + _showMessage(message, err, 'danger', 10); return false; From 914fdc7b83ea5e73d2ee665fbb32bfdedc5e2dc7 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 11 Jul 2017 19:16:58 +0700 Subject: [PATCH 137/155] ignite-2.1 Web Console: improved demo. --- .../apache/ignite/console/demo/AgentClusterDemo.java | 12 ++++++++++++ .../console/demo/service/DemoCachesLoadService.java | 1 + 2 files changed, 13 insertions(+) diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java index cf7ae55dea964..73577b5672708 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java @@ -28,6 +28,8 @@ import org.apache.ignite.IgniteServices; import org.apache.ignite.Ignition; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.MemoryConfiguration; +import org.apache.ignite.configuration.MemoryPolicyConfiguration; import org.apache.ignite.console.demo.service.DemoCachesLoadService; import org.apache.ignite.console.demo.service.DemoComputeLoadService; import org.apache.ignite.console.demo.service.DemoRandomCacheLoadService; @@ -123,6 +125,16 @@ private static IgniteConfiguration igniteConfiguration(int basePort, int gridIdx cfg.setGridLogger(new Slf4jLogger(log)); cfg.setMetricsLogFrequency(0); + MemoryConfiguration memCfg = new MemoryConfiguration(); + + MemoryPolicyConfiguration memPlc = new MemoryPolicyConfiguration(); + memPlc.setName("demo"); + memPlc.setMetricsEnabled(true); + + memCfg.setMemoryPolicies(memPlc); + + cfg.setMemoryConfiguration(memCfg); + if (client) cfg.setClientMode(true); diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java index 1e862783355f7..40fd4ac1ba1fe 100644 --- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java +++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java @@ -208,6 +208,7 @@ private static CacheConfiguration cacheConfiguration(String name) { ccfg.setQueryDetailMetricsSize(10); ccfg.setStatisticsEnabled(true); ccfg.setSqlFunctionClasses(SQLFunctions.class); + ccfg.setMemoryPolicyName("demo"); return ccfg; } From 66ccf8594cc6de74c3d87620bc3c85cd86b571fa Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 11 Jul 2017 19:17:48 +0700 Subject: [PATCH 138/155] ignite-2.1 Minor update of generated classnames. --- .../visor/VisorCoordinatorNodeTask.java | 39 +++++++++++++++++++ .../resources/META-INF/classnames.properties | 13 +++++-- 2 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/VisorCoordinatorNodeTask.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorCoordinatorNodeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorCoordinatorNodeTask.java new file mode 100644 index 0000000000000..f744e9ead376d --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorCoordinatorNodeTask.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.UUID; +import org.apache.ignite.cluster.ClusterNode; + +/** + * Base class for Visor tasks intended to execute job on coordinator node. + */ +public abstract class VisorCoordinatorNodeTask extends VisorOneNodeTask { + /** {@inheritDoc} */ + @Override protected Collection jobNodes(VisorTaskArgument arg) { + ClusterNode crd = ignite.context().discovery().discoCache().oldestAliveServerNode(); + + Collection nids = new ArrayList<>(1); + + nids.add(crd == null ? ignite.localNode().id() : crd.id()); + + return nids; + } +} diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 8c0f400e52899..675bd56c5bc64 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -337,7 +337,6 @@ org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager$1 org.apache.ignite.internal.marshaller.optimized.OptimizedFieldType org.apache.ignite.internal.mem.IgniteOutOfMemoryException org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl$Segment -org.apache.ignite.internal.pagemem.snapshot.SnapshotCheckParameters org.apache.ignite.internal.pagemem.wal.StorageException org.apache.ignite.internal.pagemem.wal.WALIterator org.apache.ignite.internal.pagemem.wal.record.TxRecord$TxAction @@ -870,7 +869,11 @@ org.apache.ignite.internal.processors.cache.persistence.FullPageIdIterableCompar org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$8 org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$CheckpointEntryType org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$RebalanceIteratorAdapter +org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory +org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl$Segment +org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage +org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Bool org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$DestroyBag org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Result @@ -1077,11 +1080,12 @@ org.apache.ignite.internal.processors.datastructures.AtomicDataStructureValue org.apache.ignite.internal.processors.datastructures.DataStructureInfoKey org.apache.ignite.internal.processors.datastructures.DataStructureType org.apache.ignite.internal.processors.datastructures.DataStructuresCacheKey -org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$11 -org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$13 -org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$15 +org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$10 +org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$12 +org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$14 org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$16 org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$17 +org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$18 org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$4 org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$5 org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor$8 @@ -1699,6 +1703,7 @@ org.apache.ignite.internal.util.typedef.T4 org.apache.ignite.internal.util.typedef.T5 org.apache.ignite.internal.util.typedef.T6 org.apache.ignite.internal.util.typedef.internal.SB +org.apache.ignite.internal.visor.VisorCoordinatorNodeTask org.apache.ignite.internal.visor.VisorDataTransferObject org.apache.ignite.internal.visor.VisorEither org.apache.ignite.internal.visor.VisorJob From 2c737f0c6f42e983606e70faab5e9e6336b0a241 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Tue, 11 Jul 2017 15:23:26 +0300 Subject: [PATCH 139/155] IGNITE-5711: Allowed to run queries on caches without query entities. This closes #2264. --- .../datagrid/CacheQueryDdlExample.java | 3 +- .../jdbc2/JdbcDefaultNoOpCacheTest.java | 33 ++++++++++++ .../ignite/jdbc/JdbcDefaultNoOpCacheTest.java | 35 +++++++++++++ .../ignite/jdbc/JdbcNoDefaultCacheTest.java | 50 +++++++++++-------- .../jdbc/suite/IgniteJdbcDriverTestSuite.java | 5 +- .../processors/cache/GridCacheProcessor.java | 4 +- .../processors/cache/IgniteCacheProxy.java | 5 +- ...ridCacheQueryIndexingDisabledSelfTest.java | 16 +++--- ...gniteCacheAbstractFieldsQuerySelfTest.java | 30 ++++++++--- ...teCachePartitionedFieldsQuerySelfTest.java | 20 ++++++-- .../Cache/Query/CacheQueriesTest.cs | 19 +++---- 11 files changed, 161 insertions(+), 59 deletions(-) create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDefaultNoOpCacheTest.java create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcDefaultNoOpCacheTest.java diff --git a/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java b/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java index 201dda10bd8ac..e27907d5013a7 100644 --- a/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java +++ b/examples/src/main/java/org/apache/ignite/examples/datagrid/CacheQueryDdlExample.java @@ -50,8 +50,7 @@ public static void main(String[] args) throws Exception { // Create dummy cache to act as an entry point for SQL queries (new SQL API which do not require this // will appear in future versions, JDBC and ODBC drivers do not require it already). - CacheConfiguration cacheCfg = new CacheConfiguration<>(DUMMY_CACHE_NAME) - .setSqlSchema("PUBLIC").setIndexedTypes(Integer.class, Integer.class); + CacheConfiguration cacheCfg = new CacheConfiguration<>(DUMMY_CACHE_NAME).setSqlSchema("PUBLIC"); try ( IgniteCache cache = ignite.getOrCreateCache(cacheCfg) diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDefaultNoOpCacheTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDefaultNoOpCacheTest.java new file mode 100644 index 0000000000000..57ef52ca04b8d --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDefaultNoOpCacheTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.jdbc2; + +import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX; + +/** + * Test to check JDBC2 driver behavior when cache specified in connection string does not have any query entities. + */ +public class JdbcDefaultNoOpCacheTest extends org.apache.ignite.jdbc.JdbcDefaultNoOpCacheTest { + /** Ignite configuration URL. */ + private static final String CFG_URL = "modules/clients/src/test/config/jdbc-config.xml"; + + /** {@inheritDoc} */ + protected String getUrl() { + return CFG_URL_PREFIX + "cache=noop@" + CFG_URL; + } +} diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcDefaultNoOpCacheTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcDefaultNoOpCacheTest.java new file mode 100644 index 0000000000000..f1143f9012a2a --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcDefaultNoOpCacheTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.jdbc; + +/** + * Test to check JDBC driver behavior when cache specified in connection string does not have any query entities. + */ +public class JdbcDefaultNoOpCacheTest extends JdbcNoDefaultCacheTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + ignite(0).getOrCreateCache("noop"); + } + + /** {@inheritDoc} */ + @Override protected String getUrl() { + return super.getUrl() + "noop"; + } +} diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcNoDefaultCacheTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcNoDefaultCacheTest.java index d3d8454cea9a2..a37c16764f0b2 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcNoDefaultCacheTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcNoDefaultCacheTest.java @@ -31,7 +31,6 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * @@ -105,11 +104,18 @@ private CacheConfiguration cacheConfiguration(@NotNull String name) throws Excep stopAllGrids(); } + /** + * @return Connection URL. + */ + protected String getUrl() { + return URL; + } + /** * @throws Exception If failed. */ public void testDefaults() throws Exception { - String url = URL; + String url = getUrl(); try (Connection conn = DriverManager.getConnection(url)) { assertNotNull(conn); @@ -126,35 +132,37 @@ public void testDefaults() throws Exception { public void testNoCacheNameQuery() throws Exception { Statement stmt; - stmt = DriverManager.getConnection(URL).createStatement(); + try (Connection conn = DriverManager.getConnection(getUrl())) { + stmt = conn.createStatement(); - assertNotNull(stmt); - assertFalse(stmt.isClosed()); + assertNotNull(stmt); + assertFalse(stmt.isClosed()); - stmt.execute("select t._key, t._val from \"cache1\".Integer t"); + stmt.execute("select t._key, t._val from \"cache1\".Integer t"); - ResultSet rs = stmt.getResultSet(); + ResultSet rs = stmt.getResultSet(); - while(rs.next()) - assertEquals(rs.getInt(2), rs.getInt(1) * 2); + while (rs.next()) + assertEquals(rs.getInt(2), rs.getInt(1) * 2); - stmt.execute("select t._key, t._val from \"cache2\".Integer t"); + stmt.execute("select t._key, t._val from \"cache2\".Integer t"); - rs = stmt.getResultSet(); + rs = stmt.getResultSet(); - while(rs.next()) - assertEquals(rs.getInt(2), rs.getInt(1) * 3); + while (rs.next()) + assertEquals(rs.getInt(2), rs.getInt(1) * 3); - stmt.execute("select t._key, t._val, v._val " + - "from \"cache1\".Integer t join \"cache2\".Integer v on t._key = v._key"); + stmt.execute("select t._key, t._val, v._val " + + "from \"cache1\".Integer t join \"cache2\".Integer v on t._key = v._key"); - rs = stmt.getResultSet(); + rs = stmt.getResultSet(); - while(rs.next()) { - assertEquals(rs.getInt(2), rs.getInt(1) * 2); - assertEquals(rs.getInt(3), rs.getInt(1) * 3); - } + while (rs.next()) { + assertEquals(rs.getInt(2), rs.getInt(1) * 2); + assertEquals(rs.getInt(3), rs.getInt(1) * 3); + } - stmt.close(); + stmt.close(); + } } } diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java index 9ca3582ce55c0..8ca3d45886343 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.jdbc2.JdbcDistributedJoinsQueryTest; import org.apache.ignite.jdbc.JdbcComplexQuerySelfTest; import org.apache.ignite.jdbc.JdbcConnectionSelfTest; +import org.apache.ignite.jdbc.JdbcDefaultNoOpCacheTest; import org.apache.ignite.jdbc.JdbcEmptyCacheSelfTest; import org.apache.ignite.jdbc.JdbcLocalCachesSelfTest; import org.apache.ignite.jdbc.JdbcMetadataSelfTest; @@ -33,6 +34,7 @@ import org.apache.ignite.jdbc.JdbcStatementSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinAutoCloseServerCursorTest; import org.apache.ignite.jdbc.thin.JdbcThinComplexQuerySelfTest; +import org.apache.ignite.jdbc.thin.JdbcThinConnectionSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinDeleteStatementSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinDynamicIndexAtomicPartitionedNearSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinDynamicIndexAtomicPartitionedSelfTest; @@ -48,7 +50,6 @@ import org.apache.ignite.jdbc.thin.JdbcThinPreparedStatementSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinResultSetSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinStatementSelfTest; -import org.apache.ignite.jdbc.thin.JdbcThinConnectionSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest; /** @@ -72,6 +73,7 @@ public static TestSuite suite() throws Exception { suite.addTest(new TestSuite(JdbcEmptyCacheSelfTest.class)); suite.addTest(new TestSuite(JdbcLocalCachesSelfTest.class)); suite.addTest(new TestSuite(JdbcNoDefaultCacheTest.class)); + suite.addTest(new TestSuite(JdbcDefaultNoOpCacheTest.class)); suite.addTest(new TestSuite(JdbcPojoQuerySelfTest.class)); suite.addTest(new TestSuite(JdbcPojoLegacyQuerySelfTest.class)); @@ -87,6 +89,7 @@ public static TestSuite suite() throws Exception { suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcEmptyCacheSelfTest.class)); suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcLocalCachesSelfTest.class)); suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcNoDefaultCacheTest.class)); + suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDefaultNoOpCacheTest.class)); suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcMergeStatementSelfTest.class)); suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcBinaryMarshallerMergeStatementSelfTest.class)); suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcInsertStatementSelfTest.class)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 0488a142533c6..85772d8f9ae64 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -1672,7 +1672,7 @@ public Collection cacheNames() { String cacheName = ccfg.getName(); - if ((inclLoc || ccfg.getCacheMode() != LOCAL) && QueryUtils.isEnabled(ccfg)) + if ((inclLoc || ccfg.getCacheMode() != LOCAL)) return publicJCache(cacheName); } @@ -1682,7 +1682,7 @@ public Collection cacheNames() { CacheConfiguration ccfg = desc.cacheConfiguration(); - if (ccfg.getCacheMode() != LOCAL && QueryUtils.isEnabled(ccfg)) { + if (ccfg.getCacheMode() != LOCAL) { dynamicStartCache(null, ccfg.getName(), null, false, true, true).get(); return publicJCache(ccfg.getName()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java index c8dc8dc030e1b..347e030d2ed1d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java @@ -73,8 +73,8 @@ import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.util.GridCloseableIteratorAdapter; import org.apache.ignite.internal.util.GridEmptyIterator; -import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl; import org.apache.ignite.internal.util.future.IgniteFutureImpl; import org.apache.ignite.internal.util.lang.GridCloseableIterator; import org.apache.ignite.internal.util.lang.GridClosureException; @@ -887,7 +887,8 @@ private void convertToBinary(final Object[] args) { */ private void validate(Query qry) { if (!QueryUtils.isEnabled(ctx.config()) && !(qry instanceof ScanQuery) && - !(qry instanceof ContinuousQuery) && !(qry instanceof SpiQuery)) + !(qry instanceof ContinuousQuery) && !(qry instanceof SpiQuery) && !(qry instanceof SqlQuery) && + !(qry instanceof SqlFieldsQuery)) throw new CacheException("Indexing is disabled for cache: " + ctx.cache().name() + ". Use setIndexedTypes or setTypeMetadata methods on CacheConfiguration to enable."); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheQueryIndexingDisabledSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheQueryIndexingDisabledSelfTest.java index 1696d3a3c69ca..92a70847fbc95 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheQueryIndexingDisabledSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheQueryIndexingDisabledSelfTest.java @@ -49,19 +49,16 @@ public class GridCacheQueryIndexingDisabledSelfTest extends GridCacheAbstractSel /** * @param c Closure. */ - private void doTest(Callable c) { - GridTestUtils.assertThrows(log, c, CacheException.class, "Indexing is disabled for cache: default"); + private void doTest(Callable c, String expectedMsg) { + GridTestUtils.assertThrows(log, c, CacheException.class, expectedMsg); } /** * @throws IgniteCheckedException If failed. */ public void testSqlFieldsQuery() throws IgniteCheckedException { - doTest(new Callable() { - @Override public Object call() throws IgniteCheckedException { - return jcache().query(new SqlFieldsQuery("select * from dual")).getAll(); - } - }); + // Should not throw despite the cache not having QueryEntities. + jcache().query(new SqlFieldsQuery("select * from dual")).getAll(); } /** @@ -72,18 +69,19 @@ public void testTextQuery() throws IgniteCheckedException { @Override public Object call() throws IgniteCheckedException { return jcache().query(new TextQuery<>(String.class, "text")).getAll(); } - }); + }, "Indexing is disabled for cache: default"); } /** * @throws IgniteCheckedException If failed. */ public void testSqlQuery() throws IgniteCheckedException { + // Failure occurs not on validation stage, hence specific error message. doTest(new Callable() { @Override public Object call() throws IgniteCheckedException { return jcache().query(new SqlQuery<>(String.class, "1 = 1")).getAll(); } - }); + }, "Failed to find SQL table for type: String"); } /** diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java index 1deee05151714..322598a43e9a5 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java @@ -72,9 +72,6 @@ public abstract class IgniteCacheAbstractFieldsQuerySelfTest extends GridCommonA /** */ private static IgniteCache orgCache; - /** Cache name. */ - protected static final String CACHE = "cache"; - /** */ private static IgniteCache, Person> personCache; @@ -84,6 +81,9 @@ public abstract class IgniteCacheAbstractFieldsQuerySelfTest extends GridCommonA /** */ protected static IgniteCache intCache; + /** */ + protected static IgniteCache noOpCache; + /** Flag indicating if starting node should have cache. */ protected boolean hasCache; @@ -187,6 +187,8 @@ protected IgniteCache jcache(String name, Class clsK, Class c for (int i = 0; i < 200; i++) intCache.put(i, i); + + noOpCache = grid(0).getOrCreateCache("noop"); } /** {@inheritDoc} */ @@ -202,6 +204,7 @@ protected IgniteCache jcache(String name, Class clsK, Class c personCache = null; strCache = null; intCache = null; + noOpCache = null; } /** @return cache mode. */ @@ -345,7 +348,7 @@ else if (strCache.getName().equals(meta.cacheName())) { assert String.class.getName().equals(fields.get("_KEY")); assert String.class.getName().equals(fields.get("_VAL")); } - else if (DEFAULT_CACHE_NAME.equals(meta.cacheName())) + else if (DEFAULT_CACHE_NAME.equals(meta.cacheName()) || noOpCache.getName().equals(meta.cacheName())) assertTrue("Invalid types size", types.isEmpty()); else fail("Unknown cache: " + meta.cacheName()); @@ -479,7 +482,22 @@ else if (cnt == 1) { /** @throws Exception If failed. */ public void testExecute() throws Exception { - QueryCursor> qry = personCache.query(sqlFieldsQuery("select _KEY, name, age from Person")); + doTestExecute(personCache, sqlFieldsQuery("select _KEY, name, age from Person")); + } + + /** @throws Exception If failed. */ + public void testExecuteNoOpCache() throws Exception { + doTestExecute(noOpCache, sqlFieldsQuery("select _KEY, name, age from \"AffinityKey-Person\".Person")); + } + + /** + * Execute given query and check results. + * @param cache Cache to run query on. + * @param fldsQry Query. + * @throws Exception if failed. + */ + private void doTestExecute (IgniteCache cache, SqlFieldsQuery fldsQry) throws Exception { + QueryCursor> qry = cache.query(fldsQry); List> res = new ArrayList<>(qry.getAll()); @@ -529,8 +547,6 @@ public void testExecuteWithArguments() throws Exception { List> res = new ArrayList<>(qry.getAll()); - assert res != null; - dedup(res); assertEquals(2, res.size()); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java index 49342559b7c92..7f9989ddeaefc 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCachePartitionedFieldsQuerySelfTest.java @@ -52,6 +52,7 @@ protected NearCacheConfiguration nearConfiguration() { } /** {@inheritDoc} */ + @SuppressWarnings("unchecked") @Override protected CacheConfiguration cacheConfiguration() { CacheConfiguration cc = super.cacheConfiguration(); @@ -62,19 +63,30 @@ protected NearCacheConfiguration nearConfiguration() { /** @throws Exception If failed. */ public void testLocalQuery() throws Exception { - IgniteCache cache = jcache(Integer.class, Integer.class); + doTestLocalQuery(intCache, new SqlFieldsQuery("select _key, _val from Integer")); + } + + /** @throws Exception If failed. */ + public void testLocalQueryNoOpCache() throws Exception { + doTestLocalQuery(noOpCache, new SqlFieldsQuery("select _key, _val from \"Integer-Integer\".Integer")); + } + /** + * Execute given query locally and check results. + * @param cache Cache to run query on. + * @param fldsQry Query. + */ + private void doTestLocalQuery(IgniteCache cache, SqlFieldsQuery fldsQry) throws InterruptedException { awaitPartitionMapExchange(true, true, null); int exp = 0; - for(Cache.Entry e: cache.localEntries(CachePeekMode.PRIMARY)){ + for(Cache.Entry e: intCache.localEntries(CachePeekMode.PRIMARY)){ if(e.getValue() instanceof Integer) exp++; } - QueryCursor> qry = cache - .query(new SqlFieldsQuery("select _key, _val from Integer").setLocal(true)); + QueryCursor> qry = cache.query(fldsQry.setLocal(true)); assertEquals(exp, qry.getAll().size()); } diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs index 62c82300f9c2b..ae2fe8f074861 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs @@ -480,19 +480,16 @@ public void TestIndexingDisabledError() { var cache = GetIgnite().GetOrCreateCache("nonindexed_cache"); - var queries = new QueryBase[] - { - new TextQuery(typeof (QueryPerson), "1*"), - new SqlQuery(typeof (QueryPerson), "age < 50") - }; + // Text query. + var err = Assert.Throws(() => cache.Query(new TextQuery(typeof(QueryPerson), "1*"))); - foreach (var qry in queries) - { - var err = Assert.Throws(() => cache.Query(qry)); + Assert.AreEqual("Indexing is disabled for cache: nonindexed_cache. " + + "Use setIndexedTypes or setTypeMetadata methods on CacheConfiguration to enable.", err.Message); - Assert.AreEqual("Indexing is disabled for cache: nonindexed_cache. " + - "Use setIndexedTypes or setTypeMetadata methods on CacheConfiguration to enable.", err.Message); - } + // SQL query. + err = Assert.Throws(() => cache.Query(new SqlQuery(typeof(QueryPerson), "age < 50"))); + + Assert.AreEqual("Failed to find SQL table for type: QueryPerson", err.Message); } /// From a0b56442365bad1100b078da33f00beb6a844cf0 Mon Sep 17 00:00:00 2001 From: Sergey Chugunov Date: Tue, 11 Jul 2017 15:59:16 +0300 Subject: [PATCH 140/155] Minor fix - stopping nodes after test --- .../processors/cache/IgniteMarshallerCacheFSRestoreTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java index 38fa324e1abdd..21a3e4344ba85 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java @@ -89,6 +89,8 @@ private static class SimpleValue { /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { cleanUpWorkDir(); + + stopAllGrids(); } /** From 478d3b5d3361c3d74d0da4b6a78e9944d8b95630 Mon Sep 17 00:00:00 2001 From: "Andrey V. Mashenkov" Date: Tue, 11 Jul 2017 22:28:45 +0300 Subject: [PATCH 141/155] IGNITE-3562: Updated Lucene dependency to version 5.5.2. This closes #1987. --- .../cache/GridCacheLuceneQueryIndexTest.java | 466 ------------------ modules/indexing/pom.xml | 12 + .../query/h2/opt/GridLuceneDirectory.java | 47 +- .../query/h2/opt/GridLuceneIndex.java | 75 +-- .../query/h2/opt/GridLuceneInputStream.java | 94 ++-- .../query/h2/opt/GridLuceneLockFactory.java | 45 +- .../query/h2/opt/GridLuceneOutputStream.java | 72 +-- .../cache/GridCacheFullTextQuerySelfTest.java | 367 ++++++++++++++ ...CacheFullTextQueryNodeJoiningSelfTest.java | 4 +- .../IgniteCacheQuerySelfTestSuite.java | 7 +- parent/pom.xml | 4 +- 11 files changed, 563 insertions(+), 630 deletions(-) delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLuceneQueryIndexTest.java create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLuceneQueryIndexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLuceneQueryIndexTest.java deleted file mode 100644 index 585ef1bd7ba2f..0000000000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLuceneQueryIndexTest.java +++ /dev/null @@ -1,466 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.query.annotations.QueryTextField; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.jetbrains.annotations.Nullable; - -import static org.apache.ignite.cache.CacheMode.LOCAL; -import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; - -/** - * - */ -public class GridCacheLuceneQueryIndexTest extends GridCommonAbstractTest { - /** */ - private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); - - /** - * - */ - public GridCacheLuceneQueryIndexTest() { - super(false); - } - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - TcpDiscoverySpi disco = new TcpDiscoverySpi(); - - disco.setIpFinder(ipFinder); - - cfg.setDiscoverySpi(disco); - - cfg.setIncludeEventTypes(); - cfg.setConnectorConfiguration(null); - - CacheConfiguration cacheCfg1 = defaultCacheConfiguration(); - - cacheCfg1.setName("local1"); - cacheCfg1.setCacheMode(LOCAL); - cacheCfg1.setWriteSynchronizationMode(FULL_SYNC); - - CacheConfiguration cacheCfg2 = defaultCacheConfiguration(); - - cacheCfg2.setName("local2"); - cacheCfg2.setCacheMode(LOCAL); - cacheCfg2.setWriteSynchronizationMode(FULL_SYNC); - - cfg.setCacheConfiguration(cacheCfg1, cacheCfg2); - - return cfg; - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - stopAllGrids(); - } - - /** {@inheritDoc} */ - @Override protected long getTestTimeout() { - return 10 * 60 * 1000; - } - - /** - * Tests puts one by one. - * - * @throws Exception In case of error. - */ - public void testLuceneIndex() throws Exception { - final Ignite g = startGrid(0); - - final IgniteCache cache1 = g.cache("local1"); - final IgniteCache cache2 = g.cache("local2"); - - final AtomicInteger threadIdxGen = new AtomicInteger(); - - final int keyCnt = 10000; - - final IgniteInternalFuture fut = multithreadedAsync( - new Callable() { - @Nullable @Override public Object call() throws Exception { - int threadIdx = threadIdxGen.getAndIncrement() % 2; - - for (int i = 0; i < keyCnt; i++) { - if (threadIdx == 0) - cache1.put(i, new ObjectValue("test full text more" + i)); - else - cache2.put(i, new ObjectValue("test full text more" + i)); - - if (i % 200 == 0) - info("Put entries count: " + i); - } - - return null; - } - }, - 10); - - IgniteInternalFuture fut1 = multithreadedAsync( - new Callable() { - @Nullable @Override public Object call() throws Exception { - while (!fut.isDone()) { - Thread.sleep(10000); - -// ((GridKernal)g).internalCache("local1").context().queries().index().printH2Stats(); -// ((GridKernal)g).internalCache("local2").context().queries().index().printH2Stats(); - } - - return null; - } - }, - 1); - - fut.get(); - fut1.get(); - - assert cache1.size() == keyCnt; - assert cache2.size() == keyCnt; - } - - /** - * Tests with putAll. - * - * @throws Exception In case of error. - */ - public void testLuceneIndex1() throws Exception { - final Ignite g = startGrid(0); - - final IgniteCache cache1 = g.cache("local1"); - final IgniteCache cache2 = g.cache("local2"); - - final AtomicInteger threadIdxGen = new AtomicInteger(); - - final int keyCnt = 10000; - - final IgniteInternalFuture fut = multithreadedAsync( - new Callable() { - @Nullable @Override public Object call() throws Exception { - int threadIdx = threadIdxGen.getAndIncrement() % 2; - - Map map = new HashMap<>(); - - for (int i = 0; i < keyCnt; i++) { - if (i % 200 == 0 && !map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - - info("Put entries count: " + i); - - map = new HashMap<>(); - } - - map.put(i, new ObjectValue("String value " + i)); - } - - if (!map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - } - - return null; - } - }, - 10); - - IgniteInternalFuture fut1 = multithreadedAsync( - new Callable() { - @Nullable @Override public Object call() throws Exception { - while (!fut.isDone()) { - Thread.sleep(10000); - -// ((GridKernal)g).internalCache("local1").context().queries().index().printH2Stats(); -// ((GridKernal)g).internalCache("local2").context().queries().index().printH2Stats(); - } - - return null; - } - }, - 1); - - fut.get(); - fut1.get(); - - assert cache1.size() == keyCnt; - assert cache2.size() == keyCnt; - } - - /** - * Test same value with putAll. - * - * @throws Exception In case of error. - */ - public void testLuceneIndex2() throws Exception { - final Ignite g = startGrid(0); - - final IgniteCache cache1 = g.cache("local1"); - final IgniteCache cache2 = g.cache("local2"); - - final AtomicInteger threadIdxGen = new AtomicInteger(); - - final int keyCnt = 10000; - - final ObjectValue val = new ObjectValue("String value"); - - final IgniteInternalFuture fut = multithreadedAsync( - new Callable() { - @Nullable @Override public Object call() throws Exception { - int threadIdx = threadIdxGen.getAndIncrement() % 2; - - Map map = new HashMap<>(); - - for (int i = 0; i < keyCnt; i++) { - if (i % 200 == 0 && !map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - - info("Put entries count: " + i); - - map = new HashMap<>(); - } - - map.put(i, val); - } - - if (!map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - } - - return null; - } - }, - 10); - - IgniteInternalFuture fut1 = multithreadedAsync( - new Callable() { - @Nullable @Override public Object call() throws Exception { - while (!fut.isDone()) { - Thread.sleep(10000); - -// ((GridKernal)g).internalCache("local1").context().queries().index().printH2Stats(); -// ((GridKernal)g).internalCache("local2").context().queries().index().printH2Stats(); - } - - return null; - } - }, - 1); - - fut.get(); - fut1.get(); - - assert cache1.size() == keyCnt; - assert cache2.size() == keyCnt; - } - - /** - * Test limited values set and custom keys with putAll. - * - * @throws Exception In case of error. - */ - public void testLuceneIndex3() throws Exception { - final Ignite g = startGrid(0); - - final IgniteCache cache1 = g.cache("local1"); - final IgniteCache cache2 = g.cache("local2"); - - final AtomicInteger threadIdxGen = new AtomicInteger(); - - final int keyCnt = 10000; - - final ObjectValue[] vals = new ObjectValue[10]; - - for (int i = 0; i < vals.length; i++) - vals[i] = new ObjectValue("Object value " + i); - - final IgniteInternalFuture fut = multithreadedAsync( - new Callable() { - @Nullable @Override public Object call() throws Exception { - int threadIdx = threadIdxGen.getAndIncrement() % 2; - - Map map = new HashMap<>(); - - for (int i = 0; i < keyCnt; i++) { - if (i % 200 == 0 && !map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - - info("Put entries count: " + i); - - map = new HashMap<>(); - } - - map.put(new ObjectKey(String.valueOf(i)), F.rand(vals)); - } - - if (!map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - } - - return null; - } - }, - 1); - - IgniteInternalFuture fut1 = multithreadedAsync( - new Callable() { - @Nullable @Override public Object call() throws Exception { - while (!fut.isDone()) { - Thread.sleep(10000); - -// ((GridKernal)g).internalCache("local1").context().queries().index().printH2Stats(); -// ((GridKernal)g).internalCache("local2").context().queries().index().printH2Stats(); - } - - return null; - } - }, - 1); - - fut.get(); - fut1.get(); - - assert cache1.size() == keyCnt; - assert cache2.size() == keyCnt; - } - - /** - * Test value object. - */ - private static class ObjectValue implements Serializable { - /** String value. */ - @QueryTextField - private String strVal; - - /** - * @param strVal String value. - */ - ObjectValue(String strVal) { - this.strVal = strVal; - } - - /** - * @return Value. - */ - public String stringValue() { - return strVal; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (o == null || getClass() != o.getClass()) - return false; - - ObjectValue other = (ObjectValue)o; - - return strVal == null ? other.strVal == null : strVal.equals(other.strVal); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return strVal != null ? strVal.hashCode() : 0; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(ObjectValue.class, this); - } - } - - /** - * Test value key. - */ - private static class ObjectKey implements Serializable { - /** String key. */ - @QueryTextField - private String strKey; - - /** - * @param strKey String key. - */ - ObjectKey(String strKey) { - this.strKey = strKey; - } - - /** - * @return Key. - */ - public String stringKey() { - return strKey; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (o == null || getClass() != o.getClass()) - return false; - - ObjectKey other = (ObjectKey)o; - - return strKey == null ? other.strKey == null : strKey.equals(other.strKey); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return strKey != null ? strKey.hashCode() : 0; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(ObjectKey.class, this); - } - } -} \ No newline at end of file diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml index 5c74f3765a2f7..62fc4028578e2 100644 --- a/modules/indexing/pom.xml +++ b/modules/indexing/pom.xml @@ -53,6 +53,18 @@ ${lucene.version} + + org.apache.lucene + lucene-analyzers-common + ${lucene.version} + + + + org.apache.lucene + lucene-queryparser + ${lucene.version} + + com.h2database h2 diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java index 480922cbb53b4..ff20987b079f2 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java @@ -20,20 +20,23 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory; +import org.apache.lucene.store.BaseDirectory; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; /** * A memory-resident {@link Directory} implementation. */ -public class GridLuceneDirectory extends Directory { +public class GridLuceneDirectory extends BaseDirectory { /** */ protected final Map fileMap = new ConcurrentHashMap<>(); @@ -49,14 +52,9 @@ public class GridLuceneDirectory extends Directory { * @param mem Memory. */ public GridLuceneDirectory(GridUnsafeMemory mem) { - this.mem = mem; + super(new GridLuceneLockFactory()); - try { - setLockFactory(new GridLuceneLockFactory()); - } - catch (IOException e) { - throw new IllegalStateException(e); - } + this.mem = mem; } /** {@inheritDoc} */ @@ -75,28 +73,16 @@ public GridLuceneDirectory(GridUnsafeMemory mem) { } /** {@inheritDoc} */ - @Override public final boolean fileExists(String name) { - ensureOpen(); - - return fileMap.containsKey(name); - } - - /** {@inheritDoc} */ - @Override public final long fileModified(String name) { + @Override public void renameFile(String source, String dest) throws IOException { ensureOpen(); - throw new IllegalStateException(name); - } + GridLuceneFile file = fileMap.get(source); - /** - * Set the modified time of an existing file to now. - * - * @throws IOException if the file does not exist - */ - @Override public void touchFile(String name) throws IOException { - ensureOpen(); + if (file == null) + throw new FileNotFoundException(source); - throw new IllegalStateException(name); + fileMap.put(dest, file); + fileMap.remove(source); } /** {@inheritDoc} */ @@ -137,7 +123,7 @@ private void doDeleteFile(String name) throws IOException { } /** {@inheritDoc} */ - @Override public IndexOutput createOutput(String name) throws IOException { + @Override public IndexOutput createOutput(final String name, final IOContext context) throws IOException { ensureOpen(); GridLuceneFile file = newRAMFile(); @@ -155,6 +141,11 @@ private void doDeleteFile(String name) throws IOException { return new GridLuceneOutputStream(file); } + /** {@inheritDoc} */ + @Override public void sync(final Collection names) throws IOException { + // Noop. No fsync needed as all data is in-memory. + } + /** * Returns a new {@link GridLuceneFile} for storing data. This method can be * overridden to return different {@link GridLuceneFile} impls, that e.g. override. @@ -166,7 +157,7 @@ protected GridLuceneFile newRAMFile() { } /** {@inheritDoc} */ - @Override public IndexInput openInput(String name) throws IOException { + @Override public IndexInput openInput(final String name, final IOContext context) throws IOException { ensureOpen(); GridLuceneFile file = fileMap.get(name); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java index 93ebc7111fdc2..eed5ee4e36ae8 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java @@ -36,20 +36,26 @@ import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.spi.indexing.IndexingQueryFilter; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.DateTools; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; -import org.apache.lucene.queryParser.MultiFieldQueryParser; -import org.apache.lucene.search.Filter; +import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TermRangeFilter; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.util.Version; +import org.apache.lucene.util.BytesRef; import org.h2.util.JdbcUtils; import org.jetbrains.annotations.Nullable; @@ -108,8 +114,8 @@ public GridLuceneIndex(GridKernalContext ctx, @Nullable GridUnsafeMemory mem, dir = new GridLuceneDirectory(mem == null ? new GridUnsafeMemory(0) : mem); try { - writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_30, new StandardAnalyzer( - Version.LUCENE_30))); + writer = new IndexWriter(dir, + new IndexWriterConfig(new StandardAnalyzer())); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -163,7 +169,7 @@ public void store(CacheObject k, CacheObject v, GridCacheVersion ver, long expir boolean stringsFound = false; if (type.valueTextIndex() || type.valueClass() == String.class) { - doc.add(new Field(VAL_STR_FIELD_NAME, val.toString(), Field.Store.YES, Field.Index.ANALYZED)); + doc.add(new TextField(VAL_STR_FIELD_NAME, val.toString(), Field.Store.YES)); stringsFound = true; } @@ -172,32 +178,34 @@ public void store(CacheObject k, CacheObject v, GridCacheVersion ver, long expir Object fieldVal = type.value(idxdFields[i], key, val); if (fieldVal != null) { - doc.add(new Field(idxdFields[i], fieldVal.toString(), Field.Store.YES, Field.Index.ANALYZED)); + doc.add(new TextField(idxdFields[i], fieldVal.toString(), Field.Store.YES)); stringsFound = true; } } - String keyStr = org.apache.commons.codec.binary.Base64.encodeBase64String(k.valueBytes(coctx)); + BytesRef keyByteRef = new BytesRef(k.valueBytes(coctx)); try { - // Delete first to avoid duplicates. - writer.deleteDocuments(new Term(KEY_FIELD_NAME, keyStr)); + final Term term = new Term(KEY_FIELD_NAME, keyByteRef); + + if (!stringsFound) { + writer.deleteDocuments(term); - if (!stringsFound) return; // We did not find any strings to be indexed, will not store data at all. + } - doc.add(new Field(KEY_FIELD_NAME, keyStr, Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new StringField(KEY_FIELD_NAME, keyByteRef, Field.Store.YES)); if (type.valueClass() != String.class) - doc.add(new Field(VAL_FIELD_NAME, v.valueBytes(coctx))); + doc.add(new StoredField(VAL_FIELD_NAME, v.valueBytes(coctx))); - doc.add(new Field(VER_FIELD_NAME, ver.toString().getBytes())); + doc.add(new StoredField(VER_FIELD_NAME, ver.toString().getBytes())); - doc.add(new Field(EXPIRATION_TIME_FIELD_NAME, DateTools.timeToString(expires, - DateTools.Resolution.MILLISECOND), Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new LongField(EXPIRATION_TIME_FIELD_NAME, expires, Field.Store.YES)); - writer.addDocument(doc); + // Next implies remove than add atomically operation. + writer.updateDocument(term, doc); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -216,7 +224,7 @@ public void store(CacheObject k, CacheObject v, GridCacheVersion ver, long expir public void remove(CacheObject key) throws IgniteCheckedException { try { writer.deleteDocuments(new Term(KEY_FIELD_NAME, - org.apache.commons.codec.binary.Base64.encodeBase64String(key.valueBytes(objectContext())))); + new BytesRef(key.valueBytes(objectContext())))); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -247,7 +255,8 @@ public GridCloseableIterator> query(String qry, updateCntr.addAndGet(-updates); } - reader = IndexReader.open(writer, true); + //We can cache reader\searcher and change this to 'openIfChanged' + reader = DirectoryReader.open(writer, true); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -255,17 +264,22 @@ public GridCloseableIterator> query(String qry, IndexSearcher searcher = new IndexSearcher(reader); - MultiFieldQueryParser parser = new MultiFieldQueryParser(Version.LUCENE_30, idxdFields, + MultiFieldQueryParser parser = new MultiFieldQueryParser(idxdFields, writer.getAnalyzer()); - // Filter expired items. - Filter f = new TermRangeFilter(EXPIRATION_TIME_FIELD_NAME, DateTools.timeToString(U.currentTimeMillis(), - DateTools.Resolution.MILLISECOND), null, false, false); - TopDocs docs; try { - docs = searcher.search(parser.parse(qry), f, Integer.MAX_VALUE); + // Filter expired items. + Query filter = NumericRangeQuery.newLongRange(EXPIRATION_TIME_FIELD_NAME, U.currentTimeMillis(), + null, false, false); + + BooleanQuery query = new BooleanQuery.Builder() + .add(parser.parse(qry), BooleanClause.Occur.MUST) + .add(filter, BooleanClause.Occur.FILTER) + .build(); + + docs = searcher.search(query, Integer.MAX_VALUE); } catch (Exception e) { throw new IgniteCheckedException(e); @@ -342,7 +356,7 @@ private It(IndexReader reader, IndexSearcher searcher, ScoreDoc[] docs, IgniteBi * @return {@code True} if key passes filter. */ private boolean filter(K key, V val) { - return filters == null || filters.apply(key, val) ; + return filters == null || filters.apply(key, val); } /** @@ -383,11 +397,11 @@ private void findNext() throws IgniteCheckedException { if (ctx != null && ctx.deploy().enabled()) ldr = ctx.cache().internalCache(cacheName).context().deploy().globalLoader(); - K k = unmarshall(org.apache.commons.codec.binary.Base64.decodeBase64(doc.get(KEY_FIELD_NAME)), ldr); + K k = unmarshall(doc.getBinaryValue(KEY_FIELD_NAME).bytes, ldr); V v = type.valueClass() == String.class ? (V)doc.get(VAL_STR_FIELD_NAME) : - this.unmarshall(doc.getBinaryValue(VAL_FIELD_NAME), ldr); + this.unmarshall(doc.getBinaryValue(VAL_FIELD_NAME).bytes, ldr); assert v != null; @@ -416,7 +430,6 @@ private void findNext() throws IgniteCheckedException { /** {@inheritDoc} */ @Override protected void onClose() throws IgniteCheckedException { - U.closeQuiet(searcher); U.closeQuiet(reader); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java index eda97f36f487b..4820af1b0967d 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; import static org.apache.ignite.internal.processors.query.h2.opt.GridLuceneOutputStream.BUFFER_SIZE; @@ -61,11 +60,23 @@ public class GridLuceneInputStream extends IndexInput { * @throws IOException If failed. */ public GridLuceneInputStream(String name, GridLuceneFile f) throws IOException { + this(name, f, f.getLength()); + } + + /** + * Constructor. + * + * @param name Name. + * @param f File. + * @param length inputStream length. + * @throws IOException If failed. + */ + public GridLuceneInputStream(String name, GridLuceneFile f, final long length) throws IOException { super("RAMInputStream(name=" + name + ")"); file = f; - length = file.getLength(); + this.length = length; if (length / BUFFER_SIZE >= Integer.MAX_VALUE) throw new IOException("RAMInputStream too large length=" + length + ": " + name); @@ -149,39 +160,14 @@ private void switchCurrentBuffer(boolean enforceEOF) throws IOException { } /** {@inheritDoc} */ - @Override public void copyBytes(IndexOutput out, long numBytes) throws IOException { - assert numBytes >= 0 : "numBytes=" + numBytes; - - GridLuceneOutputStream gridOut = out instanceof GridLuceneOutputStream ? (GridLuceneOutputStream)out : null; - - long left = numBytes; - - while (left > 0) { - if (bufPosition == bufLength) { - ++currBufIdx; - - switchCurrentBuffer(true); - } - - final int bytesInBuf = bufLength - bufPosition; - final int toCp = (int)(bytesInBuf < left ? bytesInBuf : left); - - if (gridOut != null) - gridOut.writeBytes(currBuf + bufPosition, toCp); - else { - byte[] buff = new byte[toCp]; - - mem.readBytes(currBuf + bufPosition, buff); - - out.writeBytes(buff, toCp); - } - - bufPosition += toCp; + @Override + public IndexInput slice(final String sliceDescription, final long offset, final long length) throws IOException { + if (offset < 0 || length < 0 || offset + length > this.length) + throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: " + this); - left -= toCp; - } + final String newResourceDescription = (sliceDescription == null) ? toString() : (toString() + " [slice=" + sliceDescription + "]"); - assert left == 0 : "Insufficient bytes to copy: numBytes=" + numBytes + " copied=" + (numBytes - left); + return new SlicedInputStream(newResourceDescription, offset, length); } /** @@ -226,4 +212,46 @@ void readBytes(long ptr, int len) throws IOException { bufPosition = (int)(pos % BUFFER_SIZE); } + + /** */ + private class SlicedInputStream extends GridLuceneInputStream { + /** */ + private final long offset; + + /** */ + public SlicedInputStream(String newResourceDescription, long offset, long length) throws IOException { + super(newResourceDescription, GridLuceneInputStream.this.file, offset + length); + + this.offset = offset; + + seek(0L); + } + + /** {@inheritDoc} */ + @Override + public void seek(long pos) throws IOException { + if (pos < 0L) { + throw new IllegalArgumentException("Seeking to negative position: " + this); + } + super.seek(pos + offset); + } + + /** {@inheritDoc} */ + @Override + public long getFilePointer() { + return super.getFilePointer() - offset; + } + + /** {@inheritDoc} */ + @Override + public long length() { + return super.length() - offset; + } + + /** {@inheritDoc} */ + @Override + public IndexInput slice(String sliceDescription, long ofs, long len) throws IOException { + return super.slice(sliceDescription, offset + ofs, len); + } + } } \ No newline at end of file diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneLockFactory.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneLockFactory.java index c2ee76825367d..e14a4080d801c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneLockFactory.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneLockFactory.java @@ -19,8 +19,11 @@ import java.io.IOException; import org.apache.ignite.internal.util.GridConcurrentHashSet; +import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.LockObtainFailedException; /** * Lucene {@link LockFactory} implementation. @@ -31,13 +34,11 @@ public class GridLuceneLockFactory extends LockFactory { private final GridConcurrentHashSet locks = new GridConcurrentHashSet<>(); /** {@inheritDoc} */ - @Override public Lock makeLock(String lockName) { - return new LockImpl(lockName); - } - - /** {@inheritDoc} */ - @Override public void clearLock(String lockName) throws IOException { - locks.remove(lockName); + @Override public Lock obtainLock(Directory dir, String lockName) throws IOException { + if (locks.add(lockName)) + return new LockImpl(lockName); + else + throw new LockObtainFailedException("lock instance already obtained: (dir=" + dir + ", lockName=" + lockName + ")"); } /** @@ -47,6 +48,9 @@ private class LockImpl extends Lock { /** */ private final String lockName; + /** */ + private volatile boolean closed; + /** * @param lockName Lock name. */ @@ -55,18 +59,33 @@ private LockImpl(String lockName) { } /** {@inheritDoc} */ - @Override public boolean obtain() throws IOException { - return locks.add(lockName); + @Override public void ensureValid() throws IOException { + if (closed) + throw new AlreadyClosedException("Lock instance already released: " + this); + + // check we are still in the locks map (some debugger or something crazy didn't remove us) + if (!locks.contains(lockName)) + throw new AlreadyClosedException("Lock instance was invalidated from map: " + this); } /** {@inheritDoc} */ - @Override public void release() throws IOException { - locks.remove(lockName); + @Override public void close() throws IOException { + if (closed) + return; + + try { + if (!locks.remove(lockName)) + throw new AlreadyClosedException("Lock was already released: " + this); + } + finally { + closed = true; + } } /** {@inheritDoc} */ - @Override public boolean isLocked() throws IOException { - return locks.contains(lockName); + @Override + public String toString() { + return super.toString() + ": " + lockName; } } } \ No newline at end of file diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java index 8d3d79cd27770..caea226022dc6 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java @@ -18,7 +18,10 @@ package org.apache.ignite.internal.processors.query.h2.opt; import java.io.IOException; +import java.util.zip.CRC32; +import java.util.zip.Checksum; import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory; +import org.apache.lucene.store.BufferedChecksum; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IndexOutput; @@ -50,12 +53,17 @@ public class GridLuceneOutputStream extends IndexOutput { /** */ private final GridUnsafeMemory mem; + /** */ + private final Checksum crc; + /** * Constructor. * * @param f File. */ public GridLuceneOutputStream(GridLuceneFile f) { + super("RAMOutputStream(name=\"noname\")"); + file = f; mem = f.getDirectory().memory(); @@ -64,6 +72,8 @@ public GridLuceneOutputStream(GridLuceneFile f) { // first needed buffer lazily currBufIdx = -1; currBuf = 0; + + crc = new BufferedChecksum(new CRC32()); } /** @@ -77,6 +87,7 @@ public void reset() { bufLength = 0; file.setLength(0); + crc.reset(); } /** {@inheritDoc} */ @@ -85,23 +96,8 @@ public void reset() { } /** {@inheritDoc} */ - @Override public void seek(long pos) throws IOException { - // set the file length in case we seek back - // and flush() has not been called yet - setFileLength(); - - if (pos < bufStart || pos >= bufStart + bufLength) { - currBufIdx = (int)(pos / BUFFER_SIZE); - - switchCurrentBuffer(); - } - - bufPosition = (int)(pos % BUFFER_SIZE); - } - - /** {@inheritDoc} */ - @Override public long length() { - return file.getLength(); + @Override public long getChecksum() throws IOException { + return crc.getValue(); } /** {@inheritDoc} */ @@ -112,6 +108,8 @@ public void reset() { switchCurrentBuffer(); } + crc.update(b); + mem.writeByte(currBuf + bufPosition++, b); } @@ -119,6 +117,8 @@ public void reset() { @Override public void writeBytes(byte[] b, int offset, int len) throws IOException { assert b != null; + crc.update(b, offset, len); + while (len > 0) { if (bufPosition == bufLength) { currBufIdx++; @@ -159,8 +159,8 @@ private void setFileLength() { file.setLength(pointer); } - /** {@inheritDoc} */ - @Override public void flush() throws IOException { + /** Forces any buffered output to be written. */ + private void flush() throws IOException { setFileLength(); } @@ -169,15 +169,6 @@ private void setFileLength() { return currBufIdx < 0 ? 0 : bufStart + bufPosition; } - /** - * Returns byte usage of all buffers. - * - * @return Bytes used. - */ - public long sizeInBytes() { - return (long)file.numBuffers() * (long)BUFFER_SIZE; - } - /** {@inheritDoc} */ @Override public void copyBytes(DataInput input, long numBytes) throws IOException { assert numBytes >= 0 : "numBytes=" + numBytes; @@ -210,29 +201,4 @@ public long sizeInBytes() { bufPosition += toCp; } } - - /** - * For direct usage by {@link GridLuceneInputStream}. - * - * @param ptr Pointer. - * @param len Length. - * @throws IOException If failed. - */ - void writeBytes(long ptr, int len) throws IOException { - while (len > 0) { - if (bufPosition == bufLength) { - currBufIdx++; - switchCurrentBuffer(); - } - - int remainInBuf = BUFFER_SIZE - bufPosition; - int bytesToCp = len < remainInBuf ? len : remainInBuf; - - mem.copyMemory(ptr, currBuf + bufPosition, bytesToCp); - - ptr += bytesToCp; - len -= bytesToCp; - bufPosition += bytesToCp; - } - } } \ No newline at end of file diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java new file mode 100644 index 0000000000000..747038d570b29 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import javax.cache.Cache; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cache.query.Query; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.TextQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.cache.query.annotations.QueryTextField; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * FullTest queries left test. + */ +public class GridCacheFullTextQuerySelfTest extends GridCommonAbstractTest { + /** Cache size. */ + private static final int MAX_ITEM_COUNT = 100; + + /** Cache name */ + private static final String PERSON_CACHE = "Person"; + + /** */ + private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + + disco.setIpFinder(ipFinder); + + cfg.setDiscoverySpi(disco); + + cfg.setIncludeEventTypes(); + + cfg.setConnectorConfiguration(null); + + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName(PERSON_CACHE) + .setCacheMode(PARTITIONED) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setWriteSynchronizationMode(FULL_SYNC) + .setBackups(0) + .setIndexedTypes(Integer.class, Person.class); + + cfg.setCacheConfiguration(cacheCfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrids(2); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testLocalTextQueryWithKeepBinary() throws Exception { + checkTextQuery(true, true); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testLocalTextQuery() throws Exception { + checkTextQuery(true, false); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testTextQueryWithKeepBinary() throws Exception { + checkTextQuery(false, true); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testTextQuery() throws Exception { + checkTextQuery(false, true); + } + + /** + * @param loc local query flag. + * @param keepBinary keep binary flag. + */ + private void checkTextQuery(boolean loc, boolean keepBinary) throws Exception { + final IgniteEx ignite = grid(0); + + // 1. Populate cache with data, calculating expected count in parallel. + Set exp = populateCache(ignite, loc, MAX_ITEM_COUNT, new IgnitePredicate() { + @Override + public boolean apply(Integer x) { + return String.valueOf(x).startsWith("1"); + } + }); + + // 2. Validate results. + TextQuery qry = new TextQuery<>(Person.class, "1*").setLocal(loc); + + validateQueryResults(ignite, qry, exp, keepBinary); + + clearCache(ignite); + } + + /** + * Clear cache with check. + */ + private static void clearCache(IgniteEx ignite) { + IgniteCache cache = ignite.cache(PERSON_CACHE); + + cache.clear(); + + List all = cache.query(new TextQuery<>(Person.class, "1*")).getAll(); + + assertTrue(all.isEmpty()); + } + + /** + * Fill cache. + * + * @throws IgniteCheckedException if failed. + */ + private static Set populateCache(IgniteEx ignite, boolean loc, int cnt, + IgnitePredicate expectedEntryFilter) throws IgniteCheckedException { + IgniteInternalCache cache = ignite.cachex(PERSON_CACHE); + + assertNotNull(cache); + + Random rand = new Random(); + + HashSet exp = new HashSet<>(); + + Affinity aff = cache.affinity(); + + ClusterNode localNode = cache.context().localNode(); + + for (int i = 0; i < cnt; i++) { + int val = rand.nextInt(cnt); + + cache.put(val, new Person(String.valueOf(val), val)); + + if (expectedEntryFilter.apply(val) && (!loc || aff.isPrimary(localNode, val))) + exp.add(val); + } + + return exp; + } + + /** + * Check query results. + * + * @throws IgniteCheckedException if failed. + */ + private static void validateQueryResults(IgniteEx ignite, Query qry, Set exp, + boolean keepBinary) throws IgniteCheckedException { + IgniteCache cache = ignite.cache(PERSON_CACHE); + + if (keepBinary) { + IgniteCache cache0 = cache.withKeepBinary(); + + try (QueryCursor> cursor = cache0.query(qry)) { + Set exp0 = new HashSet<>(exp); + + List> all = new ArrayList<>(); + + for (Cache.Entry entry : cursor.getAll()) { + all.add(entry); + + assertEquals(entry.getKey().toString(), entry.getValue().field("name")); + + assertEquals(entry.getKey(), entry.getValue().field("age")); + + exp0.remove(entry.getKey()); + } + + checkForMissedKeys(ignite, exp0, all); + } + + try (QueryCursor> cursor = cache0.query(qry)) { + Set exp0 = new HashSet<>(exp); + + List> all = new ArrayList<>(); + + for (Cache.Entry entry : cursor.getAll()) { + all.add(entry); + + assertEquals(entry.getKey().toString(), entry.getValue().field("name")); + + assertEquals(entry.getKey(), entry.getValue().field("age")); + + exp0.remove(entry.getKey()); + } + + checkForMissedKeys(ignite, exp0, all); + } + } + else { + try (QueryCursor> cursor = cache.query(qry)) { + Set exp0 = new HashSet<>(exp); + + List> all = new ArrayList<>(); + + for (Cache.Entry entry : cursor.getAll()) { + all.add(entry); + + assertEquals(entry.getKey().toString(), entry.getValue().name); + + assertEquals(entry.getKey(), Integer.valueOf(entry.getValue().age)); + + exp0.remove(entry.getKey()); + } + + checkForMissedKeys(ignite, exp0, all); + } + + try (QueryCursor> cursor = cache.query(qry)) { + Set exp0 = new HashSet<>(exp); + + List> all = new ArrayList<>(); + + for (Cache.Entry entry : cursor.getAll()) { + all.add(entry); + + assertEquals(entry.getKey().toString(), entry.getValue().name); + + assertEquals(entry.getKey().intValue(), entry.getValue().age); + + exp0.remove(entry.getKey()); + } + + checkForMissedKeys(ignite, exp0, all); + } + } + } + + /** + * Check if there is missed keys. + * + * @throws IgniteCheckedException if failed. + */ + private static void checkForMissedKeys(IgniteEx ignite, Collection exp, + List> all) throws IgniteCheckedException { + if (exp.size() == 0) + return; + + IgniteInternalCache cache = ignite.cachex(PERSON_CACHE); + + assertNotNull(cache); + + StringBuilder sb = new StringBuilder(); + + Affinity aff = cache.affinity(); + + for (Integer key : exp) { + Integer part = aff.partition(key); + + sb.append( + String.format("Query did not return expected key '%d' (exists: %s), partition '%d', partition nodes: ", + key, cache.get(key) != null, part)); + + Collection partNodes = aff.mapPartitionToPrimaryAndBackups(part); + + for (ClusterNode node : partNodes) + sb.append(node).append(" "); + + sb.append(";\n"); + } + + sb.append("Returned keys: "); + + for (Cache.Entry e : all) + sb.append(e.getKey()).append(" "); + + sb.append(";\n"); + + fail(sb.toString()); + } + + /** + * Test model class. + */ + public static class Person implements Serializable { + /** */ + @QueryTextField + String name; + + /** */ + @QuerySqlField(index = true) + int age; + + /** */ + @QuerySqlField final Date birthday; + + /** + * Constructor + */ + public Person(String name, int age) { + this.name = name; + this.age = age % 2000; + + Calendar cal = Calendar.getInstance(); + cal.add(Calendar.YEAR, -age); + + birthday = cal.getTime(); + } + } +} \ No newline at end of file diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java index 2a75bd366b82e..ba0324fa25b77 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java @@ -22,6 +22,7 @@ import javax.cache.Cache; import org.apache.ignite.Ignite; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheRebalanceMode; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.QueryIndex; import org.apache.ignite.cache.QueryIndexType; @@ -61,6 +62,7 @@ public class IgniteCacheFullTextQueryNodeJoiningSelfTest extends GridCommonAbstr cache.setAtomicityMode(atomicityMode()); cache.setWriteSynchronizationMode(FULL_SYNC); cache.setBackups(1); + cache.setRebalanceMode(CacheRebalanceMode.SYNC); QueryEntity qryEntity = new QueryEntity(); @@ -121,7 +123,7 @@ public void testFullTextQueryNodeJoin() throws Exception { QueryCursor, IndexedEntity>> res = started.cache(DEFAULT_CACHE_NAME) .query(new TextQuery, IndexedEntity>(IndexedEntity.class, "indexed")); - assertEquals(1000, res.getAll().size()); + assertEquals("Failed iteration: " + i, 1000, res.getAll().size()); } } finally { diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java index 012ed29cca8fb..258eed80811c4 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.CacheReplicatedQueryMetricsLocalSelfTest; import org.apache.ignite.internal.processors.cache.CacheSqlQueryValueCopySelfTest; import org.apache.ignite.internal.processors.cache.GridCacheCrossCacheQuerySelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheFullTextQuerySelfTest; import org.apache.ignite.internal.processors.cache.GridCacheQueryIndexDisabledSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheQueryIndexingDisabledSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheQueryInternalKeysSelfTest; @@ -266,13 +267,13 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteCacheAtomicNearEnabledFieldsQuerySelfTest.class); suite.addTestSuite(IgniteCachePartitionedFieldsQueryP2PEnabledSelfTest.class); suite.addTestSuite(IgniteCacheFieldsQueryNoDataSelfTest.class); - suite.addTestSuite(GridCacheQueryIndexingDisabledSelfTest.class); - suite.addTestSuite(GridOrderedMessageCancelSelfTest.class); - suite.addTestSuite(CacheQueryEvictDataLostTest.class); + // Full text queries. + suite.addTestSuite(GridCacheFullTextQuerySelfTest.class); + // Ignite cache and H2 comparison. suite.addTestSuite(BaseH2CompareQueryTest.class); suite.addTestSuite(H2CompareBigQueryTest.class); diff --git a/parent/pom.xml b/parent/pom.xml index a481657c6119e..7443753a18d01 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -88,8 +88,8 @@ r938 0.10.0.1 4.0.2 - 3.5.0_1 - 3.5.0 + 5.5.2_1 + 5.5.2 2.0.8_6 5.0.0 5.0.0 From 707c454ad9c3b4132e2d0a20d15dc1eb2ed295b0 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Wed, 12 Jul 2017 10:53:46 +0300 Subject: [PATCH 142/155] Corrected fix for REST processor wrt authentication --- .../processors/rest/GridRestProcessor.java | 45 +++++-------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java index fd5583d3edbe4..98428836ba83b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java @@ -243,23 +243,21 @@ private IgniteInternalFuture handleRequest(final GridRestReque SecurityContext secCtx0 = ses.secCtx; - if (ctx.state().publicApiActiveState() || !isClusterActivateTaskRequest(req)) { - try { - if (secCtx0 == null) - ses.secCtx = secCtx0 = authenticate(req); + try { + if (secCtx0 == null) + ses.secCtx = secCtx0 = authenticate(req); - authorize(req, secCtx0); - } - catch (SecurityException e) { - assert secCtx0 != null; + authorize(req, secCtx0); + } + catch (SecurityException e) { + assert secCtx0 != null; - GridRestResponse res = new GridRestResponse(STATUS_SECURITY_CHECK_FAILED, e.getMessage()); + GridRestResponse res = new GridRestResponse(STATUS_SECURITY_CHECK_FAILED, e.getMessage()); - return new GridFinishedFuture<>(res); - } - catch (IgniteCheckedException e) { - return new GridFinishedFuture<>(new GridRestResponse(STATUS_AUTH_FAILED, e.getMessage())); - } + return new GridFinishedFuture<>(res); + } + catch (IgniteCheckedException e) { + return new GridFinishedFuture<>(new GridRestResponse(STATUS_AUTH_FAILED, e.getMessage())); } } @@ -320,25 +318,6 @@ private IgniteInternalFuture handleRequest(final GridRestReque }); } - /** - * We skip authentication for activate cluster request. - * It's necessary workaround to make possible cluster activation through Visor, - * as security checks require working caches. - * - * @param req Request. - */ - private boolean isClusterActivateTaskRequest(GridRestRequest req) { - if (req instanceof GridRestTaskRequest) { - GridRestTaskRequest taskReq = (GridRestTaskRequest)req; - - if (VisorGatewayTask.class.getCanonicalName().equals(taskReq.taskName()) && - taskReq.params().contains(VisorChangeGridActiveStateTask.class.getCanonicalName())) - return true; - } - - return false; - } - /** * @param req Request. * @return Not null session. From f3828261b30c12d5aa181914033afe46c787f87e Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 12 Jul 2017 14:57:50 +0700 Subject: [PATCH 143/155] IGNITE-5639 Added duration for empty result set. --- modules/web-console/frontend/views/sql/sql.tpl.pug | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug index f215c432a7583..0b8011aca2f5c 100644 --- a/modules/web-console/frontend/views/sql/sql.tpl.pug +++ b/modules/web-console/frontend/views/sql/sql.tpl.pug @@ -196,7 +196,7 @@ mixin paragraph-scan .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()') .error(ng-switch-when='error') Error: {{paragraph.error.message}} - .empty(ng-switch-when='empty') Result set is empty + .empty(ng-switch-when='empty') Result set is empty. Duration: #[b {{paragraph.duration | duration}}] .table(ng-switch-when='table') +table-result-heading-scan +table-result-body @@ -247,7 +247,7 @@ mixin paragraph-query label Error: {{paragraph.error.message}} br a(ng-show='paragraph.resultType() === "error"' ng-click='showStackTrace(paragraph)') Show more - .empty(ng-switch-when='empty') Result set is empty + .empty(ng-switch-when='empty') Result set is empty. Duration: #[b {{paragraph.duration | duration}}] .table(ng-switch-when='table') +table-result-heading-query +table-result-body From 5859b192ba28d53e1bccb01ce3005821e26b5347 Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 12 Jul 2017 12:46:42 +0300 Subject: [PATCH 144/155] AI 2.1 release notes. --- RELEASE_NOTES.txt | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index b64aced13e588..92286cf422e49 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,6 +1,39 @@ Apache Ignite Release Notes =========================== +Apache Ignite In-Memory Data Fabric 2.1 +--------------------------------------- +Ignite: +* Persistent cache store +* Added IgniteFuture.listenAsync() and IgniteFuture.chainAsync() mehtods +* Deprecated IgniteConfiguration.marshaller +* Updated Lucene dependency to version 5.5.2 +* Machine learning: implemented K-means clusterization algorithm optimized for distributed storages +* SQL: CREATE TABLE and DROP TABLE commands support +* SQL: New thin JDBC driver +* SQL: Improved performance of certain queries, when affinity node can be calculated in advance +* SQL: Fixed return type of AVG() function +* SQL: BLOB type support added to thick JDBC driver +* SQL: Improved LocalDate, LocalTime and LocalDateTime support for Java 8 +* SQL: Added FieldsQueryCursor interface to get fields metadata for SqlFieldsQuery +* ODBC: Implemented DML statement batching +* Massive performance and stability improvements + +Ignite.NET: +* Automatic remote assembly loading +* NuGet-based standalone node deployment +* Added conditional data removeal via LINQ DeleteAll +* Added TimestampAttribute to control DateTime serialization mode +* Added local collections joins support to LINQ. + +Ignite CPP: +* Added Compute::Call and Compute::Broadcast methods + +Web Console: +* Implemented support for UNIQUE indexes for key fields on import model from RDBMS +* Added option to show full stack trace on Queries screen +* Added PK alias generation on Models screen. + Apache Ignite In-Memory Data Fabric 2.0 --------------------------------------- Ignite: From 8afdc7baae73ecba67e0735baa97d03f2c4fc715 Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 12 Jul 2017 13:51:43 +0300 Subject: [PATCH 145/155] Removed CacheBinaryAutoStoreExample and relevant bean "h2-example-db" from example-default.xml because example duplicated existing CacheAutoStoreExample. --- examples/config/example-default.xml | 7 - .../auto/CacheBinaryAutoStoreExample.java | 170 ------------------ .../datagrid/store/auto/package-info.java | 22 --- 3 files changed, 199 deletions(-) delete mode 100644 examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/CacheBinaryAutoStoreExample.java delete mode 100644 examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/package-info.java diff --git a/examples/config/example-default.xml b/examples/config/example-default.xml index 6bd6f16fa2957..e6c359d90f82f 100644 --- a/examples/config/example-default.xml +++ b/examples/config/example-default.xml @@ -28,13 +28,6 @@ http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd"> - - - - - - - diff --git a/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/CacheBinaryAutoStoreExample.java b/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/CacheBinaryAutoStoreExample.java deleted file mode 100644 index c80c87b3b39b6..0000000000000 --- a/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/CacheBinaryAutoStoreExample.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.examples.binary.datagrid.store.auto; - -import java.sql.Types; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.Ignition; -import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore; -import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory; -import org.apache.ignite.cache.store.jdbc.JdbcType; -import org.apache.ignite.cache.store.jdbc.JdbcTypeField; -import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.examples.ExampleNodeStartup; -import org.apache.ignite.examples.model.Person; -import org.apache.ignite.examples.util.DbH2ServerStartup; -import org.apache.ignite.transactions.Transaction; - -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; - -/** - * Demonstrates usage of cache with underlying persistent store configured. - *

- * This example uses {@link CacheJdbcPojoStore} as a persistent store. - *

- * To start the example, you should: - *

    - *
  • Start H2 database TCP server using {@link DbH2ServerStartup}.
    • - *
  • Start a few nodes using {@link ExampleNodeStartup} or by starting remote nodes as specified below.
    • - *
  • Start example using {@link CacheBinaryAutoStoreExample}.
    • - *
- *

- * Remote nodes should always be started with special configuration file which - * contains H2 data source bean descriptor: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}. - *

- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will - * start node with {@code examples/config/example-ignite.xml} configuration. - */ -public class CacheBinaryAutoStoreExample { - /** Global person ID to use across entire example. */ - private static final Long id = 25121642L; - - /** Cache name. */ - public static final String CACHE_NAME = CacheBinaryAutoStoreExample.class.getSimpleName(); - - /** - * Configure cache with store. - */ - private static CacheConfiguration cacheConfiguration() { - CacheJdbcPojoStoreFactory storeFactory = new CacheJdbcPojoStoreFactory<>(); - - storeFactory.setDataSourceBean("h2-example-db"); - storeFactory.setDialect(new H2Dialect()); - - JdbcType jdbcType = new JdbcType(); - - jdbcType.setCacheName(CACHE_NAME); - jdbcType.setDatabaseSchema("PUBLIC"); - jdbcType.setDatabaseTable("PERSON"); - - jdbcType.setKeyType("java.lang.Long"); - jdbcType.setKeyFields(new JdbcTypeField(Types.BIGINT, "ID", Long.class, "id")); - - jdbcType.setValueType("org.apache.ignite.examples.model.Person"); - jdbcType.setValueFields( - new JdbcTypeField(Types.BIGINT, "ID", Long.class, "id"), - new JdbcTypeField(Types.VARCHAR, "FIRST_NAME", String.class, "firstName"), - new JdbcTypeField(Types.VARCHAR, "LAST_NAME", String.class, "lastName") - ); - - storeFactory.setTypes(jdbcType); - - CacheConfiguration cfg = new CacheConfiguration<>(CACHE_NAME); - - cfg.setCacheStoreFactory(storeFactory); - - // Set atomicity as transaction, since we are showing transactions in the example. - cfg.setAtomicityMode(TRANSACTIONAL); - - // This option will allow to start remote nodes without having user classes in classpath. - cfg.setStoreKeepBinary(true); - - cfg.setReadThrough(true); - cfg.setWriteThrough(true); - - return cfg; - } - - /** - * Executes example. - * - * @param args Command line arguments, none required. - * @throws Exception If example execution failed. - */ - public static void main(String[] args) throws Exception { - // To start ignite with desired configuration uncomment the appropriate line. - try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) { - System.out.println(); - System.out.println(">>> Populate database with data..."); - DbH2ServerStartup.populateDatabase(); - - System.out.println(); - System.out.println(">>> Cache auto store example started..."); - - try (IgniteCache cache = ignite.getOrCreateCache(cacheConfiguration())) { - try (Transaction tx = ignite.transactions().txStart()) { - Person val = cache.get(id); - - System.out.println(">>> Read value: " + val); - - val = cache.getAndPut(id, new Person(id, 1L, "Isaac", "Newton", 100.10, "English physicist and mathematician")); - - System.out.println(">>> Overwrote old value: " + val); - - val = cache.get(id); - - System.out.println(">>> Read value: " + val); - - System.out.println(">>> Update salary in transaction..."); - - val.salary *= 2; - - cache.put(id, val); - - tx.commit(); - } - - System.out.println(">>> Read value after commit: " + cache.get(id)); - - cache.clear(); - - System.out.println(">>> ------------------------------------------"); - System.out.println(">>> Load data to cache from DB with custom SQL..."); - - // Load cache on all data nodes with custom SQL statement. - cache.loadCache(null, "java.lang.Long", "select * from PERSON where id <= 3"); - - System.out.println(">>> Loaded cache entries: " + cache.size()); - - cache.clear(); - - // Load cache on all data nodes with default SQL statement. - System.out.println(">>> Load ALL data to cache from DB..."); - cache.loadCache(null); - - System.out.println(">>> Loaded cache entries: " + cache.size()); - } - finally { - // Distributed cache could be removed from cluster only by #destroyCache() call. - ignite.destroyCache(CACHE_NAME); - } - } - } -} diff --git a/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/package-info.java b/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/package-info.java deleted file mode 100644 index 153f2109c9571..0000000000000 --- a/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * - * Contains automatic JDBC store example. - */ -package org.apache.ignite.examples.binary.datagrid.store.auto; From c6ee085b8a1321ce7fa15f8adf74fa7a01f7a445 Mon Sep 17 00:00:00 2001 From: Dmitriy Govorukhin Date: Wed, 12 Jul 2017 14:22:03 +0300 Subject: [PATCH 146/155] Fixed page acquire during checkpoint --- .../persistence/pagemem/PageMemoryImpl.java | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index e4428a2ab3b41..47381d72cc5d1 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -803,9 +803,14 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck int tag; + boolean tmpBuffer = false; + seg.readLock().lock(); try { + if (!isInCheckpoint(fullId)) + return null; + tag = seg.partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId())); relPtr = seg.loadedPages.get( @@ -826,6 +831,8 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck // Pin the page until page will not be copied. if (PageHeader.tempBufferPointer(absPtr) == INVALID_REL_PTR) PageHeader.acquirePage(absPtr); + else + tmpBuffer = true; } } finally { @@ -869,7 +876,7 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck } } else - return copyPageForCheckpoint(absPtr, fullId, tmpBuf, tracker) ? tag : null; + return copyPageForCheckpoint(absPtr, fullId, tmpBuf, tmpBuffer, tracker) ? tag : null; } /** @@ -877,19 +884,24 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck * @param fullId Full id. * @param tmpBuf Tmp buffer. */ - private boolean copyPageForCheckpoint(long absPtr, FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) { + private boolean copyPageForCheckpoint( + long absPtr, + FullPageId fullId, + ByteBuffer tmpBuf, + boolean tmpBuffer, + CheckpointMetricsTracker tracker + ) { assert absPtr != 0; + assert PageHeader.isAcquired(absPtr); rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS); try { long tmpRelPtr = PageHeader.tempBufferPointer(absPtr); - if (!clearCheckpoint(fullId)){ - assert tmpRelPtr == INVALID_REL_PTR; + boolean success = clearCheckpoint(fullId); - return false; - } + assert success : "Page was pin when we resolve abs pointer, it can not be evicted"; if (tmpRelPtr != INVALID_REL_PTR){ PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR); @@ -907,6 +919,12 @@ private boolean copyPageForCheckpoint(long absPtr, FullPageId fullId, ByteBuffer // We pinned the page when allocated the temp buffer, release it now. PageHeader.releasePage(absPtr); + + // Need release again because we pin page when resolve abs pointer, + // and page did not have tmp buffer page. + if (!tmpBuffer) + PageHeader.releasePage(absPtr); + } else { copyInBuffer(absPtr, tmpBuf); From 0cb6ac06adddd43ac72c707b29d7216bd4cb711a Mon Sep 17 00:00:00 2001 From: Oleg Ostanin Date: Wed, 12 Jul 2017 15:57:40 +0300 Subject: [PATCH 147/155] IGNITE-5740 - Added transaction load timing benchmark --- .../yardstick/IgniteBenchmarkArguments.java | 33 +++++ .../cache/IgnitePutTxLoadBenchmark.java | 119 ++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxLoadBenchmark.java diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java index 5ec6c54c53551..594fa1f1b5907 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java @@ -208,6 +208,18 @@ public class IgniteBenchmarkArguments { @Parameter(names = {"-ps", "--pageSize"}, description = "Page size") private int pageSize = MemoryConfiguration.DFLT_PAGE_SIZE; + /** */ + @Parameter(names = {"-sl", "--stringLength"}, description = "Test string length") + private int stringLength = 500; + + /** */ + @Parameter(names = {"-wt", "--warningTime"}, description = "Warning time interval for printing log") + private long warningTime = 500; + + /** */ + @Parameter(names = {"-prb", "--printRollBacks"}, description = "Print rollBacks") + private boolean printRollBacks; + /** */ @Parameter(names = {"-prt", "--partitions"}, description = "Number of cache partitions") private int partitions = 10; @@ -506,6 +518,27 @@ public int getPageSize() { return pageSize; } + /** + * @return Test string length. + */ + public int getStringLength() { + return stringLength; + } + + /** + * @return Warning time interval. + */ + public long getWarningTime() { + return warningTime; + } + + /** + * @return Flag for printing rollbacks. + */ + public boolean printRollBacks() { + return printRollBacks; + } + /** * @return Number of partitioned caches. */ diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxLoadBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxLoadBenchmark.java new file mode 100644 index 0000000000000..7ac7c3acf9e9e --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxLoadBenchmark.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Map; +import java.util.Random; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.IgniteTransactions; +import org.apache.ignite.transactions.Transaction; +import org.apache.ignite.transactions.TransactionMetrics; +import org.yardstickframework.BenchmarkConfiguration; +import org.yardstickframework.BenchmarkUtils; + +/** + * Ignite benchmark that performs transactional load put operations. + */ +public class IgnitePutTxLoadBenchmark extends IgniteCacheAbstractBenchmark { + /** */ + private ArrayList> cacheList; + + /** */ + private String val; + + /** */ + private Random random; + + /** {@inheritDoc} */ + @Override public void setUp(BenchmarkConfiguration cfg) throws Exception { + super.setUp(cfg); + + if (!IgniteSystemProperties.getBoolean("SKIP_MAP_CHECK")) + ignite().compute().broadcast(new WaitMapExchangeFinishCallable()); + + cacheList = new ArrayList<>(args.cachesCount()); + + for (int i = 0; i < args.cachesCount(); i++) + cacheList.add(ignite().cache("tx-" + i)); + + val = createVal(args.getStringLength()); + + random = new Random(); + } + + /** {@inheritDoc} */ + @Override public boolean test(Map ctx) throws Exception { + IgniteTransactions transactions = ignite().transactions(); + + long startTime; + long endTime; + + try (Transaction tx = transactions.txStart(args.txConcurrency(), args.txIsolation())) { + ArrayList keyList = new ArrayList<>(args.scaleFactor()); + + for (int i = 0; i < args.scaleFactor(); i++) + keyList.add(random.nextLong()); + + Collections.sort(keyList); + + for (int i = 0; i < args.scaleFactor(); i++){ + IgniteCache curCache = cacheList.get(random.nextInt(cacheList.size())); + curCache.put(keyList.get(i), val); + } + + startTime = System.currentTimeMillis(); + + tx.commit(); + + endTime = System.currentTimeMillis(); + + } + + TransactionMetrics tm = transactions.metrics(); + + if (endTime - startTime > args.getWarningTime()) + BenchmarkUtils.println("Transaction commit time = " + (tm.commitTime() - startTime)); + + if (tm.txRollbacks() > 0 && args.printRollBacks()) + BenchmarkUtils.println("Transaction rollbacks = " + tm.txRollbacks()); + + return true; + } + + /** {@inheritDoc} */ + @Override protected IgniteCache cache() { + return ignite().cache("tx"); + } + + /** + * Creates String val + * @param lgth String length + * @return String for inserting in cache + */ + private String createVal(int lgth){ + StringBuilder sb = new StringBuilder(lgth); + + for(int i = 0; i < lgth; i++) + sb.append('x'); + + return sb.toString(); + } +} From 3787181310597b7a6e633e745ba08209abd038a9 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Wed, 12 Jul 2017 18:28:57 +0300 Subject: [PATCH 148/155] More verbose logging --- .../dht/preloader/GridDhtPartitionsExchangeFuture.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 5760f875f8425..3b6fe91136587 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -1008,11 +1008,19 @@ private void distributedExchange() throws IgniteCheckedException { */ private void tryToPerformLocalSnapshotOperation() { try { + long start = U.currentTimeMillis(); + IgniteInternalFuture fut = cctx.snapshot() .tryStartLocalSnapshotOperation(discoEvt); if (fut != null) fut.get(); + + long end = U.currentTimeMillis(); + + if (log.isInfoEnabled()) + log.info("Snapshot initialization completed [topVer=" + exchangeId().topologyVersion() + + ", time=" + (end - start) + "ms]"); } catch (IgniteCheckedException e) { U.error(log, "Error while starting snapshot operation", e); From 21964fb5f6fb6fee891283332202cbc9ed5ac3f3 Mon Sep 17 00:00:00 2001 From: Dmitry Pavlov Date: Wed, 12 Jul 2017 18:59:10 +0300 Subject: [PATCH 149/155] Optimized snapshot progress tracking --- .../ignite/internal/pagemem/FullPageId.java | 6 +- .../ignite/internal/pagemem/PageIdUtils.java | 14 +- .../pagemem/store/IgnitePageStoreManager.java | 1 + .../internal/pagemem/store/PageStore.java | 2 + .../MetaPageUpdateLastAllocatedIndex.java | 2 +- .../persistence/DbCheckpointListener.java | 7 +- .../FullPageIdIterableComparator.java | 51 ------ .../GridCacheDatabaseSharedManager.java | 63 +++++--- .../persistence/GridCacheOffheapManager.java | 56 +++---- .../cache/persistence/file/FilePageStore.java | 2 +- .../file/FilePageStoreManager.java | 1 + .../persistence/pagemem/PageMemoryEx.java | 8 +- .../persistence/pagemem/PageMemoryImpl.java | 10 +- .../partstate/GroupPartitionId.java | 145 ++++++++++++++++++ .../partstate/PagesAllocationRange.java | 68 ++++++++ .../partstate/PartitionAllocationMap.java | 113 ++++++++++++++ .../snapshot/IgniteCacheSnapshotManager.java | 17 +- .../cache/persistence/tree/io/PageMetaIO.java | 27 ++-- .../persistence/tree/io/TrackingPageIO.java | 8 +- .../persistence/tree/util/PageHandler.java | 3 +- 20 files changed, 466 insertions(+), 138 deletions(-) delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/GroupPartitionId.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java index 00f52c1250963..9e249437f5bb2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java @@ -21,7 +21,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; /** - * Compound object used to address a page in the global page space. + * Compound object used to address a page in the global page space. *

Page ID structure

*

* Generally, a full page ID consists of a cache ID and page ID. A page ID consists of @@ -49,13 +49,13 @@ * Effective page ID is page ID with zeroed bits used for page ID rotation. */ public class FullPageId { - /** */ + /** Page ID. */ private final long pageId; /** */ private final long effectivePageId; - /** */ + /** Cache group ID. */ private final int grpId; /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java index 92f427a9b0f06..6f4ba939ee5a5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java @@ -49,10 +49,10 @@ public final class PageIdUtils { /** */ public static final long TAG_MASK = ~(-1L << TAG_SIZE); - /** */ + /** Page Index is a monotonically growing number within each partition */ public static final long PART_ID_MASK = ~(-1L << PART_ID_SIZE); - /** */ + /** Flags mask. Flags consists from a number of reserved bits, and page type (data/index page) */ public static final long FLAG_MASK = ~(-1L << FLAG_SIZE); /** */ @@ -92,10 +92,10 @@ public static long link(long pageId, int itemId) { } /** - * Extracts a page index from the given pageId. + * Extracts a page index from the given page ID. * - * @param pageId Page id. - * @return Page ID. + * @param pageId Page ID. + * @return Page index. */ public static int pageIndex(long pageId) { return (int)(pageId & PAGE_IDX_MASK); // 4 bytes @@ -150,7 +150,9 @@ public static int tag(long link) { /** * @param partId Partition ID. - * @return Part ID constructed from the given cache ID and partition ID. + * @param flag Flags (a number of reserved bits, and page type (data/index page)) + * @param pageIdx Page index, monotonically growing number within each partition + * @return Page ID constructed from the given pageIdx and partition ID, see {@link FullPageId} */ public static long pageId(int partId, byte flag, int pageIdx) { long pageId = flag & FLAG_MASK; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java index a1b766f7489a7..eaa85adebe0cb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java @@ -77,6 +77,7 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac * * @param grpId Cache group ID of the evicted partition. * @param partId Partition ID. + * @param tag Partition tag (growing 1-based partition file version). * @throws IgniteCheckedException If failed to handle partition destroy callback. */ public void onPartitionDestroyed(int grpId, int partId, int tag) throws IgniteCheckedException; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java index be83704414203..4698a6b116fb0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java @@ -70,6 +70,8 @@ public interface PageStore { * * @param pageId Page ID. * @param pageBuf Page buffer to write. + * @param tag Partition file version, 1-based incrementing counter. For outdated pages {@code tag} has lower value, + * and write does nothing * @throws IgniteCheckedException If page writing failed (IO error occurred). */ public void write(long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java index 60aebdeeb4aef..11b2a6746dde1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java @@ -44,7 +44,7 @@ public MetaPageUpdateLastAllocatedIndex(int grpId, long pageId, int lastAllocate PageMetaIO io = PageMetaIO.VERSIONS.forVersion(PageIO.getVersion(pageAddr)); - io.setLastPageCount(pageAddr, lastAllocatedIdx); + io.setLastAllocatedPageCount(pageAddr, lastAllocatedIdx); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java index 0b28b6ab4d546..1c438b86b1daa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java @@ -17,9 +17,8 @@ package org.apache.ignite.internal.processors.cache.persistence; -import java.util.Map; import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap; /** * @@ -35,9 +34,9 @@ public interface Context { public boolean nextSnapshot(); /** - * + * @return Partition allocation statistic map */ - public Map, T2> partitionStatMap(); + public PartitionAllocationMap partitionStatMap(); /** * @param cacheOrGrpName Cache or group name. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java deleted file mode 100644 index c056c52d00dc6..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.persistence; - -import java.io.Serializable; -import java.util.Comparator; -import org.apache.ignite.internal.util.typedef.T2; - -/** - * - */ -public class FullPageIdIterableComparator implements Comparator>, Serializable { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - public static final FullPageIdIterableComparator INSTANCE = new FullPageIdIterableComparator(); - - /** {@inheritDoc} */ - @Override public int compare(T2 o1, T2 o2) { - if (o1.get1() < o2.get1()) - return -1; - - if (o1.get1() > o2.get1()) - return 1; - - if (o1.get2() < o2.get2()) - return -1; - - if (o1.get2() > o2.get2()) - return 1; - - return 0; - } -} - diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 51367316ba8c5..9f2067af7445a 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -40,7 +40,6 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; -import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -52,6 +51,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; @@ -109,6 +109,7 @@ import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; @@ -263,8 +264,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** */ private boolean stopping; - /** Checkpoint runner thread pool. */ - private ExecutorService asyncRunner; + /** Checkpoint runner thread pool. If null tasks are to be run in single thread */ + @Nullable private ExecutorService asyncRunner; /** Buffer for the checkpoint threads. */ private ThreadLocal threadBuf; @@ -1916,6 +1917,8 @@ private void doCheckpoint() { asyncRunner == null ? 1 : chp.cpPages.collectionsSize()); tracker.onPagesWriteStart(); + final AtomicInteger writtenPagesCtr = new AtomicInteger(); + final int totalPagesToWriteCnt = chp.cpPages.size(); if (asyncRunner != null) { for (int i = 0; i < chp.cpPages.collectionsSize(); i++) { @@ -1923,7 +1926,9 @@ private void doCheckpoint() { tracker, chp.cpPages.innerCollection(i), updStores, - doneWriteFut + doneWriteFut, + writtenPagesCtr, + totalPagesToWriteCnt ); try { @@ -1937,7 +1942,12 @@ private void doCheckpoint() { } else { // Single-threaded checkpoint. - Runnable write = new WriteCheckpointPages(tracker, chp.cpPages, updStores, doneWriteFut); + Runnable write = new WriteCheckpointPages(tracker, + chp.cpPages, + updStores, + doneWriteFut, + writtenPagesCtr, + totalPagesToWriteCnt); write.run(); } @@ -2092,15 +2102,15 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws curCpProgress = curr; } - final NavigableMap, T2> map = - new TreeMap<>(FullPageIdIterableComparator.INSTANCE); + final PartitionAllocationMap map = new PartitionAllocationMap(); DbCheckpointListener.Context ctx0 = new DbCheckpointListener.Context() { @Override public boolean nextSnapshot() { return curr.nextSnapshot; } - @Override public Map, T2> partitionStatMap() { + /** {@inheritDoc} */ + @Override public PartitionAllocationMap partitionStatMap() { return map; } @@ -2278,14 +2288,12 @@ public void shutdownNow() { } } - /** - * - */ + /** Pages write task */ private class WriteCheckpointPages implements Runnable { /** */ private CheckpointMetricsTracker tracker; - /** */ + /** Collection of page IDs to write under this task. Overall pages to write may be greater than this collection*/ private Collection writePageIds; /** */ @@ -2294,19 +2302,34 @@ private class WriteCheckpointPages implements Runnable { /** */ private CountDownFuture doneFut; + /** Counter for all written pages. May be shared between several workers */ + private AtomicInteger writtenPagesCntr; + + /** Total pages to write, counter may be greater than {@link #writePageIds} size*/ + private final int totalPagesToWrite; + /** - * @param writePageIds Write page IDs. + * Creates task for write pages + * @param tracker + * @param writePageIds Collection of page IDs to write. + * @param updStores + * @param doneFut + * @param writtenPagesCntr all written pages counter, may be shared between several write tasks + * @param totalPagesToWrite total pages to be written under this checkpoint */ private WriteCheckpointPages( - CheckpointMetricsTracker tracker, - Collection writePageIds, - GridConcurrentHashSet updStores, - CountDownFuture doneFut - ) { + final CheckpointMetricsTracker tracker, + final Collection writePageIds, + final GridConcurrentHashSet updStores, + final CountDownFuture doneFut, + @NotNull final AtomicInteger writtenPagesCntr, + final int totalPagesToWrite) { this.tracker = tracker; this.writePageIds = writePageIds; this.updStores = updStores; this.doneFut = doneFut; + this.writtenPagesCntr = writtenPagesCntr; + this.totalPagesToWrite = totalPagesToWrite; } /** {@inheritDoc} */ @@ -2354,7 +2377,9 @@ private WriteCheckpointPages( tmpWriteBuf.rewind(); } - snapshotMgr.onPageWrite(fullId, tmpWriteBuf); + int curWrittenPages = writtenPagesCntr.incrementAndGet(); + + snapshotMgr.onPageWrite(fullId, tmpWriteBuf, curWrittenPages, totalPagesToWrite); tmpWriteBuf.rewind(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index bd902fb36f7da..6e6b7dfb38d23 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -52,6 +52,9 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; +import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PagesAllocationRange; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionCountersIO; @@ -63,7 +66,6 @@ import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.lang.GridCursor; import org.apache.ignite.internal.util.tostring.GridToStringInclude; -import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; @@ -207,9 +209,9 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav long partMetaPage = pageMem.acquirePage(grpId, partMetaId); try { - long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage); + long partMetaPageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage); - if (pageAddr == 0L) { + if (partMetaPageAddr == 0L) { U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage + ", saveMeta=" + saveMeta + ", beforeDestroy=" + beforeDestroy + ", size=" + size + ", updCntr=" + updCntr + ", state=" + state + ']'); @@ -220,21 +222,21 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav boolean changed = false; try { - PagePartitionMetaIO io = PageIO.getPageIO(pageAddr); + PagePartitionMetaIO io = PageIO.getPageIO(partMetaPageAddr); - changed |= io.setUpdateCounter(pageAddr, updCntr); - changed |= io.setGlobalRemoveId(pageAddr, rmvId); - changed |= io.setSize(pageAddr, size); + changed |= io.setUpdateCounter(partMetaPageAddr, updCntr); + changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId); + changed |= io.setSize(partMetaPageAddr, size); if (state != null) - changed |= io.setPartitionState(pageAddr, (byte)state.ordinal()); + changed |= io.setPartitionState(partMetaPageAddr, (byte)state.ordinal()); else assert grp.isLocal() : grp.cacheOrGroupName(); long cntrsPageId; if (grp.sharedGroup()) { - cntrsPageId = io.getCountersPageId(pageAddr); + cntrsPageId = io.getCountersPageId(partMetaPageAddr); byte[] data = serializeCacheSizes(store.cacheSizes()); @@ -247,7 +249,7 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav if (init && items > 0) { cntrsPageId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA); - io.setCountersPageId(pageAddr, cntrsPageId); + io.setCountersPageId(partMetaPageAddr, cntrsPageId); changed = true; } @@ -301,7 +303,7 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav if (needSnapshot) { pageCnt = this.ctx.pageStore().pages(grpId, store.partId()); - io.setCandidatePageCount(pageAddr, pageCnt); + io.setCandidatePageCount(partMetaPageAddr, pageCnt); if (saveMeta) { long metaPageId = pageMem.metaPageId(grpId); @@ -345,13 +347,13 @@ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean sav if (partMap.containsKey(store.partId()) && partMap.get(store.partId()) == GridDhtPartitionState.OWNING) - addPartition(ctx.partitionStatMap(), pageAddr, io, grpId, store.partId(), + addPartition(ctx.partitionStatMap(), partMetaPageAddr, io, grpId, store.partId(), this.ctx.pageStore().pages(grpId, store.partId())); changed = true; } else - pageCnt = io.getCandidatePageCount(pageAddr); + pageCnt = io.getCandidatePageCount(partMetaPageAddr); if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null)) wal.log(new MetaPageUpdatePartitionDataRecord( @@ -397,27 +399,29 @@ private byte[] serializeCacheSizes(Map cacheSizes) { /** * @param map Map to add values to. - * @param pageAddr page address + * @param metaPageAddr Meta page address * @param io Page Meta IO * @param cacheId Cache ID. - * @param partition Partition ID. - * @param pages Number of pages to add. + * @param partId Partition ID. Or {@link PageIdAllocator#INDEX_PARTITION} for index partition + * @param currAllocatedPageCnt total number of pages allocated for partition [partition, cacheId] */ private static void addPartition( - Map, T2> map, - long pageAddr, - PageMetaIO io, - int cacheId, - int partition, - int pages + final PartitionAllocationMap map, + final long metaPageAddr, + final PageMetaIO io, + final int cacheId, + final int partId, + final int currAllocatedPageCnt ) { - if (pages <= 1) + if (currAllocatedPageCnt <= 1) return; - assert PageIO.getPageId(pageAddr) != 0; + assert PageIO.getPageId(metaPageAddr) != 0; - int lastAllocatedIdx = io.getLastPageCount(pageAddr); - map.put(new T2<>(cacheId, partition), new T2<>(lastAllocatedIdx, pages)); + int lastAllocatedPageCnt = io.getLastAllocatedPageCount(metaPageAddr); + map.put( + new GroupPartitionId(cacheId, partId), + new PagesAllocationRange(lastAllocatedPageCnt, currAllocatedPageCnt)); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java index c827e96d609f1..a7ca13c2d1e62 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java @@ -77,7 +77,7 @@ public class FilePageStore implements PageStore { /** */ private volatile boolean recover; - /** */ + /** Partition file version, 1-based incrementing counter. For outdated pages tag has low value, and write does nothing */ private volatile int tag; /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index af20136b5be09..e2ad070741e6f 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -318,6 +318,7 @@ public void read(int cacheId, long pageId, ByteBuffer pageBuf, boolean keepCrc) * @param cacheId Cache ID to write. * @param pageId Page ID. * @param pageBuf Page buffer. + * @param tag Partition tag (growing 1-based partition file version). Used to validate page is not outdated * @return PageStore to which the page has been written. * @throws IgniteCheckedException If IO error occurred. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java index 7c63d419b2c4a..53e21b7246919 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java @@ -115,19 +115,19 @@ void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc, * * @param pageId Page ID to get byte buffer for. The page ID must be present in the collection returned by * the {@link #beginCheckpoint()} method call. - * @param tmpBuf Temporary buffer to write changes into. + * @param outBuf Temporary buffer to write changes into. * @param tracker Checkpoint metrics tracker. - * @return {@code True} if data were read, {@code false} otherwise (data already saved to storage). + * @return {@code Partition tag} if data was read, {@code null} otherwise (data already saved to storage). * @throws IgniteException If failed to obtain page data. */ - @Nullable public Integer getForCheckpoint(FullPageId pageId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker); + @Nullable public Integer getForCheckpoint(FullPageId pageId, ByteBuffer outBuf, CheckpointMetricsTracker tracker); /** * Marks partition as invalid / outdated. * * @param cacheId Cache ID. * @param partId Partition ID. - * @return New partition tag. + * @return New partition tag (growing 1-based partition file version). */ public int invalidate(int cacheId, int partId); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index 47381d72cc5d1..1b4cf81c9f2c2 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -792,8 +792,8 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck } /** {@inheritDoc} */ - @Override public Integer getForCheckpoint(FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) { - assert tmpBuf.remaining() == pageSize(); + @Override public Integer getForCheckpoint(FullPageId fullId, ByteBuffer outBuf, CheckpointMetricsTracker tracker) { + assert outBuf.remaining() == pageSize(); Segment seg = segment(fullId.groupId(), fullId.pageId()); @@ -876,7 +876,7 @@ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheck } } else - return copyPageForCheckpoint(absPtr, fullId, tmpBuf, tmpBuffer, tracker) ? tag : null; + return copyPageForCheckpoint(absPtr, fullId, outBuf, tmpBuffer, tracker) ? tag : null; } /** @@ -1565,7 +1565,7 @@ private class Segment extends ReentrantReadWriteLock { /** */ private final int maxDirtyPages; - /** */ + /** Maps partition (cacheId, partId) to its tag. Tag is 1-based incrementing partition file counter */ private final Map, Integer> partTagMap = new HashMap<>(); /** @@ -1903,7 +1903,7 @@ private long absolute(long relPtr) { /** * @param grpId Cache group ID. * @param partId Partition ID. - * @return Partition tag. + * @return Partition tag. Growing 1 based partition file version */ private int partTag(int grpId, int partId) { assert getReadHoldCount() > 0 || getWriteHoldCount() > 0; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/GroupPartitionId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/GroupPartitionId.java new file mode 100644 index 0000000000000..dbdf670ab5f5e --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/GroupPartitionId.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.partstate; + +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageIdAllocator; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.jetbrains.annotations.NotNull; + +/** + * Pair of cache group ID with partition ID. Immutable, comparable class, may be used as key in maps + */ +public class GroupPartitionId implements Comparable { + /** Index for super(meta) page. There is always such page for iterated cache partition */ + private static final int METAPAGE_IDX = 0; + + /** Cache group ID. */ + private final int grpId; + + /** Partition ID. */ + private final int partId; + + /** + * Creates group-partition tuple. + * + * @param grpId Group ID. + * @param partId Partition ID. + */ + public GroupPartitionId(final int grpId, final int partId) { + this.grpId = grpId; + this.partId = partId; + } + + /** + * @param partId Partition ID. + * @return flag to be used for partition + */ + private static byte getFlagByPartId(final int partId) { + return partId == PageIdAllocator.INDEX_PARTITION ? PageMemory.FLAG_IDX : PageMemory.FLAG_DATA; + } + + /** + * @return cache ID + */ + public int getGroupId() { + return grpId; + } + + /** + * @return Partition ID + */ + public int getPartitionId() { + return partId; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(GroupPartitionId.class, this); + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + GroupPartitionId key = (GroupPartitionId)o; + + if (grpId != key.grpId) + return false; + + return partId == key.partId; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int result = grpId; + + result = 31 * result + partId; + + return result; + } + + /** {@inheritDoc} */ + @Override public int compareTo(@NotNull GroupPartitionId o) { + if (getGroupId() < o.getGroupId()) + return -1; + + if (getGroupId() > o.getGroupId()) + return 1; + + if (getPartitionId() < o.getPartitionId()) + return -1; + + if (getPartitionId() > o.getPartitionId()) + return 1; + return 0; + } + + /** + * @param pageIdx Page Index, monotonically growing number within each partition + * @return page ID (64 bits) constructed from partition ID and given index + */ + private long createPageId(final int pageIdx) { + final int partId = getPartitionId(); + + return PageIdUtils.pageId(partId, getFlagByPartId(partId), pageIdx); + } + + /** + * Returns Full page ID. For index 0 will return super-page of next partition + * + * @param pageIdx Page Index, monotonically growing number within each partition + * @return FullPageId consists of cache ID (32 bits) and page ID (64 bits). + */ + @NotNull private FullPageId createFullPageId(final int pageIdx) { + return new FullPageId(createPageId(pageIdx), getGroupId()); + } + + /** + * @return will return super-page (metapage) of this partition + */ + @NotNull public FullPageId createFirstPageFullId() { + return createFullPageId(METAPAGE_IDX); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java new file mode 100644 index 0000000000000..e7170c3213afa --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.partstate; + +/** + * Range of pages allocated. + * Contains currently allocated page count and previously observed page count. + * May be used for tracking history of recent allocation for partition [partition, cacheId] + */ +public class PagesAllocationRange { + /** + * Previously observed total number of allocated pages. May be stored using PageMetaIO. + * Used to separate newly allocated pages with previously observed state + * Minimum value is 0. Can't be greater than {@link #currAllocatedPageCnt} + */ + private final int lastAllocatedPageCnt; + + /** Total current number of pages allocated, minimum value is 0. */ + private final int currAllocatedPageCnt; + + /** + * Creates pages range + * + * @param lastAllocatedPageCnt Last allocated pages count. + * @param currAllocatedPageCnt Currently allocated pages count. + */ + public PagesAllocationRange(final int lastAllocatedPageCnt, final int currAllocatedPageCnt) { + this.lastAllocatedPageCnt = lastAllocatedPageCnt; + this.currAllocatedPageCnt = currAllocatedPageCnt; + } + + /** + * @return Total current number of pages allocated, minimum value is 0. + */ + public int getCurrAllocatedPageCnt() { + return currAllocatedPageCnt; + } + + /** + * @return Previously observed total number of allocated pages. + */ + public int getLastAllocatedPageCnt() { + return lastAllocatedPageCnt; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "PagesAllocationRange{" + + "lastAllocatedPageCnt=" + lastAllocatedPageCnt + + ", currAllocatedPageCnt=" + currAllocatedPageCnt + + '}'; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java new file mode 100644 index 0000000000000..9ed40005ea4b5 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.partstate; + +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; +import java.util.TreeMap; +import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Information structure with partitions state. + * Page counts map. + */ +public class PartitionAllocationMap { + /** Maps following pairs: (groupId, partId) -> (lastAllocatedCount, allocatedCount) */ + private final NavigableMap map = new TreeMap<>(); + + /** + * Returns the value to which the specified key is mapped, + * or {@code null} if this map contains no mapping for the key. + * + * @param key to get + * @return value or null + */ + @Nullable public PagesAllocationRange get(GroupPartitionId key) { + return map.get(key); + } + + /** + * Extracts partition information from full page ID + * + * @param fullId page related to some cache + * @return pair of cache ID and partition ID + */ + @NotNull public static GroupPartitionId createCachePartId(@NotNull final FullPageId fullId) { + return new GroupPartitionId(fullId.groupId(), PageIdUtils.partId(fullId.pageId())); + } + + /** @return true if this map contains no key-value mappings */ + public boolean isEmpty() { + return map.isEmpty(); + } + + /** @return the number of key-value mappings in this map. */ + public int size() { + return map.size(); + } + + /** @return keys (all caches partitions) */ + public Set keySet() { + return map.keySet(); + } + + /** @return values (allocation ranges) */ + public Iterable values() { + return map.values(); + } + + /** @return Returns the first (lowest) key currently in this map. */ + public GroupPartitionId firstKey() { + return map.firstKey(); + } + + /** + * Returns next (higher) key for provided cache and partition or null + * + * @param key cache and partition to search + * @return first found key which is greater than provided + */ + @Nullable public GroupPartitionId nextKey(@NotNull final GroupPartitionId key) { + return map.navigableKeySet().higher(key); + } + + /** @return set view of the mappings contained in this map, sorted in ascending key order */ + public Set> entrySet() { + return map.entrySet(); + } + + /** @return true if this map contains a mapping for the specified key */ + public boolean containsKey(GroupPartitionId key) { + return map.containsKey(key); + } + + /** + * @param key key with which the specified value is to be associated + * @param val value to be associated with the specified key + * @return the previous value associated with key, or null if there was no mapping for + * key. + */ + public PagesAllocationRange put(GroupPartitionId key, PagesAllocationRange val) { + return map.put(key, val); + } + +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java index 0a27bcd2d8f6e..50e6515a45357 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; import java.nio.ByteBuffer; -import java.util.NavigableMap; import java.util.UUID; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.events.DiscoveryEvent; @@ -29,8 +28,8 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; +import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap; import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport; -import org.apache.ignite.internal.util.typedef.T2; import org.jetbrains.annotations.Nullable; /** @@ -64,12 +63,13 @@ public class IgniteCacheSnapshotManager extends Gri /** * @param snapshotOperation current snapshot operation. + * @param map (cacheId, partId) -> (lastAllocatedIndex, count) * * @return {@code true} if next operation must be snapshot, {@code false} if checkpoint must be executed. */ public boolean onMarkCheckPointBegin( T snapshotOperation, - NavigableMap, T2> map + PartitionAllocationMap map ) throws IgniteCheckedException { return false; } @@ -107,9 +107,16 @@ public void beforePageWrite(FullPageId fullId) { } /** - * @param fullId Full id. + * @param fullId Full page id. + * @param tmpWriteBuf buffer + * @param writtenPages Overall pages written, negative value means there is no progress tracked + * @param totalPages Overall pages count to be written, should be positive */ - public void onPageWrite(FullPageId fullId, ByteBuffer tmpWriteBuf) { + public void onPageWrite( + final FullPageId fullId, + final ByteBuffer tmpWriteBuf, + final int writtenPages, + final int totalPages) { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java index ac482e88a8e99..becd3e441f391 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java @@ -43,10 +43,10 @@ public class PageMetaIO extends PageIO { /** Last successful full snapshot tag offset. */ private static final int LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF = NEXT_SNAPSHOT_TAG_OFF + 8; - /** Last allocated index offset. */ + /** Last allocated pages count offset. */ private static final int LAST_PAGE_COUNT_OFF = LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF + 8; - /** Candidate allocated index offset. */ + /** Candidate allocated page count offset. */ private static final int CANDIDATE_PAGE_COUNT_OFF = LAST_PAGE_COUNT_OFF + 4; /** End of page meta. */ @@ -82,7 +82,7 @@ protected PageMetaIO(int type, int ver) { setLastSuccessfulSnapshotId(pageAddr, 0); setNextSnapshotTag(pageAddr, 1); setLastSuccessfulSnapshotTag(pageAddr, 0); - setLastPageCount(pageAddr, 0); + setLastAllocatedPageCount(pageAddr, 0); setCandidatePageCount(pageAddr, 0); } @@ -179,24 +179,31 @@ public long getNextSnapshotTag(long pageAddr) { } /** - * @param pageAddr Page address. - * @param pageCnt Last allocated index. + * Sets last allocated pages count, used to save and observe previous allocated count + * + * @param pageAddr Meta Page address. + * @param pageCnt Last allocated pages count to set */ - public void setLastPageCount(long pageAddr, int pageCnt) { + public void setLastAllocatedPageCount(final long pageAddr, final int pageCnt) { PageUtils.putInt(pageAddr, LAST_PAGE_COUNT_OFF, pageCnt); } /** - * @param buf Buffer. + * Gets last allocated pages count from given buffer + * + * @param buf Buffer to read data from. */ - public int getLastPageCount(@NotNull ByteBuffer buf) { + public int getLastAllocatedPageCount(@NotNull final ByteBuffer buf) { return buf.getInt(LAST_PAGE_COUNT_OFF); } /** - * @param pageAddr Page address. + * Gets last allocated pages count by provided address + * + * @param pageAddr Meta page address. + * @return Last allocated page count */ - public int getLastPageCount(long pageAddr) { + public int getLastAllocatedPageCount(final long pageAddr) { return PageUtils.getInt(pageAddr, LAST_PAGE_COUNT_OFF); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java index 22631305587ce..205177849d924 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler; +import org.jetbrains.annotations.Nullable; /** * We use dedicated page for tracking pages updates. @@ -182,6 +183,7 @@ long getLastSnapshotTag(ByteBuffer buf) { * @param buf Buffer. * @param pageId Page id. * @param curSnapshotTag Snapshot tag. + * @param lastSuccessfulSnapshotTag Last successful snapshot id. * @param pageSize Page size. */ public boolean wasChanged(ByteBuffer buf, long pageId, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) { @@ -265,10 +267,12 @@ public int countOfPageToTrack(int pageSize) { * @param buf Buffer. * @param start Start. * @param curSnapshotTag Snapshot id. + * @param lastSuccessfulSnapshotTag Last successful snapshot id. * @param pageSize Page size. - * @return set pageId if it was changed or next closest one, if there is no changed page null will be returned + * @return set pageId if it was changed or next closest one, if there is no changed page {@code null} will be returned */ - public Long findNextChangedPage(ByteBuffer buf, long start, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) { + @Nullable public Long findNextChangedPage(ByteBuffer buf, long start, long curSnapshotTag, + long lastSuccessfulSnapshotTag, int pageSize) { validateSnapshotId(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize); int cntOfPage = countOfPageToTrack(pageSize); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java index a87525af16c38..33169800c7067 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.PageSupport; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; @@ -440,7 +441,7 @@ private static void doInitPage( * @return {@code true} If we need to make a delta WAL record for the change in this page. */ public static boolean isWalDeltaRecordNeeded( - PageMemory pageMem, + PageSupport pageMem, int cacheId, long pageId, long page, From 689b1b6e9c3e723cf394c7ff2427097b21d96ce3 Mon Sep 17 00:00:00 2001 From: Alexey Goncharuk Date: Thu, 13 Jul 2017 10:12:01 +0300 Subject: [PATCH 150/155] IGNITE-5479 - Cleanup public API for PersistentStoreConfiguration --- .../configuration/MemoryConfiguration.java | 4 +- .../MemoryPolicyConfiguration.java | 4 +- .../PersistentStoreConfiguration.java | 61 ++++++++++--------- .../GridCacheDatabaseSharedManager.java | 23 ++++--- .../IgniteCacheDatabaseSharedManager.java | 21 +++---- .../wal/FileWriteAheadLogManager.java | 6 +- .../utils/PlatformConfigurationUtils.java | 4 +- .../VisorPersistentStoreConfiguration.java | 36 +++++------ .../db/wal/IgnitePdsWalTlbTest.java | 2 +- .../Cache/PersistentStoreTest.cs | 8 ++- .../Config/spring-test.xml | 4 ++ .../IgniteConfigurationTest.cs | 8 ++- .../PersistentStoreConfiguration.cs | 31 ++++++---- modules/web-console/backend/app/mongo.js | 2 +- .../generator/ConfigurationGenerator.js | 8 +-- .../generator/defaults/Cluster.service.js | 2 +- .../configuration/clusters/persistence.pug | 2 +- 17 files changed, 128 insertions(+), 98 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java index 5cf6cb76707e0..f83d7d095064f 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java @@ -64,7 +64,7 @@ public class MemoryConfiguration implements Serializable { /** Default memory policy start size (256 MB). */ @SuppressWarnings("UnnecessaryBoxing") - public static final Long DFLT_MEMORY_POLICY_INITIAL_SIZE = new Long(256L * 1024 * 1024); + public static final long DFLT_MEMORY_POLICY_INITIAL_SIZE = 256L * 1024 * 1024; /** Fraction of available memory to allocate for default MemoryPolicy. */ private static final double DFLT_MEMORY_POLICY_FRACTION = 0.8; @@ -222,6 +222,8 @@ public MemoryPolicyConfiguration createDefaultPolicyConfig() { if (maxSize < DFLT_MEMORY_POLICY_INITIAL_SIZE) memPlc.setInitialSize(maxSize); + else + memPlc.setInitialSize(DFLT_MEMORY_POLICY_INITIAL_SIZE); memPlc.setMaxSize(maxSize); diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryPolicyConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryPolicyConfiguration.java index b496901731df7..dff8b2b2ec490 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryPolicyConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryPolicyConfiguration.java @@ -78,7 +78,7 @@ public final class MemoryPolicyConfiguration implements Serializable { private String name = DFLT_MEM_PLC_DEFAULT_NAME; /** Memory policy start size. */ - private Long initialSize = MemoryConfiguration.DFLT_MEMORY_POLICY_INITIAL_SIZE; + private long initialSize; /** Memory policy maximum size. */ private long maxSize = MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE; @@ -172,7 +172,7 @@ public MemoryPolicyConfiguration setMaxSize(long maxSize) { * * @return Memory policy start size. */ - public Long getInitialSize() { + public long getInitialSize() { return initialSize; } diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java index 479248364b4b9..e8a0ff47e7015 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java @@ -44,10 +44,6 @@ public class PersistentStoreConfiguration implements Serializable { /** Default length of interval over which rate-based metric is calculated. */ public static final int DFLT_RATE_TIME_INTERVAL_MILLIS = 60_000; - /** */ - @SuppressWarnings("UnnecessaryBoxing") - public static final Long DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE = new Long(256L * 1024 * 1024); - /** Default number of checkpointing threads. */ public static final int DFLT_CHECKPOINTING_THREADS = 1; @@ -70,7 +66,7 @@ public class PersistentStoreConfiguration implements Serializable { public static final int DFLT_WAL_FLUSH_FREQ = 2000; /** Default wal fsync delay. */ - public static final int DFLT_WAL_FSYNC_DELAY = 1; + public static final int DFLT_WAL_FSYNC_DELAY = 1000; /** Default wal record iterator buffer size. */ public static final int DFLT_WAL_RECORD_ITERATOR_BUFFER_SIZE = 64 * 1024 * 1024; @@ -90,11 +86,11 @@ public class PersistentStoreConfiguration implements Serializable { /** Checkpointing frequency. */ private long checkpointingFreq = DFLT_CHECKPOINTING_FREQ; - /** Lock wait time. */ - private int lockWaitTime = DFLT_LOCK_WAIT_TIME; + /** Lock wait time, in milliseconds. */ + private long lockWaitTime = DFLT_LOCK_WAIT_TIME; /** */ - private Long checkpointingPageBufSize = DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE; + private long checkpointingPageBufSize; /** */ private int checkpointingThreads = DFLT_CHECKPOINTING_THREADS; @@ -124,10 +120,10 @@ public class PersistentStoreConfiguration implements Serializable { private int tlbSize = DFLT_TLB_SIZE; /** Wal flush frequency in milliseconds. */ - private int walFlushFreq = DFLT_WAL_FLUSH_FREQ; + private long walFlushFreq = DFLT_WAL_FLUSH_FREQ; /** Wal fsync delay. */ - private int walFsyncDelay = DFLT_WAL_FSYNC_DELAY; + private long walFsyncDelay = DFLT_WAL_FSYNC_DELAY; /** Wal record iterator buffer size. */ private int walRecordIterBuffSize = DFLT_WAL_RECORD_ITERATOR_BUFFER_SIZE; @@ -200,9 +196,10 @@ public PersistentStoreConfiguration setCheckpointingFrequency(long checkpointing /** * Gets amount of memory allocated for a checkpointing temporary buffer. * - * @return checkpointing page buffer size in bytes. + * @return Checkpointing page buffer size in bytes or {@code 0} for Ignite + * to choose the buffer size automatically. */ - public Long getCheckpointingPageBufferSize() { + public long getCheckpointingPageBufferSize() { return checkpointingPageBufSize; } @@ -211,7 +208,8 @@ public Long getCheckpointingPageBufferSize() { * copies of pages that are being written to disk and being update in parallel while the checkpointing is in * progress. * - * @param checkpointingPageBufSize checkpointing page buffer size in bytes. + * @param checkpointingPageBufSize Checkpointing page buffer size in bytes or {@code 0} for Ignite to + * choose the buffer size automatically. * @return {@code this} for chaining. */ public PersistentStoreConfiguration setCheckpointingPageBufferSize(long checkpointingPageBufSize) { @@ -243,21 +241,23 @@ public PersistentStoreConfiguration setCheckpointingThreads(int checkpointingThr } /** - * Time out in second, while wait and try get file lock for start persist manager. + * Time out in milliseonds to wait when acquiring persistence store lock file before failing the + * local node. * - * @return Time for wait. + * @return Lock wait time in milliseconds. */ - public int getLockWaitTime() { + public long getLockWaitTime() { return lockWaitTime; } /** - * Time out in milliseconds, while wait and try get file lock for start persist manager. + * Time out in milliseconds to wait when acquiring persistence store lock file before failing the + * local node. * - * @param lockWaitTime Lock wait time. + * @param lockWaitTime Lock wait time in milliseconds. * @return {@code this} for chaining. */ - public PersistentStoreConfiguration setLockWaitTime(int lockWaitTime) { + public PersistentStoreConfiguration setLockWaitTime(long lockWaitTime) { this.lockWaitTime = lockWaitTime; return this; @@ -474,19 +474,22 @@ public PersistentStoreConfiguration setTlbSize(int tlbSize) { } /** - * Property define how often will be fsync, in milliseconds. - * In background mode, exist thread which do fsync by timeout. + * This property define how often WAL will be fsync-ed in {@code BACKGROUND} mode. Ignored for + * all other WAL modes. * - * @return Flush frequency. + * @return WAL flush frequency, in milliseconds. */ - public int getWalFlushFrequency() { + public long getWalFlushFrequency() { return walFlushFreq; } /** - * @param walFlushFreq Wal flush frequency, in milliseconds. + * This property define how often WAL will be fsync-ed in {@code BACKGROUND} mode. Ignored for + * all other WAL modes. + * + * @param walFlushFreq WAL flush frequency, in milliseconds. */ - public PersistentStoreConfiguration setWalFlushFrequency(int walFlushFreq) { + public PersistentStoreConfiguration setWalFlushFrequency(long walFlushFreq) { this.walFlushFreq = walFlushFreq; return this; @@ -495,15 +498,15 @@ public PersistentStoreConfiguration setWalFlushFrequency(int walFlushFreq) { /** * Gets the fsync delay, in nanoseconds. */ - public int getWalFsyncDelay() { + public long getWalFsyncDelayNanos() { return walFsyncDelay <= 0 ? DFLT_WAL_FSYNC_DELAY : walFsyncDelay; } /** - * @param walFsyncDelay Wal fsync delay, in nanoseconds. + * @param walFsyncDelayNanos Wal fsync delay, in nanoseconds. */ - public PersistentStoreConfiguration setWalFsyncDelay(int walFsyncDelay) { - this.walFsyncDelay = walFsyncDelay; + public PersistentStoreConfiguration setWalFsyncDelayNanos(long walFsyncDelayNanos) { + walFsyncDelay = walFsyncDelayNanos; return this; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 9f2067af7445a..d147f3699cf0d 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -151,6 +151,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan /** */ public static final String IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC = "IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC"; + /** Default checkpointing page buffer size (may be adjusted by Ignite). */ + public static final Long DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE = 256L * 1024 * 1024; + /** Skip sync. */ private final boolean skipSync = IgniteSystemProperties.getBoolean(IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC); @@ -277,7 +280,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan private FileLockHolder fileLockHolder; /** Lock wait time. */ - private final int lockWaitTime; + private final long lockWaitTime; /** */ private Map>> reservedForExchange; @@ -375,10 +378,10 @@ public IgniteInternalFuture enableCheckpoints(boolean enable) { } /** - * @throws IgniteCheckedException If failed. + * */ - private void initDataBase() throws IgniteCheckedException { - Long cpBufSize = persistenceCfg.getCheckpointingPageBufferSize(); + private void initDataBase() { + long cpBufSize = persistenceCfg.getCheckpointingPageBufferSize(); if (persistenceCfg.getCheckpointingThreads() > 1) asyncRunner = new ThreadPoolExecutor( @@ -391,7 +394,9 @@ private void initDataBase() throws IgniteCheckedException { // Intentionally use identity comparison to check if configuration default has changed. //noinspection NumberEquality - if (cpBufSize == PersistentStoreConfiguration.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE) { + if (cpBufSize == 0L) { + cpBufSize = DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE; + MemoryConfiguration memCfg = cctx.kernalContext().config().getMemoryConfiguration(); assert memCfg != null; @@ -2928,10 +2933,10 @@ private FileLockHolder(String path, GridKernalContext ctx, IgniteLogger log) { } /** - * @param lockWaitTime During which time thread will try capture file lock. + * @param lockWaitTimeMillis During which time thread will try capture file lock. * @throws IgniteCheckedException If failed to capture file lock. */ - public void tryLock(int lockWaitTime) throws IgniteCheckedException { + public void tryLock(long lockWaitTimeMillis) throws IgniteCheckedException { assert lockFile != null; FileChannel ch = lockFile.getChannel(); @@ -2965,7 +2970,7 @@ public void tryLock(int lockWaitTime) throws IgniteCheckedException { String content = null; // Try to get lock, if not available wait 1 sec and re-try. - for (int i = 0; i < lockWaitTime; i += 1000) { + for (int i = 0; i < lockWaitTimeMillis; i += 1000) { try { lock = ch.tryLock(0, 1, false); if (lock != null && lock.isValid()) { @@ -2989,7 +2994,7 @@ public void tryLock(int lockWaitTime) throws IgniteCheckedException { if (content == null) content = readContent(); - failMsg = "Failed to acquire file lock during " + (lockWaitTime / 1000) + + failMsg = "Failed to acquire file lock during " + (lockWaitTimeMillis / 1000) + " sec, (locked by " + content + "): " + file.getAbsolutePath(); } catch (Exception e) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index c5f174cea626c..c503fb4aa95ca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -294,15 +294,6 @@ private boolean hasCustomDefaultMemoryPolicy(MemoryPolicyConfiguration[] memPlcs return false; } - /** - * @param dbCfg Database configuration. - * @param memPlcCfg MemoryPolicy configuration. - * @param memMetrics MemoryMetrics instance. - */ - private MemoryPolicy createDefaultMemoryPolicy(MemoryConfiguration dbCfg, MemoryPolicyConfiguration memPlcCfg, MemoryMetricsImpl memMetrics) { - return initMemory(dbCfg, memPlcCfg, memMetrics); - } - /** * @param sysCacheInitSize Initial size of PageMemory to be created for system cache. * @param sysCacheMaxSize Maximum size of PageMemory to be created for system cache. @@ -454,6 +445,14 @@ private static void checkDefaultPolicyConfiguration( * @throws IgniteCheckedException If config is invalid. */ private void checkPolicySize(MemoryPolicyConfiguration plcCfg) throws IgniteCheckedException { + boolean dfltInitSize = false; + + if (plcCfg.getInitialSize() == 0) { + plcCfg.setInitialSize(DFLT_MEMORY_POLICY_INITIAL_SIZE); + + dfltInitSize = true; + } + if (plcCfg.getInitialSize() < MIN_PAGE_MEMORY_SIZE) throw new IgniteCheckedException("MemoryPolicy must have size more than 10MB (use " + "MemoryPolicyConfiguration.initialSize property to set correct size in bytes) " + @@ -461,8 +460,8 @@ private void checkPolicySize(MemoryPolicyConfiguration plcCfg) throws IgniteChec ); if (plcCfg.getMaxSize() < plcCfg.getInitialSize()) { - // We will know for sure if initialSize has been changed if we compare Longs by "==". - if (plcCfg.getInitialSize() == DFLT_MEMORY_POLICY_INITIAL_SIZE) { + // If initial size was not set, use the max size. + if (dfltInitSize) { plcCfg.setInitialSize(plcCfg.getMaxSize()); LT.warn(log, "MemoryPolicy maxSize=" + U.readableSize(plcCfg.getMaxSize(), true) + diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 5c112fb1b72b1..897f903deda04 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -124,7 +124,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl private final int tlbSize; /** WAL flush frequency. Makes sense only for {@link WALMode#BACKGROUND} log WALMode. */ - private final int flushFreq; + private final long flushFreq; /** Fsync delay. */ private final long fsyncDelay; @@ -229,7 +229,7 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) { mode = psCfg.getWalMode(); tlbSize = psCfg.getTlbSize(); flushFreq = psCfg.getWalFlushFrequency(); - fsyncDelay = psCfg.getWalFsyncDelay(); + fsyncDelay = psCfg.getWalFsyncDelayNanos(); alwaysWriteFullPages = psCfg.isAlwaysWriteFullPages(); ioFactory = psCfg.getFileIOFactory(); walAutoArchiveAfterInactivity = psCfg.getWalAutoArchiveAfterInactivity(); @@ -1666,7 +1666,7 @@ private class FileWriteHandle extends FileHandle { /** Condition activated each time writeBuffer() completes. Used to wait previously flushed write to complete */ private final Condition writeComplete = lock.newCondition(); - /** Condition for timed wait of several threads, see {@link PersistentStoreConfiguration#getWalFsyncDelay()} */ + /** Condition for timed wait of several threads, see {@link PersistentStoreConfiguration#getWalFsyncDelayNanos()} */ private final Condition fsync = lock.newCondition(); /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java index 03b30dbf2ab8d..8c9f5e10fdd96 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java @@ -1499,7 +1499,7 @@ private static PersistentStoreConfiguration readPersistentStoreConfiguration(Bin .setWalMode(WALMode.fromOrdinal(in.readInt())) .setTlbSize(in.readInt()) .setWalFlushFrequency((int) in.readLong()) - .setWalFsyncDelay(in.readInt()) + .setWalFsyncDelayNanos(in.readLong()) .setWalRecordIteratorBufferSize(in.readInt()) .setAlwaysWriteFullPages(in.readBoolean()) .setMetricsEnabled(in.readBoolean()) @@ -1531,7 +1531,7 @@ private static void writePersistentStoreConfiguration(BinaryRawWriter w, Persist w.writeInt(cfg.getWalMode().ordinal()); w.writeInt(cfg.getTlbSize()); w.writeLong(cfg.getWalFlushFrequency()); - w.writeInt(cfg.getWalFsyncDelay()); + w.writeLong(cfg.getWalFsyncDelayNanos()); w.writeInt(cfg.getWalRecordIteratorBufferSize()); w.writeBoolean(cfg.isAlwaysWriteFullPages()); w.writeBoolean(cfg.isMetricsEnabled()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistentStoreConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistentStoreConfiguration.java index 3559845eac29f..128f43a189d91 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistentStoreConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistentStoreConfiguration.java @@ -39,10 +39,10 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject { private long checkpointingFreq; /** Lock wait time. */ - private int lockWaitTime; + private long lockWaitTime; /** */ - private Long checkpointingPageBufSize; + private long checkpointingPageBufSize; /** */ private int checkpointingThreads; @@ -72,10 +72,10 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject { private int tlbSize; /** Wal flush frequency. */ - private int walFlushFreq; + private long walFlushFreq; - /** Wal fsync delay. */ - private int walFsyncDelay; + /** Wal fsync delay in nanoseconds. */ + private long walFsyncDelay; /** Wal record iterator buffer size. */ private int walRecordIterBuffSize; @@ -114,7 +114,7 @@ public VisorPersistentStoreConfiguration(PersistentStoreConfiguration cfg) { walMode = cfg.getWalMode(); tlbSize = cfg.getTlbSize(); walFlushFreq = cfg.getWalFlushFrequency(); - walFsyncDelay = cfg.getWalFsyncDelay(); + walFsyncDelay = cfg.getWalFsyncDelayNanos(); walRecordIterBuffSize = cfg.getWalRecordIteratorBufferSize(); alwaysWriteFullPages = cfg.isAlwaysWriteFullPages(); subIntervals = cfg.getSubIntervals(); @@ -138,7 +138,7 @@ public long getCheckpointingFrequency() { /** * @return Checkpointing page buffer size in bytes. */ - public Long getCheckpointingPageBufferSize() { + public long getCheckpointingPageBufferSize() { return checkpointingPageBufSize; } @@ -152,7 +152,7 @@ public int getCheckpointingThreads() { /** * @return Time for wait. */ - public int getLockWaitTime() { + public long getLockWaitTime() { return lockWaitTime; } @@ -229,14 +229,14 @@ public int getTlbSize() { /** * @return Flush frequency. */ - public int getWalFlushFrequency() { + public long getWalFlushFrequency() { return walFlushFreq; } /** * Gets the fsync delay, in nanoseconds. */ - public int getWalFsyncDelay() { + public long getWalFsyncDelayNanos() { return walFsyncDelay; } @@ -258,8 +258,8 @@ public boolean isAlwaysWriteFullPages() { @Override protected void writeExternalData(ObjectOutput out) throws IOException { U.writeString(out, persistenceStorePath); out.writeLong(checkpointingFreq); - out.writeInt(lockWaitTime); - out.writeObject(checkpointingPageBufSize); + out.writeLong(lockWaitTime); + out.writeLong(checkpointingPageBufSize); out.writeInt(checkpointingThreads); out.writeInt(walHistSize); out.writeInt(walSegments); @@ -269,8 +269,8 @@ public boolean isAlwaysWriteFullPages() { out.writeBoolean(metricsEnabled); U.writeEnum(out, walMode); out.writeInt(tlbSize); - out.writeInt(walFlushFreq); - out.writeInt(walFsyncDelay); + out.writeLong(walFlushFreq); + out.writeLong(walFsyncDelay); out.writeInt(walRecordIterBuffSize); out.writeBoolean(alwaysWriteFullPages); out.writeInt(subIntervals); @@ -281,8 +281,8 @@ public boolean isAlwaysWriteFullPages() { @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { persistenceStorePath = U.readString(in); checkpointingFreq = in.readLong(); - lockWaitTime = in.readInt(); - checkpointingPageBufSize = (Long)in.readObject(); + lockWaitTime = in.readLong(); + checkpointingPageBufSize = in.readLong(); checkpointingThreads = in.readInt(); walHistSize = in.readInt(); walSegments = in.readInt(); @@ -292,8 +292,8 @@ public boolean isAlwaysWriteFullPages() { metricsEnabled = in.readBoolean(); walMode = WALMode.fromOrdinal(in.readByte()); tlbSize = in.readInt(); - walFlushFreq = in.readInt(); - walFsyncDelay = in.readInt(); + walFlushFreq = in.readLong(); + walFsyncDelay = in.readLong(); walRecordIterBuffSize = in.readInt(); alwaysWriteFullPages = in.readBoolean(); subIntervals = in.readInt(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java index a1a7286ac0913..fd0fd34fe9135 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java @@ -31,7 +31,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import static org.apache.ignite.configuration.PersistentStoreConfiguration.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE; +import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE; /** * diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs index 3320dd745c2f1..d321639751b4b 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs @@ -59,7 +59,8 @@ public void TestCacheDataSurvivesNodeRestart() PersistentStorePath = Path.Combine(_tempDir, "Store"), WalStorePath = Path.Combine(_tempDir, "WalStore"), WalArchivePath = Path.Combine(_tempDir, "WalArchive"), - MetricsEnabled = true + MetricsEnabled = true, + CheckpointingPageBufferSize = 1024 * 1024 // TODO: Use default (IGNITE-5717) } }; @@ -116,7 +117,10 @@ public void TestGridActivationWithPersistence() { var cfg = new IgniteConfiguration(GetTestConfiguration()) { - PersistentStoreConfiguration = new PersistentStoreConfiguration() + PersistentStoreConfiguration = new PersistentStoreConfiguration + { + CheckpointingPageBufferSize = 1024 * 1024 // TODO: Use default (IGNITE-5717) + } }; using (var ignite = Ignition.Start(cfg)) diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/spring-test.xml b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/spring-test.xml index dd0669ad71b20..31fa3b339e070 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/spring-test.xml +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/spring-test.xml @@ -56,5 +56,9 @@ + + + + diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs index 81fd226ef6ee2..93d6af3dc3c72 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs @@ -316,6 +316,9 @@ public void TestSpringXml() Assert.AreEqual(MemoryPolicyConfiguration.DefaultMaxSize, plc.MaxSize); Assert.AreEqual(MemoryPolicyConfiguration.DefaultSubIntervals, plc.SubIntervals); Assert.AreEqual(MemoryPolicyConfiguration.DefaultRateTimeInterval, plc.RateTimeInterval); + + // Check PersistentStoreConfiguration defaults. + CheckDefaultProperties(resCfg.PersistentStoreConfiguration); } } @@ -532,8 +535,7 @@ private static void CheckDefaultProperties(PersistentStoreConfiguration cfg) Assert.AreEqual(PersistentStoreConfiguration.DefaultTlbSize, cfg.TlbSize); Assert.AreEqual(PersistentStoreConfiguration.DefaultCheckpointingFrequency, cfg.CheckpointingFrequency); Assert.AreEqual(PersistentStoreConfiguration.DefaultCheckpointingThreads, cfg.CheckpointingThreads); - Assert.AreEqual(PersistentStoreConfiguration.DefaultCheckpointingPageBufferSize, - cfg.CheckpointingPageBufferSize); + Assert.AreEqual(default(long), cfg.CheckpointingPageBufferSize); Assert.AreEqual(PersistentStoreConfiguration.DefaultLockWaitTime, cfg.LockWaitTime); Assert.AreEqual(PersistentStoreConfiguration.DefaultWalFlushFrequency, cfg.WalFlushFrequency); Assert.AreEqual(PersistentStoreConfiguration.DefaultWalFsyncDelayNanos, cfg.WalFsyncDelayNanos); @@ -546,6 +548,8 @@ private static void CheckDefaultProperties(PersistentStoreConfiguration cfg) Assert.IsFalse(cfg.MetricsEnabled); Assert.AreEqual(PersistentStoreConfiguration.DefaultSubIntervals, cfg.SubIntervals); Assert.AreEqual(PersistentStoreConfiguration.DefaultRateTimeInterval, cfg.RateTimeInterval); + Assert.AreEqual(PersistentStoreConfiguration.DefaultWalStorePath, cfg.WalStorePath); + Assert.AreEqual(PersistentStoreConfiguration.DefaultWalArchivePath, cfg.WalArchivePath); } ///

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs index 43b17ac7effc2..cac6cc83a191a 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs @@ -29,11 +29,6 @@ namespace Apache.Ignite.Core.PersistentStore /// public class PersistentStoreConfiguration { - /// - /// Default value for . - /// - public const long DefaultCheckpointingPageBufferSize = 256L * 1024 * 1024; - /// /// Default value for . /// @@ -82,7 +77,7 @@ public class PersistentStoreConfiguration /// /// Default value for . /// - public const int DefaultWalFsyncDelayNanos = 1; + public const long DefaultWalFsyncDelayNanos = 1000; /// /// The default sub intervals. @@ -96,12 +91,21 @@ public class PersistentStoreConfiguration /// public static readonly TimeSpan DefaultRateTimeInterval = TimeSpan.FromSeconds(60); + /// + /// Default value for . + /// + public const string DefaultWalStorePath = "db/wal"; + + /// + /// Default value for . + /// + public const string DefaultWalArchivePath = "db/wal/archive"; + /// /// Initializes a new instance of the class. /// public PersistentStoreConfiguration() { - CheckpointingPageBufferSize = DefaultCheckpointingPageBufferSize; CheckpointingThreads = DefaultCheckpointingThreads; CheckpointingFrequency = DefaultCheckpointingFrequency; LockWaitTime = DefaultLockWaitTime; @@ -114,6 +118,8 @@ public PersistentStoreConfiguration() WalFsyncDelayNanos = DefaultWalFsyncDelayNanos; RateTimeInterval = DefaultRateTimeInterval; SubIntervals = DefaultSubIntervals; + WalArchivePath = DefaultWalArchivePath; + WalStorePath = DefaultWalStorePath; } /// @@ -137,7 +143,7 @@ internal PersistentStoreConfiguration(IBinaryRawReader reader) WalMode = (WalMode)reader.ReadInt(); TlbSize = reader.ReadInt(); WalFlushFrequency = reader.ReadLongAsTimespan(); - WalFsyncDelayNanos = reader.ReadInt(); + WalFsyncDelayNanos = reader.ReadLong(); WalRecordIteratorBufferSize = reader.ReadInt(); AlwaysWriteFullPages = reader.ReadBoolean(); MetricsEnabled = reader.ReadBoolean(); @@ -166,7 +172,7 @@ internal void Write(IBinaryRawWriter writer) writer.WriteInt((int)WalMode); writer.WriteInt(TlbSize); writer.WriteTimeSpanAsLong(WalFlushFrequency); - writer.WriteInt(WalFsyncDelayNanos); + writer.WriteLong(WalFsyncDelayNanos); writer.WriteInt(WalRecordIteratorBufferSize); writer.WriteBoolean(AlwaysWriteFullPages); writer.WriteBoolean(MetricsEnabled); @@ -188,8 +194,9 @@ internal void Write(IBinaryRawWriter writer) /// /// Gets or sets the size of the checkpointing page buffer. + /// + /// Default is 0: Ignite will choose buffer size automatically. /// - [DefaultValue(DefaultCheckpointingPageBufferSize)] public long CheckpointingPageBufferSize { get; set; } /// @@ -227,12 +234,14 @@ internal void Write(IBinaryRawWriter writer) /// /// Gets or sets the path to the directory where WAL (Write Ahead Log) is stored. /// + [DefaultValue(DefaultWalStorePath)] public string WalStorePath { get; set; } /// /// Gets or sets the path to the directory where WAL (Write Ahead Log) archive is stored. /// Every WAL segment will be fully copied to this directory before it can be reused for WAL purposes. /// + [DefaultValue(DefaultWalArchivePath)] public string WalArchivePath { get; set; } /// @@ -256,7 +265,7 @@ internal void Write(IBinaryRawWriter writer) /// Gets or sets the WAL (Write Ahead Log) fsync (disk sync) delay, in nanoseconds /// [DefaultValue(DefaultWalFsyncDelayNanos)] - public int WalFsyncDelayNanos { get; set; } + public long WalFsyncDelayNanos { get; set; } /// /// Gets or sets the size of the WAL (Write Ahead Log) record iterator buffer, in bytes. diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js index aa11e0b5d0fa9..57af928bd12c4 100644 --- a/modules/web-console/backend/app/mongo.js +++ b/modules/web-console/backend/app/mongo.js @@ -1006,7 +1006,7 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose walSegmentSize: Number, walHistorySize: Number, walFlushFrequency: Number, - walFsyncDelay: Number, + walFsyncDelayNanos: Number, walRecordIteratorBufferSize: Number, lockWaitTime: Number, rateTimeInterval: Number, diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js index 8c9b14cc12e52..8299b9bb7a1c0 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js @@ -1477,17 +1477,17 @@ export default class IgniteConfigurationGenerator { .boolProperty('metricsEnabled') .boolProperty('alwaysWriteFullPages') .intProperty('checkpointingFrequency') - .intProperty('checkpointingPageBufferSize') + .longProperty('checkpointingPageBufferSize') .intProperty('checkpointingThreads') .stringProperty('walStorePath') .stringProperty('walArchivePath') .intProperty('walSegments') .intProperty('walSegmentSize') .intProperty('walHistorySize') - .intProperty('walFlushFrequency') - .intProperty('walFsyncDelay') + .longProperty('walFlushFrequency') + .longProperty('walFsyncDelayNanos') .intProperty('walRecordIteratorBufferSize') - .intProperty('lockWaitTime') + .longProperty('lockWaitTime') .intProperty('rateTimeInterval') .intProperty('tlbSize') .intProperty('subIntervals'); diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js index a4680bf899887..1c7fc44eee5f7 100644 --- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js +++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js @@ -340,7 +340,7 @@ const DFLT_CLUSTER = { walSegmentSize: 67108864, walHistorySize: 20, walFlushFrequency: 2000, - walFsyncDelay: 1, + walFsyncDelayNanos: 1000, walRecordIteratorBufferSize: 67108864, lockWaitTime: 10000, rateTimeInterval: 60000, diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug index 7fd0966b08b81..50a569b22de18 100644 --- a/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug +++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug @@ -62,7 +62,7 @@ include /app/helpers/jade/mixins +number('WAL flush frequency:', `${model}.walFlushFrequency`, '"PersistenceWalFlushFrequency"', enabled, '2000', '1', 'How often will be fsync, in milliseconds. In background mode, exist thread which do fsync by timeout') .settings-row - +number('WAL fsync delay:', `${model}.walFsyncDelay`, '"PersistenceWalFsyncDelay"', enabled, '1', '1', 'WAL fsync delay, in nanoseconds') + +number('WAL fsync delay:', `${model}.walFsyncDelayNanos`, '"PersistenceWalFsyncDelay"', enabled, '1000', '1', 'WAL fsync delay, in nanoseconds') .settings-row +number('WAL record iterator buffer size:', `${model}.walRecordIteratorBufferSize`, '"PersistenceWalRecordIteratorBufferSize"', enabled, '67108864', '1', 'How many bytes iterator read from disk(for one reading), during go ahead WAL') From 3c1749da82e663500e45a34369eac48dbbc62bdc Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 13 Jul 2017 11:25:55 +0300 Subject: [PATCH 151/155] IGNITE-5744 Ignore non user caches when automatically choosing a queryable cache inside JDBC driver --- .../internal/processors/cache/GridCacheProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 85772d8f9ae64..f7c6e111b3b08 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -1668,6 +1668,9 @@ public Collection cacheNames() { public IgniteCacheProxy getOrStartPublicCache(boolean start, boolean inclLoc) throws IgniteCheckedException { // Try to find started cache first. for (Map.Entry> e : caches.entrySet()) { + if (!e.getValue().context().userCache()) + continue; + CacheConfiguration ccfg = e.getValue().configuration(); String cacheName = ccfg.getName(); @@ -1680,6 +1683,9 @@ public Collection cacheNames() { for (Map.Entry e : cachesInfo.registeredCaches().entrySet()) { DynamicCacheDescriptor desc = e.getValue(); + if (!desc.cacheType().userCache()) + continue; + CacheConfiguration ccfg = desc.cacheConfiguration(); if (ccfg.getCacheMode() != LOCAL) { From 1a2dde91083a2d733e2ccc1c03bc9959caa921af Mon Sep 17 00:00:00 2001 From: Oleg Ostanin Date: Mon, 17 Jul 2017 15:10:36 +0300 Subject: [PATCH 152/155] IGNITE-5762: LGPL modules are no longer deployed into Maven repository. --- examples/pom.xml | 7 +++++++ modules/geospatial/pom.xml | 24 ++++++++++++++++++++++++ modules/hibernate-4.2/pom.xml | 24 ++++++++++++++++++++++++ modules/hibernate-5.1/pom.xml | 24 ++++++++++++++++++++++++ modules/schedule/pom.xml | 24 ++++++++++++++++++++++++ 5 files changed, 103 insertions(+) diff --git a/examples/pom.xml b/examples/pom.xml index 58da3421bb784..60427816bdc61 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -312,6 +312,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + diff --git a/modules/geospatial/pom.xml b/modules/geospatial/pom.xml index c715ffa55f17e..488a4a4a35edc 100644 --- a/modules/geospatial/pom.xml +++ b/modules/geospatial/pom.xml @@ -83,6 +83,23 @@ + + + lgpl + + + + org.apache.maven.plugins + maven-deploy-plugin + + false + + + + + + +