From aaf0be1de5c56141342d9bc04961511caeb1480a Mon Sep 17 00:00:00 2001 From: Dmitrii Ryabov Date: Fri, 12 Oct 2018 12:02:27 +0300 Subject: [PATCH] 9769 test flakiness --- .../atomic/IgniteCacheAtomicProtocolTest.java | 702 +----------------- .../junits/common/GridCommonAbstractTest.java | 25 + .../testsuites/IgniteCacheTestSuite5.java | 48 +- 3 files changed, 34 insertions(+), 741 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java index 14c85717ae857..0bcf64b1b7d51 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java @@ -119,634 +119,6 @@ private void blockRebalance() { super.afterTest(); } - /** - * @throws Exception If failed. - */ - public void testPutAllPrimaryFailure1() throws Exception { - putAllPrimaryFailure(true, false); - } - - /** - * @throws Exception If failed. - */ - public void testPutAllPrimaryFailure1_UnstableTopology() throws Exception { - blockRebalance = true; - - putAllPrimaryFailure(true, false); - } - - /** - * @throws Exception If failed. - */ - public void testPutAllPrimaryFailure2() throws Exception { - putAllPrimaryFailure(true, true); - } - - /** - * @throws Exception If failed. - */ - public void testPutAllPrimaryFailure2_UnstableTopology() throws Exception { - blockRebalance = true; - - putAllPrimaryFailure(true, true); - } - - /** - * @param fail0 Fail node 0 flag. - * @param fail1 Fail node 1 flag. - * @throws Exception If failed. - */ - private void putAllPrimaryFailure(boolean fail0, boolean fail1) throws Exception { - ccfg = cacheConfiguration(1, FULL_SYNC); - - startServers(4); - - client = true; - - Ignite client = startGrid(4); - - IgniteCache nearCache = client.cache(TEST_CACHE); - - if (!blockRebalance) - awaitPartitionMapExchange(); - - Ignite srv0 = ignite(0); - Ignite srv1 = ignite(1); - - Integer key1 = primaryKey(srv0.cache(TEST_CACHE)); - Integer key2 = primaryKey(srv1.cache(TEST_CACHE)); - - Map map = new HashMap<>(); - map.put(key1, key1); - map.put(key2, key2); - - assertEquals(2, map.size()); - - if (fail0) { - testSpi(client).blockMessages(GridNearAtomicFullUpdateRequest.class, srv0.name()); - testSpi(client).blockMessages(GridNearAtomicCheckUpdateRequest.class, srv0.name()); - } - - if (fail1) { - testSpi(client).blockMessages(GridNearAtomicFullUpdateRequest.class, srv1.name()); - testSpi(client).blockMessages(GridNearAtomicCheckUpdateRequest.class, srv1.name()); - } - - log.info("Start put [key1=" + key1 + ", key2=" + key2 + ']'); - - IgniteFuture fut = nearCache.putAllAsync(map); - - U.sleep(500); - - assertFalse(fut.isDone()); - - if (fail0) - stopGrid(0); - - if (fail1) - stopGrid(1); - - fut.get(); - - checkData(map); - } - - /** - * @throws Exception If failed. - */ - public void testPutAllBackupFailure1() throws Exception { - putAllBackupFailure1(); - } - - /** - * @throws Exception If failed. - */ - public void testPutAllBackupFailure1_UnstableTopology() throws Exception { - blockRebalance = true; - - putAllBackupFailure1(); - } - - /** - * @throws Exception If failed. - */ - private void putAllBackupFailure1() throws Exception { - ccfg = cacheConfiguration(1, FULL_SYNC); - - startServers(4); - - client = true; - - Ignite client = startGrid(4); - - IgniteCache nearCache = client.cache(TEST_CACHE); - - if (!blockRebalance) - awaitPartitionMapExchange(); - - Ignite srv0 = ignite(0); - - List keys = primaryKeys(srv0.cache(TEST_CACHE), 3); - - Ignite backup = backup(client.affinity(TEST_CACHE), keys.get(0)); - - testSpi(backup).blockMessages(GridDhtAtomicNearResponse.class, client.name()); - - Map map = new HashMap<>(); - - for (Integer key : keys) - map.put(key, key); - - log.info("Start put [map=" + map + ']'); - - IgniteFuture fut = nearCache.putAllAsync(map); - - U.sleep(500); - - assertFalse(fut.isDone()); - - stopGrid(backup.name()); - - fut.get(); - - checkData(map); - } - - /** - * @throws Exception If failed. - */ - public void testPutBackupFailure1() throws Exception { - putBackupFailure1(); - } - - /** - * @throws Exception If failed. - */ - public void testPutBackupFailure1_UnstableTopology() throws Exception { - blockRebalance = true; - - putBackupFailure1(); - } - - /** - * @throws Exception If failed. - */ - private void putBackupFailure1() throws Exception { - ccfg = cacheConfiguration(1, FULL_SYNC); - - startServers(4); - - client = true; - - Ignite client = startGrid(4); - - IgniteCache nearCache = client.cache(TEST_CACHE); - - if (!blockRebalance) - awaitPartitionMapExchange(); - - Ignite srv0 = ignite(0); - - Integer key = primaryKey(srv0.cache(TEST_CACHE)); - - Ignite backup = backup(client.affinity(TEST_CACHE), key); - - testSpi(backup).blockMessages(GridDhtAtomicNearResponse.class, client.name()); - - log.info("Start put [key=" + key + ']'); - - IgniteFuture fut = nearCache.putAsync(key, key); - - U.sleep(500); - - assertFalse(fut.isDone()); - - stopGrid(backup.name()); - - fut.get(); - - checkData(F.asMap(key, key)); - } - - /** - * @throws Exception If failed. - */ - public void testFullAsyncPutRemap() throws Exception { - fullAsyncRemap(false); - } - - /** - * @throws Exception If failed. - */ - public void testFullAsyncPutAllRemap() throws Exception { - fullAsyncRemap(true); - } - - /** - * @param putAll Test putAll flag. - * @throws Exception If failed. - */ - private void fullAsyncRemap(boolean putAll) throws Exception { - Ignite srv0 = startGrid(0); - - client = true; - - Ignite clientNode = startGrid(1); - - client = false; - - final IgniteCache nearCache = clientNode.createCache(cacheConfiguration(1, FULL_ASYNC)); - - List keys = movingKeysAfterJoin(srv0, TEST_CACHE, putAll ? 10 : 1); - - testSpi(clientNode).blockMessages(GridNearAtomicSingleUpdateRequest.class, srv0.name()); - testSpi(clientNode).blockMessages(GridNearAtomicFullUpdateRequest.class, srv0.name()); - - final Map map = new HashMap<>(); - - for (Integer key : keys) - map.put(key, -key); - - if (putAll) - nearCache.putAll(map); - else - nearCache.put(keys.get(0), map.get(keys.get(0))); - - Affinity aff = clientNode.affinity(TEST_CACHE); - - startGrid(2); - - awaitPartitionMapExchange(); - - int keysMoved = 0; - - for (Integer key : keys) { - if (!aff.isPrimary(srv0.cluster().localNode(), key)) - keysMoved++; - } - - assertEquals(keys.size(), keysMoved); - - testSpi(clientNode).stopBlock(true); - - GridTestUtils.waitForCondition(new GridAbsPredicate() { - @Override public boolean apply() { - for (Integer key : map.keySet()) { - if (nearCache.get(key) == null) - return false; - } - - return true; - } - }, 5000); - - checkData(map); - } - - /** - * @throws Exception If failed. - */ - public void testPutPrimarySync() throws Exception { - startGrids(2); - - client = true; - - Ignite clientNode = startGrid(2); - - client = false; - - final IgniteCache nearCache = clientNode.createCache(cacheConfiguration(1, PRIMARY_SYNC)); - - awaitPartitionMapExchange(); - - Ignite srv0 = grid(0); - final Ignite srv1 = grid(1); - - final Integer key = primaryKey(srv0.cache(TEST_CACHE)); - - testSpi(srv0).blockMessages(GridDhtAtomicSingleUpdateRequest.class, srv1.name()); - - IgniteFuture fut = nearCache.putAsync(key, key); - - fut.get(5, TimeUnit.SECONDS); - - assertEquals(key, srv0.cache(TEST_CACHE).get(key)); - - assertNull(srv1.cache(TEST_CACHE).localPeek(key)); - - testSpi(srv0).stopBlock(true); - - GridTestUtils.waitForCondition(new GridAbsPredicate() { - @Override public boolean apply() { - return srv1.cache(TEST_CACHE).localPeek(key) != null; - } - }, 5000); - - checkData(F.asMap(key, key)); - } - - /** - * @throws Exception If failed. - */ - public void testPutNearNodeFailure() throws Exception { - startGrids(2); - - client = true; - - Ignite clientNode = startGrid(2); - - final IgniteCache nearCache = clientNode.createCache(cacheConfiguration(1, FULL_SYNC)); - - awaitPartitionMapExchange(); - - final Ignite srv0 = grid(0); - final Ignite srv1 = grid(1); - - final Integer key = primaryKey(srv0.cache(TEST_CACHE)); - - nearCache.putAsync(key, key); - - testSpi(srv1).blockMessages(GridDhtAtomicNearResponse.class, clientNode.name()); - - stopGrid(2); - - GridTestUtils.waitForCondition(new GridAbsPredicate() { - @Override public boolean apply() { - return ((IgniteKernal)srv0).context().cache().context().mvcc().atomicFuturesCount() == 0; - } - }, 5000); - - assertEquals(0, ((IgniteKernal)srv0).context().cache().context().mvcc().atomicFuturesCount()); - assertEquals(0, ((IgniteKernal)srv1).context().cache().context().mvcc().atomicFuturesCount()); - - checkData(F.asMap(key, key)); - } - - /** - * @throws Exception If failed. - */ - public void testPutAllNearNodeFailure() throws Exception { - final int SRVS = 4; - - startGrids(SRVS); - - client = true; - - Ignite clientNode = startGrid(SRVS); - - final IgniteCache nearCache = clientNode.createCache(cacheConfiguration(1, FULL_SYNC)); - - awaitPartitionMapExchange(); - - for (int i = 0; i < SRVS; i++) - testSpi(grid(i)).blockMessages(GridDhtAtomicNearResponse.class, clientNode.name()); - - final Map map = new HashMap<>(); - - for (int i = 0; i < 100; i++) - map.put(i, i); - - nearCache.putAllAsync(map); - - boolean wait = GridTestUtils.waitForCondition(new GridAbsPredicate() { - @Override public boolean apply() { - IgniteCache cache = ignite(0).cache(TEST_CACHE); - - for (Integer key : map.keySet()) { - if (cache.get(key) == null) - return false; - } - - return true; - } - }, 5000); - - assertTrue(wait); - - stopGrid(SRVS); - - GridTestUtils.waitForCondition(new GridAbsPredicate() { - @Override public boolean apply() { - for (int i = 0; i < SRVS; i++) { - if (grid(i).context().cache().context().mvcc().atomicFuturesCount() != 0) - return false; - } - - return true; - } - }, 5000); - - for (int i = 0; i < SRVS; i++) - assertEquals(0, grid(i).context().cache().context().mvcc().atomicFuturesCount()); - - checkData(map); - } - - /** - * @throws Exception If failed. - */ - public void testCacheOperations0() throws Exception { - cacheOperations(0); - } - - /** - * @throws Exception If failed. - */ - public void testCacheOperations_UnstableTopology0() throws Exception { - blockRebalance = true; - - cacheOperations(0); - } - - /** - * @throws Exception If failed. - */ - public void testCacheOperations1() throws Exception { - cacheOperations(1); - } - - /** - * @throws Exception If failed. - */ - public void testCacheOperations_UnstableTopology1() throws Exception { - blockRebalance = true; - - cacheOperations(1); - } - - /** - * @throws Exception If failed. - */ - public void testCacheOperations2() throws Exception { - cacheOperations(2); - } - - /** - * @throws Exception If failed. - */ - public void testCacheOperations_UnstableTopology2() throws Exception { - blockRebalance = true; - - cacheOperations(2); - } - - /** - * @param backups Number of backups. - * @throws Exception If failed. - */ - private void cacheOperations(int backups) throws Exception { - ccfg = cacheConfiguration(backups, FULL_SYNC); - - final int SRVS = 4; - - startServers(SRVS); - - client = true; - - Ignite clientNode = startGrid(SRVS); - - final IgniteCache nearCache = clientNode.cache(TEST_CACHE); - - Integer key = primaryKey(ignite(0).cache(TEST_CACHE)); - - nearCache.replace(key, 1); - - nearCache.remove(key); - - nearCache.invoke(key, new SetValueEntryProcessor(null)); - - Map map = new HashMap<>(); - - List keys = primaryKeys(ignite(0).cache(TEST_CACHE), 2); - - map.put(keys.get(0), new SetValueEntryProcessor(1)); - map.put(keys.get(1), new SetValueEntryProcessor(null)); - - nearCache.invokeAll(map); - - Set rmvAllKeys = new HashSet<>(); - - for (int i = 0; i < 100; i++) { - nearCache.put(i, i); - - if (i % 2 == 0) - rmvAllKeys.add(i); - } - - nearCache.removeAll(rmvAllKeys); - } - - /** - * @throws Exception If failed. - */ - public void testPutMissedDhtRequest_UnstableTopology() throws Exception { - blockRebalance = true; - - ccfg = cacheConfiguration(1, FULL_SYNC); - - startServers(4); - - client = true; - - Ignite client = startGrid(4); - - IgniteCache nearCache = client.cache(TEST_CACHE); - - testSpi(ignite(0)).blockMessages(new IgniteBiPredicate() { - @Override public boolean apply(ClusterNode node, Message msg) { - return msg instanceof GridDhtAtomicAbstractUpdateRequest; - } - }); - - Integer key = primaryKey(ignite(0).cache(TEST_CACHE)); - - log.info("Start put [key=" + key + ']'); - - IgniteFuture fut = nearCache.putAsync(key, key); - - U.sleep(500); - - assertFalse(fut.isDone()); - - stopGrid(0); - - fut.get(); - - checkData(F.asMap(key, key)); - } - - /** - * @throws Exception If failed. - */ - public void testPutAllMissedDhtRequest_UnstableTopology1() throws Exception { - putAllMissedDhtRequest_UnstableTopology(true, false); - } - - /** - * @throws Exception If failed. - */ - public void testPutAllMissedDhtRequest_UnstableTopology2() throws Exception { - putAllMissedDhtRequest_UnstableTopology(true, true); - } - - /** - * @param fail0 Fail node 0 flag. - * @param fail1 Fail node 1 flag. - * @throws Exception If failed. - */ - private void putAllMissedDhtRequest_UnstableTopology(boolean fail0, boolean fail1) throws Exception { - blockRebalance = true; - - ccfg = cacheConfiguration(1, FULL_SYNC); - - startServers(4); - - client = true; - - Ignite client = startGrid(4); - - IgniteCache nearCache = client.cache(TEST_CACHE); - - if (fail0) { - testSpi(ignite(0)).blockMessages(new IgniteBiPredicate() { - @Override public boolean apply(ClusterNode node, Message msg) { - return msg instanceof GridDhtAtomicAbstractUpdateRequest; - } - }); - } - if (fail1) { - testSpi(ignite(2)).blockMessages(new IgniteBiPredicate() { - @Override public boolean apply(ClusterNode node, Message msg) { - return msg instanceof GridDhtAtomicAbstractUpdateRequest; - } - }); - } - - Integer key1 = primaryKey(ignite(0).cache(TEST_CACHE)); - Integer key2 = primaryKey(ignite(2).cache(TEST_CACHE)); - - log.info("Start put [key1=" + key1 + ", key2=" + key1 + ']'); - - Map map = new HashMap<>(); - map.put(key1, 10); - map.put(key2, 20); - - IgniteFuture fut = nearCache.putAllAsync(map); - - U.sleep(500); - - assertFalse(fut.isDone()); - - if (fail0) - stopGrid(0); - if (fail1) - stopGrid(2); - - fut.get(); - - checkData(map); - } - /** * @throws Exception If failed. */ @@ -769,28 +141,6 @@ public void testPutReaderUpdate2() throws Exception { readerUpdateDhtFails(true, true, false); } - /** - * @throws Exception If failed. - */ - public void testPutAllReaderUpdate1() throws Exception { - readerUpdateDhtFails(false, false, true); - - stopAllGrids(); - - readerUpdateDhtFails(false, true, true); - } - - /** - * @throws Exception If failed. - */ - public void testPutAllReaderUpdate2() throws Exception { - readerUpdateDhtFails(true, false, true); - - stopAllGrids(); - - readerUpdateDhtFails(true, true, true); - } - /** * @param updateNearEnabled {@code True} if enable near cache for second put. * @param delayReader If {@code true} delay reader response, otherwise delay backup response. @@ -806,13 +156,16 @@ private void readerUpdateDhtFails(boolean updateNearEnabled, startServers(2); - // Waiting for minor topology changing because of late affinity assignment. - awaitPartitionMapExchange(); - Ignite srv0 = ignite(0); Ignite srv1 = ignite(1); - List keys = primaryKeys(srv0.cache(TEST_CACHE), putAll ? 3 : 1); + IgniteCache cache = srv0.cache(TEST_CACHE); + + // Waiting for minor topology changing because of late affinity assignment. + awaitPartitionMapExchange(true, true, null); + awaitForAffinityTopology(2, 1); + + List keys = primaryKeys(cache, putAll ? 3 : 1); ccfg = null; @@ -882,24 +235,6 @@ private void checkData(Map expData) { checkCacheData(expData, TEST_CACHE); } - /** - * @param aff Affinity. - * @param key Key. - * @return Backup node for given key. - */ - private Ignite backup(Affinity aff, Object key) { - for (Ignite ignite : G.allGrids()) { - ClusterNode node = ignite.cluster().localNode(); - - if (aff.isPrimaryOrBackup(node, key) && !aff.isPrimary(node, key)) - return ignite; - } - - fail("Failed to find backup for key: " + key); - - return null; - } - /** * @param node Node. * @return Node communication SPI. @@ -940,27 +275,4 @@ private void startServers(int cnt) throws Exception { startGrid(cnt - 1); } - - /** - * - */ - public static class SetValueEntryProcessor implements CacheEntryProcessor { - /** */ - private Integer val; - - /** - * @param val Value. - */ - SetValueEntryProcessor(Integer val) { - this.val = val; - } - - /** {@inheritDoc} */ - @Override public Object process(MutableEntry entry, Object... args) { - if (val != null) - entry.setValue(val); - - return null; - } - } } diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java index 7e98ec763e3aa..207e663848310 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java @@ -77,6 +77,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; import org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl; +import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; @@ -797,6 +798,30 @@ protected void awaitPartitionMapExchange( log.info("awaitPartitionMapExchange finished"); } + /** + * @param topVer Topology version. + * @param minorTopVer Minor topology version + * @throws IgniteCheckedException If failed to wait. + */ + protected void awaitForAffinityTopology(int topVer, int minorTopVer) throws IgniteCheckedException { + awaitForAffinityTopology(new AffinityTopologyVersion(topVer, minorTopVer)); + } + + /** + * @param ver Affinity topology version. + * @throws IgniteCheckedException If failed to wait. + */ + protected void awaitForAffinityTopology(AffinityTopologyVersion ver) throws IgniteCheckedException { + new AffinityTopologyVersion(2, 1); + for (Ignite ignite : G.allGrids()) { + IgniteInternalFuture fut = ((IgniteEx)ignite).context().cache().context().exchange() + .affinityReadyFuture(ver); + + if (fut != null) + fut.get(getTestTimeout()); + } + } + /** * @param top Topology. * @param topVer Version to wait for. diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java index a583317824b6f..0a0146ce8591d 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java @@ -63,53 +63,9 @@ public static TestSuite suite() throws Exception { TestSuite suite = new TestSuite("IgniteCache Test Suite part 5"); - suite.addTestSuite(CacheSerializableTransactionsTest.class); - suite.addTestSuite(CacheNearReaderUpdateTest.class); - suite.addTestSuite(IgniteCacheStoreCollectionTest.class); - suite.addTestSuite(IgniteCacheWriteBehindNoUpdateSelfTest.class); - suite.addTestSuite(IgniteCachePutStackOverflowSelfTest.class); - suite.addTestSuite(CacheKeepBinaryTransactionTest.class); + for (int i = 0; i < 100; i++) + suite.addTestSuite(IgniteCacheAtomicProtocolTest.class); - suite.addTestSuite(CacheLateAffinityAssignmentTest.class); - suite.addTestSuite(CacheLateAffinityAssignmentNodeJoinValidationTest.class); - suite.addTestSuite(EntryVersionConsistencyReadThroughTest.class); - suite.addTestSuite(IgniteCacheSyncRebalanceModeSelfTest.class); - - suite.addTest(IgniteCacheReadThroughEvictionsVariationsSuite.suite()); - suite.addTestSuite(IgniteCacheTxIteratorSelfTest.class); - - suite.addTestSuite(ClusterStatePartitionedSelfTest.class); - suite.addTestSuite(ClusterStateReplicatedSelfTest.class); - suite.addTestSuite(ClusterReadOnlyModeTest.class); - suite.addTestSuite(IgniteCachePartitionLossPolicySelfTest.class); - suite.addTestSuite(IgniteCacheGroupsPartitionLossPolicySelfTest.class); - - suite.addTestSuite(CacheRebalancingSelfTest.class); - suite.addTestSuite(CacheManualRebalancingTest.class); - - // Affinity tests. - suite.addTestSuite(GridCacheAffinityBackupsSelfTest.class); - suite.addTestSuite(IgniteCacheAffinitySelfTest.class); - suite.addTestSuite(AffinityClientNodeSelfTest.class); - suite.addTestSuite(LocalAffinityFunctionTest.class); - suite.addTestSuite(AffinityHistoryCleanupTest.class); - - suite.addTestSuite(AffinityDistributionLoggingTest.class); - - suite.addTestSuite(IgniteCacheAtomicProtocolTest.class); - - suite.addTestSuite(PartitionsExchangeOnDiscoveryHistoryOverflowTest.class); - - suite.addTestSuite(GridCachePartitionExchangeManagerHistSizeTest.class); - - suite.addTestSuite(NotMappedPartitionInTxTest.class); - - suite.addTestSuite(ConcurrentCacheStartTest.class); - - - //suite.addTestSuite(GridCacheAtomicPreloadSelfTest.class); - //suite.addTestSuite(IgniteCacheContainsKeyColocatedAtomicSelfTest.class); - //suite.addTestSuite(IgniteCacheContainsKeyNearAtomicSelfTest.class); return suite; } }