From 312e76d2e85917cfcda530cc40a2c8de74310e0d Mon Sep 17 00:00:00 2001 From: sboikov Date: Tue, 9 Jul 2019 13:42:56 +0300 Subject: [PATCH 01/30] ignite-11704 --- .../processors/cache/CacheGroupContext.java | 9 + .../processors/cache/GridCacheContext.java | 4 +- .../processors/cache/GridCacheMapEntry.java | 195 ++++++-------- .../cache/IgniteCacheOffheapManager.java | 43 +++- .../cache/IgniteCacheOffheapManagerImpl.java | 243 +++++++++++++++--- .../cache/persistence/CacheDataRow.java | 5 + .../persistence/CacheDataRowAdapter.java | 5 + .../persistence/GridCacheOffheapManager.java | 19 +- .../processors/cache/tree/DataRow.java | 6 +- .../GridCacheAbstractFullApiSelfTest.java | 2 +- .../database/CacheFreeListSelfTest.java | 5 + .../query/h2/database/H2PkHashIndex.java | 2 +- .../processors/query/h2/opt/H2CacheRow.java | 5 + 13 files changed, 364 insertions(+), 179 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index 4fc43fdd3e6dd..56c2449ced03c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -46,6 +46,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentRequest; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; @@ -1299,6 +1300,14 @@ public boolean hasAtomicCaches() { return hasAtomicCaches; } + public boolean supportsTombstone() { + return !hasAtomicCaches && !mvccEnabled && !isLocal(); + } + + public boolean createTombstone(@Nullable GridDhtLocalPartition part) { + return part != null && supportsTombstone(); + } + /** * @return Metrics. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 9d52c752c78ea..87c62534821b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -625,8 +625,8 @@ public byte ioPolicy() { public void cache(GridCacheAdapter cache) { this.cache = cache; - deferredDel = cache.isDht() || cache.isDhtAtomic() || cache.isColocated() || - (cache.isNear() && cache.configuration().getAtomicityMode() == ATOMIC); + deferredDel = !grp.supportsTombstone() && (cache.isDht() || cache.isDhtAtomic() || cache.isColocated() || + (cache.isNear() && cache.configuration().getAtomicityMode() == ATOMIC)); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index f1b7ec76b2152..adc86994e4b66 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -98,9 +98,9 @@ import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.thread.IgniteThread; import org.jetbrains.annotations.NotNull; @@ -1713,13 +1713,14 @@ protected Object keyValue(boolean cpy) { interceptRes = cctx.config().getInterceptor().onBeforeRemove(entry0); if (cctx.cancelRemove(interceptRes)) { - CacheObject ret = cctx.toCacheObject(cctx.unwrapTemporary(interceptRes.get2())); - return new GridCacheUpdateTxResult(false, logPtr); } } - removeValue(); + if (cctx.group().createTombstone(localPartition())) + cctx.offheap().removeWithTombstone(cctx, key, newVer, partition(), localPartition()); + else + removeValue(); update(null, 0, 0, newVer, true); @@ -3335,12 +3336,10 @@ private boolean skipInterceptor(@Nullable GridCacheVersion explicitVer) { boolean update; - IgnitePredicate p = new IgnitePredicate() { - @Override public boolean apply(@Nullable CacheDataRow row) { + IgniteBiPredicate p = new IgniteBiPredicate() { + @Override public boolean apply(@Nullable CacheObject val, GridCacheVersion currentVer) { boolean update0; - GridCacheVersion currentVer = row != null ? row.version() : GridCacheMapEntry.this.ver; - boolean isStartVer = cctx.shared().versions().isStartVersion(currentVer); if (cctx.group().persistenceEnabled()) { @@ -3356,14 +3355,14 @@ private boolean skipInterceptor(@Nullable GridCacheVersion explicitVer) { else update0 = isStartVer; - update0 |= (!preload && deletedUnlocked()); + update0 |= (!preload && val == null); return update0; } }; if (unswapped) { - update = p.apply(null); + update = p.apply(this.val, this.ver); if (update) { // If entry is already unswapped and we are modifying it, we must run deletion callbacks for old value. @@ -3394,7 +3393,7 @@ else if (val == null) // cannot identify whether the entry is exist on the fly unswap(false); - if (update = p.apply(null)) { + if (update = p.apply(this.val, this.ver)) { // If entry is already unswapped and we are modifying it, we must run deletion callbacks for old value. long oldExpTime = expireTimeUnlocked(); long delta = (oldExpTime == 0 ? 0 : oldExpTime - U.currentTimeMillis()); @@ -4266,7 +4265,7 @@ protected boolean storeValue(@Nullable CacheObject val, * @param val Value. * @param expireTime Expire time. * @param ver New entry version. - * @param predicate Optional predicate. + * @param p Optional predicate. * @return {@code True} if storage was modified. * @throws IgniteCheckedException If update failed. */ @@ -4274,10 +4273,10 @@ protected boolean storeValue( @Nullable CacheObject val, long expireTime, GridCacheVersion ver, - @Nullable IgnitePredicate predicate) throws IgniteCheckedException { + @Nullable IgniteBiPredicate p) throws IgniteCheckedException { assert lock.isHeldByCurrentThread(); - UpdateClosure closure = new UpdateClosure(this, val, ver, expireTime, predicate); + UpdateClosure closure = new UpdateClosure(this, val, ver, expireTime, p); cctx.offheap().invoke(cctx, key, localPartition(), closure); @@ -5680,6 +5679,44 @@ private LazyValueEntry(KeyCacheObject key, boolean keepBinary) { } } + private boolean checkRowExpired(CacheDataRow row) throws IgniteCheckedException { + assert row != null; + + if (!(row.expireTime() > 0 && row.expireTime() <= U.currentTimeMillis())) + return false; + + CacheObject expiredVal = row.value(); + + if (cctx.deferredDelete() && !detached() && !isInternal()) { + update(null, CU.TTL_ETERNAL, CU.EXPIRE_TIME_ETERNAL, ver, true); + + if (!deletedUnlocked()) + deletedUnlocked(true); + } + else + markObsolete0(cctx.versions().next(), true, null); + + if (cctx.events().isRecordable(EVT_CACHE_OBJECT_EXPIRED)) { + cctx.events().addEvent(partition(), + key(), + cctx.localNodeId(), + null, + EVT_CACHE_OBJECT_EXPIRED, + null, + false, + expiredVal, + expiredVal != null, + null, + null, + null, + true); + } + + cctx.continuousQueries().onEntryExpired(this, key(), expiredVal); + + return true; + } + /** * */ @@ -5697,7 +5734,7 @@ private static class UpdateClosure implements IgniteCacheOffheapManager.OffheapI private final long expireTime; /** */ - @Nullable private final IgnitePredicate predicate; + @Nullable private final IgniteBiPredicate p; /** */ private CacheDataRow newRow; @@ -5715,29 +5752,41 @@ private static class UpdateClosure implements IgniteCacheOffheapManager.OffheapI * @param expireTime New expire time. * @param predicate Optional predicate. */ - UpdateClosure(GridCacheMapEntry entry, @Nullable CacheObject val, GridCacheVersion ver, long expireTime, - @Nullable IgnitePredicate predicate) { + UpdateClosure(GridCacheMapEntry entry, + @Nullable CacheObject val, + GridCacheVersion ver, + long expireTime, + @Nullable IgniteBiPredicate p) { this.entry = entry; this.val = val; this.ver = ver; this.expireTime = expireTime; - this.predicate = predicate; + this.p = p; } /** {@inheritDoc} */ @Override public void call(@Nullable CacheDataRow oldRow) throws IgniteCheckedException { - if (oldRow != null) { + if (oldRow != null) oldRow.key(entry.key); - oldRow = checkRowExpired(oldRow); - } - this.oldRow = oldRow; - if (predicate != null && !predicate.apply(oldRow)) { - treeOp = IgniteTree.OperationType.NOOP; + if (p != null) { + CacheObject val = null; + GridCacheVersion ver = entry.ver; - return; + if (oldRow != null) { + if (!entry.checkRowExpired(oldRow) && !entry.context().offheap().isTombstone(oldRow)) + val = oldRow.value(); + + ver = oldRow.version(); + } + + if (!p.apply(val, ver)) { + treeOp = IgniteTree.OperationType.NOOP; + + return; + } } if (val != null) { @@ -5770,53 +5819,6 @@ private static class UpdateClosure implements IgniteCacheOffheapManager.OffheapI @Nullable @Override public CacheDataRow oldRow() { return oldRow; } - - /** - * Checks row for expiration and fire expire events if needed. - * - * @param row old row. - * @return {@code Null} if row was expired, row itself otherwise. - * @throws IgniteCheckedException - */ - private CacheDataRow checkRowExpired(CacheDataRow row) throws IgniteCheckedException { - assert row != null; - - if (!(row.expireTime() > 0 && row.expireTime() <= U.currentTimeMillis())) - return row; - - GridCacheContext cctx = entry.context(); - - CacheObject expiredVal = row.value(); - - if (cctx.deferredDelete() && !entry.detached() && !entry.isInternal()) { - entry.update(null, CU.TTL_ETERNAL, CU.EXPIRE_TIME_ETERNAL, entry.ver, true); - - if (!entry.deletedUnlocked() && !entry.isStartVersion()) - entry.deletedUnlocked(true); - } - else - entry.markObsolete0(cctx.versions().next(), true, null); - - if (cctx.events().isRecordable(EVT_CACHE_OBJECT_EXPIRED)) { - cctx.events().addEvent(entry.partition(), - entry.key(), - cctx.localNodeId(), - null, - EVT_CACHE_OBJECT_EXPIRED, - null, - false, - expiredVal, - expiredVal != null, - null, - null, - null, - true); - } - - cctx.continuousQueries().onEntryExpired(entry, entry.key(), expiredVal); - - return null; - } } /** @@ -5991,7 +5993,7 @@ private static class AtomicCacheUpdateClosure implements IgniteCacheOffheapManag // unswap entry.update(oldRow.value(), oldRow.expireTime(), 0, oldRow.version(), false); - if (checkRowExpired(oldRow)) { + if (entry.checkRowExpired(oldRow)) { oldRowExpiredFlag = true; oldRow = null; @@ -6142,53 +6144,6 @@ else if ((invokeRes == null || invokeRes.getValue() == null) && writeObj != null assert updateRes != null && treeOp != null; } - /** - * Check row expiration and fire expire events if needed. - * - * @param row Old row. - * @return {@code True} if row was expired, {@code False} otherwise. - * @throws IgniteCheckedException if failed. - */ - private boolean checkRowExpired(CacheDataRow row) throws IgniteCheckedException { - assert row != null; - - if (!(row.expireTime() > 0 && row.expireTime() <= U.currentTimeMillis())) - return false; - - GridCacheContext cctx = entry.context(); - - CacheObject expiredVal = row.value(); - - if (cctx.deferredDelete() && !entry.detached() && !entry.isInternal()) { - entry.update(null, CU.TTL_ETERNAL, CU.EXPIRE_TIME_ETERNAL, entry.ver, true); - - if (!entry.deletedUnlocked()) - entry.deletedUnlocked(true); - } - else - entry.markObsolete0(cctx.versions().next(), true, null); - - if (cctx.events().isRecordable(EVT_CACHE_OBJECT_EXPIRED)) { - cctx.events().addEvent(entry.partition(), - entry.key(), - cctx.localNodeId(), - null, - EVT_CACHE_OBJECT_EXPIRED, - null, - false, - expiredVal, - expiredVal != null, - null, - null, - null, - true); - } - - cctx.continuousQueries().onEntryExpired(entry, entry.key(), expiredVal); - - return true; - } - /** * @param storeLoadedVal Value loaded from store. * @param updateExpireTime {@code True} if need update expire time. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index ab8d338e46b05..4221a2d13d297 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -157,11 +157,6 @@ public interface IgniteCacheOffheapManager { */ public void destroyCacheDataStore(CacheDataStore store) throws IgniteCheckedException; - /** - * TODO: GG-10884, used on only from initialValue. - */ - public boolean containsKey(GridCacheMapEntry entry); - /** * @param cctx Cache context. * @param c Closure. @@ -400,6 +395,23 @@ public void remove( GridDhtLocalPartition part ) throws IgniteCheckedException; + /** + * @param cctx Cache context. + * @param key Key. + * @param partId Partition number. + * @param part Partition. + * @throws IgniteCheckedException If failed. + */ + public void removeWithTombstone( + GridCacheContext cctx, + KeyCacheObject key, + GridCacheVersion ver, + int partId, + GridDhtLocalPartition part + ) throws IgniteCheckedException; + + public boolean isTombstone(@Nullable CacheDataRow row) throws IgniteCheckedException; + /** * @param ldr Class loader. * @return Number of undeployed entries. @@ -707,7 +719,7 @@ public int cleanup(GridCacheContext cctx, @Nullable List * * @param cctx Cache context. * @param row Row. - * @throws IgniteCheckedException + * @throws IgniteCheckedException If failed. */ public void updateTxState(GridCacheContext cctx, CacheSearchRow row) throws IgniteCheckedException; @@ -880,7 +892,7 @@ MvccUpdateResult mvccLock( * @param ver Version. * @param expireTime Expire time. * @param mvccVer Mvcc version. - * @throws IgniteCheckedException + * @throws IgniteCheckedException If failed. */ void mvccApplyUpdate(GridCacheContext cctx, KeyCacheObject key, @@ -898,6 +910,15 @@ void mvccApplyUpdate(GridCacheContext cctx, */ public void remove(GridCacheContext cctx, KeyCacheObject key, int partId) throws IgniteCheckedException; + /** + * @param cctx Cache context. + * @param key Key. + * @param ver Version. + * @param partId Partition number. + * @throws IgniteCheckedException If failed. + */ + public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, int partId) throws IgniteCheckedException; + /** * @param cctx Cache context. * @param key Key. @@ -1004,8 +1025,12 @@ public GridCursor cursor(int cacheId, KeyCacheObject low * @return Data cursor. * @throws IgniteCheckedException If failed. */ - public GridCursor cursor(int cacheId, KeyCacheObject lower, - KeyCacheObject upper, Object x, MvccSnapshot snapshot) throws IgniteCheckedException; + public GridCursor cursor(int cacheId, + KeyCacheObject lower, + KeyCacheObject upper, + Object x, + MvccSnapshot snapshot, + boolean withTombstones) throws IgniteCheckedException; /** * Destroys the tree associated with the store. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index acd9ca04cad93..e62834591b92e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -136,6 +137,7 @@ import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.unexpectedStateException; import static org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.EMPTY_CURSOR; import static org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO.MVCC_INFO_SIZE; +import static org.apache.ignite.internal.util.IgniteTree.OperationType.IN_PLACE; import static org.apache.ignite.internal.util.IgniteTree.OperationType.NOOP; import static org.apache.ignite.internal.util.IgniteTree.OperationType.PUT; @@ -177,6 +179,9 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager /** */ protected GridStripedLock partStoreLock = new GridStripedLock(Runtime.getRuntime().availableProcessors()); + /** */ + private CacheObject NULL_VAL; + /** {@inheritDoc} */ @Override public GridAtomicLong globalRemoveId() { return globalRmvId; @@ -198,6 +203,8 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager if (grp.isLocal()) locCacheDataStore = createCacheDataStore(0); + + NULL_VAL = new CacheObjectImpl(null, ctx.marshaller().marshal(null)); } finally { ctx.database().checkpointReadUnlock(); @@ -620,6 +627,35 @@ private Iterator cacheData(boolean primary, boolean backup, Affi dataStore(part).remove(cctx, key, partId); } + /** {@inheritDoc} */ + @Override public void removeWithTombstone( + GridCacheContext cctx, + KeyCacheObject key, + GridCacheVersion ver, + int partId, + GridDhtLocalPartition part) throws IgniteCheckedException { + dataStore(part).removeWithTombstone(cctx, key, ver, partId); + } + + @Override public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { + if (row == null || !grp.supportsTombstone()) + return false; + + CacheObject val = row.value(); + + assert val != null : row; + + if (val.cacheObjectType() == CacheObject.TYPE_REGULAR) { + byte[] null_bytes = NULL_VAL.valueBytes(null); + byte[] bytes = val.valueBytes(null); + + if (Arrays.equals(null_bytes, bytes)) + return true; + } + + return false; + } + /** {@inheritDoc} */ @Override @Nullable public CacheDataRow read(GridCacheMapEntry entry) throws IgniteCheckedException { @@ -687,18 +723,6 @@ private Iterator cacheData(boolean primary, boolean backup, Affi return part != null ? dataStore(part) : null; } - /** {@inheritDoc} */ - @Override public boolean containsKey(GridCacheMapEntry entry) { - try { - return read(entry) != null; - } - catch (IgniteCheckedException e) { - U.error(log, "Failed to read value", e); - - return false; - } - } - /** {@inheritDoc} */ @Override public void onPartitionCounterUpdated(int part, long cntr) { // No-op. @@ -1624,9 +1648,6 @@ private boolean canUpdateOldRow(GridCacheContext cctx, @Nullable CacheDataRow ol if (oldRow.expireTime() != dataRow.expireTime()) return false; - // Use grp.sharedGroup() flag since it is possible cacheId is not yet set here. - boolean sizeWithCacheId = grp.sharedGroup(); - int oldLen = oldRow.size(); if (oldLen > updateValSizeThreshold) @@ -1679,13 +1700,21 @@ private void invoke0(GridCacheContext cctx, CacheSearchRow row, OffheapInvokeClo case REMOVE: { CacheDataRow oldRow = c.oldRow(); - finishRemove(cctx, row.key(), oldRow); + finishRemove(cctx, row.key(), oldRow, true); break; } - case NOOP: case IN_PLACE: + if (isTombstone(c.newRow())) + decrementSize(cctx.cacheId()); + + if (isTombstone(c.oldRow())) + incrementSize(cctx.cacheId()); + + break; + + case NOOP: break; default: @@ -1703,6 +1732,10 @@ private void invoke0(GridCacheContext cctx, CacheSearchRow row, OffheapInvokeClo @Nullable CacheDataRow oldRow) throws IgniteCheckedException { int cacheId = grp.storeCacheIdInDataPage() ? cctx.cacheId() : CU.UNDEFINED_CACHE_ID; + // Set real stored cacheId to properly calculate row size. + if (oldRow != null) + oldRow.cacheId(cacheId); + DataRow dataRow = makeDataRow(key, val, ver, expireTime, cacheId); if (canUpdateOldRow(cctx, oldRow, dataRow) && rowStore.updateRow(oldRow.link(), dataRow, grp.statisticsHolderData())) @@ -1718,8 +1751,13 @@ private void invoke0(GridCacheContext cctx, CacheSearchRow row, OffheapInvokeClo assert dataRow.link() != 0 : dataRow; - if (grp.sharedGroup() && dataRow.cacheId() == CU.UNDEFINED_CACHE_ID) - dataRow.cacheId(cctx.cacheId()); + if (grp.sharedGroup()) { + if (dataRow.cacheId() == CU.UNDEFINED_CACHE_ID) + dataRow.cacheId(cctx.cacheId()); + + if (oldRow != null && oldRow.cacheId() == CU.UNDEFINED_CACHE_ID) + oldRow.cacheId(cctx.cacheId()); + } return dataRow; } @@ -2554,7 +2592,9 @@ private int cleanup0(GridCacheContext cctx, @Nullable List cursor() throws IgniteCheckedException { - return dataTree.find(null, null); + return cursorSkipTombstone(dataTree.find(null, null)); + } + + private GridCursor cursorSkipTombstone(final GridCursor cur) { + if (!grp.supportsTombstone()) + return cur; + + return new GridCursor() { + CacheDataRow next; + + @Override public boolean next() throws IgniteCheckedException { + while (cur.next()) { + CacheDataRow next = cur.get(); + + if (!isTombstone(next)) { + this.next = next; + + return true; + } + } + + return false; + } + + @Override public CacheDataRow get() { + return next; + } + }; } /** {@inheritDoc} */ @Override public GridCursor cursor(Object x) throws IgniteCheckedException { - return dataTree.find(null, null, x); + return cursorSkipTombstone(dataTree.find(null, null, x)); } /** {@inheritDoc} */ @Override public GridCursor cursor(MvccSnapshot mvccSnapshot) throws IgniteCheckedException { - GridCursor cursor; if (mvccSnapshot != null) { assert grp.mvccEnabled(); @@ -2803,7 +2958,7 @@ private void afterRowFound(@Nullable CacheDataRow row, KeyCacheObject key) throw new MvccFirstVisibleRowTreeClosure(grp.singleCacheContext(), mvccSnapshot), null); } else - cursor = dataTree.find(null, null); + cursor = cursorSkipTombstone(dataTree.find(null, null)); return cursor; } @@ -2816,7 +2971,7 @@ private void afterRowFound(@Nullable CacheDataRow row, KeyCacheObject key) throw /** {@inheritDoc} */ @Override public GridCursor cursor(int cacheId, MvccSnapshot mvccSnapshot) throws IgniteCheckedException { - return cursor(cacheId, null, null, null, mvccSnapshot); + return cursor(cacheId, null, null, null, mvccSnapshot, false); } /** {@inheritDoc} */ @@ -2828,12 +2983,16 @@ private void afterRowFound(@Nullable CacheDataRow row, KeyCacheObject key) throw /** {@inheritDoc} */ @Override public GridCursor cursor(int cacheId, KeyCacheObject lower, KeyCacheObject upper, Object x) throws IgniteCheckedException { - return cursor(cacheId, lower, upper, null, null); + return cursor(cacheId, lower, upper, null, null, false); } /** {@inheritDoc} */ - @Override public GridCursor cursor(int cacheId, KeyCacheObject lower, - KeyCacheObject upper, Object x, MvccSnapshot snapshot) throws IgniteCheckedException { + @Override public GridCursor cursor(int cacheId, + KeyCacheObject lower, + KeyCacheObject upper, + Object x, + MvccSnapshot snapshot, + boolean withTombstones) throws IgniteCheckedException { SearchRow lowerRow; SearchRow upperRow; @@ -2857,9 +3016,13 @@ private void afterRowFound(@Nullable CacheDataRow row, KeyCacheObject key) throw cursor = dataTree.find(lowerRow, upperRow, new MvccFirstVisibleRowTreeClosure(cctx, snapshot), x); } - else + else { cursor = dataTree.find(lowerRow, upperRow, x); + if (!withTombstones) + cursor = cursorSkipTombstone(cursor); + } + return cursor; } @@ -2899,7 +3062,7 @@ private void afterRowFound(@Nullable CacheDataRow row, KeyCacheObject key) throw Exception ex = null; GridCursor cur = - cursor(cacheId, null, null, CacheDataRowAdapter.RowData.KEY_ONLY); + cursor(cacheId, null, null, CacheDataRowAdapter.RowData.KEY_ONLY, null, true); while (cur.next()) { CacheDataRow row = cur.get(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java index 746b94aa4d805..0b7c4acd3631f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java @@ -59,6 +59,11 @@ public interface CacheDataRow extends MvccUpdateVersionAware, CacheSearchRow, St */ public void key(KeyCacheObject key); + /** + * @param cacheId Cache ID. + */ + public void cacheId(int cacheId); + /** {@inheritDoc} */ @Override public default IOVersions ioVersions() { return DataPageIO.VERSIONS; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index 06e7214056e4b..f27a311f210fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -820,6 +820,11 @@ public boolean isReady() { return cacheId; } + /** {@inheritDoc} */ + @Override public void cacheId(int cacheId) { + this.cacheId = cacheId; + } + /** {@inheritDoc} */ @Override public CacheObject value() { assert val != null : "Value is not ready: " + this; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 4ebcf617e8254..5fa62c4c752f0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -1486,6 +1486,11 @@ private DataEntryRow(DataEntry entry) { return entry.cacheId(); } + /** {@inheritDoc} */ + @Override public void cacheId(int cacheId) { + throw new UnsupportedOperationException(); + } + /** {@inheritDoc} */ @Override public long mvccCoordinatorVersion() { return 0; // TODO IGNITE-7384 @@ -2417,6 +2422,15 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { delegate.remove(cctx, key, partId); } + /** {@inheritDoc} */ + @Override public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, int partId) throws IgniteCheckedException { + assert ctx.database().checkpointLockIsHeldByThread(); + + CacheDataStore delegate = init0(false); + + delegate.removeWithTombstone(cctx, key, ver, partId); + } + /** {@inheritDoc} */ @Override public CacheDataRow find(GridCacheContext cctx, KeyCacheObject key) throws IgniteCheckedException { CacheDataStore delegate = init0(true); @@ -2524,12 +2538,13 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { KeyCacheObject lower, KeyCacheObject upper, Object x, - MvccSnapshot mvccSnapshot) + MvccSnapshot mvccSnapshot, + boolean withTombstones) throws IgniteCheckedException { CacheDataStore delegate = init0(true); if (delegate != null) - return delegate.cursor(cacheId, lower, upper, x, mvccSnapshot); + return delegate.cursor(cacheId, lower, upper, x, mvccSnapshot, withTombstones); return EMPTY_CURSOR; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/DataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/DataRow.java index add2abe36c4f2..5eb50f8457f2f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/DataRow.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/DataRow.java @@ -120,10 +120,8 @@ public DataRow() { this.link = link; } - /** - * @param cacheId Cache ID. - */ - public void cacheId(int cacheId) { + /** {@inheritDoc} */ + @Override public void cacheId(int cacheId) { this.cacheId = cacheId; } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 2900edda1311e..7a823d44323cd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -6730,7 +6730,7 @@ public CheckEntriesDeletedTask(int cnt) { GridCacheEntryEx entry = ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key); - if (ignite.affinity(DEFAULT_CACHE_NAME).mapKeyToPrimaryAndBackups(key).contains(((IgniteKernal)ignite).localNode())) { + if (ctx.deferredDelete() && ignite.affinity(DEFAULT_CACHE_NAME).mapKeyToPrimaryAndBackups(key).contains(((IgniteKernal)ignite).localNode())) { assertNotNull(entry); assertTrue(entry.deleted()); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListSelfTest.java index 12b804f93a86c..87f66aff64ae1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListSelfTest.java @@ -463,6 +463,11 @@ private TestDataRow(int keySize, int valSize) { return 0; } + /** {@inheritDoc} */ + @Override public void cacheId(int cacheId) { + throw new UnsupportedOperationException(); + } + /** {@inheritDoc} */ @Override public long newMvccCoordinatorVersion() { return 0; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java index f9ab6a4d3936b..d3bff2997fa8f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java @@ -129,7 +129,7 @@ public H2PkHashIndex( continue; if (filter == null || filter.applyPartition(part)) - cursors.add(store.cursor(cctx.cacheId(), lowerObj, upperObj, null, mvccSnapshot)); + cursors.add(store.cursor(cctx.cacheId(), lowerObj, upperObj, null, mvccSnapshot, false)); } return new H2PkHashIndexCursor(cursors.iterator()); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/H2CacheRow.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/H2CacheRow.java index 4bd584ff96f92..4ac6303ad5acf 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/H2CacheRow.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/H2CacheRow.java @@ -230,6 +230,11 @@ private boolean removedRow() { return row.cacheId(); } + /** {@inheritDoc} */ + @Override public void cacheId(int cacheId) { + row.cacheId(cacheId); + } + /** {@inheritDoc} */ @Override public long mvccCoordinatorVersion() { return row.mvccCoordinatorVersion(); From d5e4e788893e6caa574eb0d1a2b62fbbea210f37 Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 10 Jul 2019 10:25:56 +0300 Subject: [PATCH 02/30] ignite-11704 --- .../ignite/internal/processors/cache/CacheGroupContext.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index 56c2449ced03c..4af5de54233fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -268,6 +268,8 @@ public class CacheGroupContext { statHolderIdx = new IoStatisticsHolderIndex(HASH_INDEX, cacheOrGroupName(), HASH_PK_IDX_NAME, mmgr); statHolderData = new IoStatisticsHolderCache(cacheOrGroupName(), grpId, mmgr); } + + hasAtomicCaches = ccfg.getAtomicityMode() == ATOMIC; } /** From 4403455baaa82b5c2546802633cbe1a676393513 Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 10 Jul 2019 22:38:35 +0300 Subject: [PATCH 03/30] ignite-11704 --- .../cache/IgniteCacheConfigVariationsFullApiTest.java | 2 +- .../persistence/IgnitePdsRemoveDuringRebalancingTest.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java index 74d94421622d4..ae9df1e1b9654 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheConfigVariationsFullApiTest.java @@ -6616,7 +6616,7 @@ public CheckEntriesDeletedTask(int cnt, String cacheName) { GridCacheEntryEx entry = ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key); - if (ignite.affinity(cacheName).mapKeyToPrimaryAndBackups(key).contains(((IgniteKernal)ignite).localNode())) { + if (ctx.deferredDelete() && ignite.affinity(cacheName).mapKeyToPrimaryAndBackups(key).contains(((IgniteKernal)ignite).localNode())) { assertNotNull(entry); assertTrue(entry.deleted()); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRemoveDuringRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRemoveDuringRebalancingTest.java index 53556bdad71c5..68c8f7a439cbc 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRemoveDuringRebalancingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRemoveDuringRebalancingTest.java @@ -48,7 +48,7 @@ public class IgnitePdsRemoveDuringRebalancingTest extends GridCommonAbstractTest IgniteConfiguration cfg = super.getConfiguration(gridName); cfg.setCacheConfiguration( - new CacheConfiguration() + new CacheConfiguration(DEFAULT_CACHE_NAME) .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) .setBackups(1) .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) @@ -96,14 +96,14 @@ public void testRemovesDuringRebalancing() throws Exception { ig.active(true); - try (IgniteDataStreamer streamer = ig.dataStreamer(null)) { + try (IgniteDataStreamer streamer = ig.dataStreamer(DEFAULT_CACHE_NAME)) { streamer.allowOverwrite(true); for (int i = 0; i < 100_000; i++) streamer.addData(i, i); } - final IgniteCache cache = ig.cache(null); + final IgniteCache cache = ig.cache(DEFAULT_CACHE_NAME); IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() throws Exception { @@ -118,7 +118,7 @@ public void testRemovesDuringRebalancing() throws Exception { IgniteEx another = grid(1); - IgniteCache cache1 = another.cache(null); + IgniteCache cache1 = another.cache(DEFAULT_CACHE_NAME); for (int i = 0; i < 100_000; i++) assertNull(cache1.localPeek(i)); From 5a05291a9414ef8f8b37ec3d1e8a2e0a9ce4c70f Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 11 Jul 2019 23:45:40 +0300 Subject: [PATCH 04/30] ignite-11704 --- .../processors/cache/CacheMetricsImpl.java | 14 + .../processors/cache/GridCacheContext.java | 14 + .../cache/IgniteCacheOffheapManager.java | 6 +- .../cache/IgniteCacheOffheapManagerImpl.java | 69 +++-- .../dht/topology/GridDhtLocalPartition.java | 2 +- .../persistence/GridCacheOffheapManager.java | 8 +- .../CollectConflictPartitionKeysTask.java | 2 +- .../verify/VerifyBackupPartitionsTask.java | 2 +- .../verify/VerifyBackupPartitionsTaskV2.java | 2 +- ...dAndDeleteGarbageInPersistenceClosure.java | 2 +- .../cache/IgniteCacheGroupsTest.java | 10 +- .../CacheRemoveWithTombstonesTest.java | 261 ++++++++++++++++++ .../persistence/db/IgnitePdsWithTtlTest.java | 2 +- .../testsuites/IgniteCacheTestSuite9.java | 3 + 14 files changed, 355 insertions(+), 42 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java index ab1c4a1cda7c1..5808db6bae11e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java @@ -29,6 +29,7 @@ import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore; import org.apache.ignite.internal.processors.metric.MetricRegistry; import org.apache.ignite.internal.processors.metric.impl.HitRateMetric; +import org.apache.ignite.internal.processors.metric.impl.LongAdderMetricImpl; import org.apache.ignite.internal.processors.metric.impl.LongMetricImpl; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -161,6 +162,9 @@ public class CacheMetricsImpl implements CacheMetrics { /** Number of currently clearing partitions for rebalancing. */ private final LongMetricImpl rebalanceClearingPartitions; + /** */ + private LongAdderMetricImpl tombstones; + /** Cache metrics. */ @GridToStringExclude private transient CacheMetricsImpl delegate; @@ -312,6 +316,8 @@ public CacheMetricsImpl(GridCacheContext cctx, boolean isNear) { rebalanceClearingPartitions = mreg.metric("RebalanceClearingPartitionsLeft", "Number of partitions need to be cleared before actual rebalance start."); + + tombstones = mreg.longAdderMetric("Tombstones", "Number of tombstone entries"); } /** @@ -1009,6 +1015,14 @@ public void addPutAndGetTimeNanos(long duration) { delegate.addPutAndGetTimeNanos(duration); } + public void tombstoneCreated() { + tombstones.increment(); + } + + public void tombstoneRemoved() { + tombstones.decrement(); + } + /** {@inheritDoc} */ @Override public String getKeyType() { CacheConfiguration ccfg = cctx.config(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 87c62534821b0..5ac56db0dcfa5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -2372,6 +2372,20 @@ protected Object readResolve() throws ObjectStreamException { } } + public void tombstoneCreated() { + GridCacheAdapter cache = this.cache; + + if (cache != null) + cache.metrics0().tombstoneCreated(); + } + + public void tombstoneRemoved() { + GridCacheAdapter cache = this.cache; + + if (cache != null) + cache.metrics0().tombstoneRemoved(); + } + /** {@inheritDoc} */ @Override public String toString() { return "GridCacheContext: " + name(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index 4221a2d13d297..c11e909bbcff0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -452,7 +452,7 @@ public GridIterator cachePartitionIterator(int cacheId, final int * @return Iterator for given partition. * @throws IgniteCheckedException If failed. */ - public GridIterator partitionIterator(final int part) throws IgniteCheckedException; + public GridIterator partitionIterator(final int part, boolean withTombstones) throws IgniteCheckedException; /** * @param part Partition number. @@ -963,7 +963,7 @@ List> mvccFindAllVersions(GridCacheContext cc * @return Data cursor. * @throws IgniteCheckedException If failed. */ - public GridCursor cursor() throws IgniteCheckedException; + public GridCursor cursor(boolean withTombstones) throws IgniteCheckedException; /** * @param x Implementation specific argument, {@code null} always means that we need to return full detached data row. @@ -984,7 +984,7 @@ List> mvccFindAllVersions(GridCacheContext cc * @return Data cursor. * @throws IgniteCheckedException If failed. */ - public GridCursor cursor(int cacheId) throws IgniteCheckedException; + public GridCursor cursor(int cacheId, boolean withTombstones) throws IgniteCheckedException; /** * @param cacheId Cache ID. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index e62834591b92e..c45e3b19e2933 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -747,8 +747,8 @@ private Iterator cacheData(boolean primary, boolean backup, Affi GridCacheVersion obsoleteVer = null; try (GridCloseableIterator it = grp.isLocal() ? - iterator(cctx.cacheId(), cacheDataStores().iterator(), null, null) : - evictionSafeIterator(cctx.cacheId(), cacheDataStores().iterator())) { + iterator(cctx.cacheId(), cacheDataStores().iterator(), null, null, true) : + evictionSafeIterator(cctx.cacheId(), cacheDataStores().iterator(), true)) { while (it.hasNext()) { cctx.shared().database().checkpointReadLock(); @@ -891,7 +891,7 @@ private Iterator cacheData(boolean primary, boolean backup, Affi @Nullable MvccSnapshot mvccSnapshot, Boolean dataPageScanEnabled ) { - return iterator(cacheId, cacheData(primary, backups, topVer), mvccSnapshot, dataPageScanEnabled); + return iterator(cacheId, cacheData(primary, backups, topVer), mvccSnapshot, dataPageScanEnabled, false); } /** {@inheritDoc} */ @@ -902,17 +902,17 @@ private Iterator cacheData(boolean primary, boolean backup, Affi if (data == null) return new GridEmptyCloseableIterator<>(); - return iterator(cacheId, singletonIterator(data), mvccSnapshot, dataPageScanEnabled); + return iterator(cacheId, singletonIterator(data), mvccSnapshot, dataPageScanEnabled, false); } /** {@inheritDoc} */ - @Override public GridIterator partitionIterator(int part) { + @Override public GridIterator partitionIterator(int part, boolean withTombstones) { CacheDataStore data = partitionData(part); if (data == null) return new GridEmptyCloseableIterator<>(); - return iterator(CU.UNDEFINED_CACHE_ID, singletonIterator(data), null, null); + return iterator(CU.UNDEFINED_CACHE_ID, singletonIterator(data), null, null, withTombstones); } /** @@ -926,7 +926,8 @@ private Iterator cacheData(boolean primary, boolean backup, Affi private GridCloseableIterator iterator(final int cacheId, final Iterator dataIt, final MvccSnapshot mvccSnapshot, - Boolean dataPageScanEnabled + Boolean dataPageScanEnabled, + boolean withTombstones ) { return new GridCloseableIteratorAdapter() { /** */ @@ -962,7 +963,7 @@ private GridCloseableIterator iterator(final int cacheId, try { if (mvccSnapshot == null) - cur = cacheId == CU.UNDEFINED_CACHE_ID ? ds.cursor() : ds.cursor(cacheId); + cur = cacheId == CU.UNDEFINED_CACHE_ID ? ds.cursor(withTombstones) : ds.cursor(cacheId, withTombstones); else { cur = cacheId == CU.UNDEFINED_CACHE_ID ? ds.cursor(mvccSnapshot) : ds.cursor(cacheId, mvccSnapshot); @@ -996,7 +997,10 @@ private GridCloseableIterator iterator(final int cacheId, * @param dataIt Data store iterator. * @return Rows iterator */ - private GridCloseableIterator evictionSafeIterator(final int cacheId, final Iterator dataIt) { + private GridCloseableIterator evictionSafeIterator( + final int cacheId, + final Iterator dataIt, + boolean withTombstones) { return new GridCloseableIteratorAdapter() { /** */ private GridCursor cur; @@ -1027,7 +1031,7 @@ private GridCloseableIterator evictionSafeIterator(final int cache if (!reservePartition(ds.partId())) continue; - cur = cacheId == CU.UNDEFINED_CACHE_ID ? ds.cursor() : ds.cursor(cacheId); + cur = cacheId == CU.UNDEFINED_CACHE_ID ? ds.cursor(withTombstones) : ds.cursor(cacheId, withTombstones); } else break; @@ -1700,17 +1704,19 @@ private void invoke0(GridCacheContext cctx, CacheSearchRow row, OffheapInvokeClo case REMOVE: { CacheDataRow oldRow = c.oldRow(); - finishRemove(cctx, row.key(), oldRow, true); + finishRemove(cctx, row.key(), oldRow, null); break; } case IN_PLACE: - if (isTombstone(c.newRow())) - decrementSize(cctx.cacheId()); + assert !isTombstone(c.newRow()); + + if (isTombstone(c.oldRow())) { + cctx.tombstoneRemoved(); - if (isTombstone(c.oldRow())) incrementSize(cctx.cacheId()); + } break; @@ -2592,7 +2598,10 @@ private int cleanup0(GridCacheContext cctx, @Nullable List cursor() throws IgniteCheckedException { - return cursorSkipTombstone(dataTree.find(null, null)); + @Override public GridCursor cursor(boolean withTombstones) throws IgniteCheckedException { + GridCursor cur = dataTree.find(null, null); + + return withTombstones ? cur : cursorSkipTombstone(cur); } private GridCursor cursorSkipTombstone(final GridCursor cur) { @@ -2964,8 +2985,8 @@ private GridCursor cursorSkipTombstone(final GridCursor< } /** {@inheritDoc} */ - @Override public GridCursor cursor(int cacheId) throws IgniteCheckedException { - return cursor(cacheId, null, null); + @Override public GridCursor cursor(int cacheId, boolean withTombstones) throws IgniteCheckedException { + return cursor(cacheId, null, null, null, null, withTombstones); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 5e637e912967f..e3e64350602f2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -1143,7 +1143,7 @@ private long clearAll(EvictionContext evictionCtx) throws NodeStoppingException CacheMapHolder hld = grp.sharedGroup() ? null : singleCacheEntryMap; try { - GridIterator it0 = grp.offheap().partitionIterator(id); + GridIterator it0 = grp.offheap().partitionIterator(id, true); while (it0.hasNext()) { ctx.database().checkpointReadLock(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 6f5a0ea4b91d0..d4bcbd868e390 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -2476,11 +2476,11 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { /** {@inheritDoc} */ - @Override public GridCursor cursor() throws IgniteCheckedException { + @Override public GridCursor cursor(boolean withTombstones) throws IgniteCheckedException { CacheDataStore delegate = init0(true); if (delegate != null) - return delegate.cursor(); + return delegate.cursor(withTombstones); return EMPTY_CURSOR; } @@ -2555,11 +2555,11 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public GridCursor cursor(int cacheId) throws IgniteCheckedException { + @Override public GridCursor cursor(int cacheId, boolean withTombstones) throws IgniteCheckedException { CacheDataStore delegate = init0(true); if (delegate != null) - return delegate.cursor(cacheId); + return delegate.cursor(cacheId, withTombstones); return EMPTY_CURSOR; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java index 8e8128cc9aaaa..bd9db7bde08d2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java @@ -178,7 +178,7 @@ private CollectPartitionEntryHashesJob(PartitionKey partKey) { partSize = part.dataStore().fullSize(); - GridIterator it = grpCtx.offheap().partitionIterator(part.id()); + GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); partEntryHashRecords = new ArrayList<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java index f70056792f266..8b18834fb9ed2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java @@ -317,7 +317,7 @@ private Map calculatePartitionHash( partSize = part.dataStore().fullSize(); - GridIterator it = grpCtx.offheap().partitionIterator(part.id()); + GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); while (it.hasNextX()) { CacheDataRow row = it.nextX(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java index 92b6e36294d15..9d8335f7aa9d0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java @@ -579,7 +579,7 @@ else if (part.state() != GridDhtPartitionState.OWNING) if (arg.checkCrc()) checkPartitionCrc(grpCtx, part, cpFlag); - GridIterator it = grpCtx.offheap().partitionIterator(part.id()); + GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); while (it.hasNextX()) { CacheDataRow row = it.nextX(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorFindAndDeleteGarbageInPersistenceClosure.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorFindAndDeleteGarbageInPersistenceClosure.java index 722686e578011..552a1a7c679ea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorFindAndDeleteGarbageInPersistenceClosure.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorFindAndDeleteGarbageInPersistenceClosure.java @@ -266,7 +266,7 @@ private Map> processPartition( if (part.state() != GridDhtPartitionState.OWNING) return Collections.emptyMap(); - GridIterator it = grpCtx.offheap().partitionIterator(part.id()); + GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); while (it.hasNextX()) { CacheDataRow row = it.nextX(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java index 0ab22fd66db55..478cf80baa774 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java @@ -1666,17 +1666,17 @@ public void testRebalance2() throws Exception { for (int p = 0; p < aff.partitions(); p++) { if (srv1Parts.contains(p)) { - GridIterator it = grpSrv0.offheap().partitionIterator(p); + GridIterator it = grpSrv0.offheap().partitionIterator(p, false); assertFalse(it.hasNext()); - it = grpSrv1.offheap().partitionIterator(p); + it = grpSrv1.offheap().partitionIterator(p, false); assertTrue(it.hasNext()); } else { - GridIterator it = grpSrv0.offheap().partitionIterator(p); + GridIterator it = grpSrv0.offheap().partitionIterator(p, false); assertTrue(it.hasNext()); - it = grpSrv1.offheap().partitionIterator(p); + it = grpSrv1.offheap().partitionIterator(p, false); assertFalse(it.hasNext()); } } @@ -3919,7 +3919,7 @@ public void testCacheIdSort() throws Exception { Integer cacheId = null; - GridIterator it = grp.offheap().partitionIterator(0); + GridIterator it = grp.offheap().partitionIterator(0, false); int c = 0; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java new file mode 100644 index 0000000000000..331fb64afdb1c --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.spi.metric.LongMetric; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; + +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheRebalanceMode.SYNC; +import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.cacheMetricsRegistryName; + +/** + * + */ +public class CacheRemoveWithTombstonesTest extends GridCommonAbstractTest { + /** */ + private boolean persistence; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + TestRecordingCommunicationSpi commSpi = new TestRecordingCommunicationSpi(); + + cfg.setCommunicationSpi(commSpi); + + if (persistence) { + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration().setMaxSize(100L * 1024 * 1024).setPersistenceEnabled(true)) + .setWalMode(WALMode.LOG_ONLY); + + cfg.setDataStorageConfiguration(dsCfg); + } + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + super.afterTest(); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testRemoveAndRebalanceRaceTx() throws Exception { + testRemoveAndRebalanceRace(TRANSACTIONAL, true); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testRemoveAndRebalanceRaceTxMvcc() throws Exception { + testRemoveAndRebalanceRace(TRANSACTIONAL_SNAPSHOT, false); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testRemoveAndRebalanceRaceAtomic() throws Exception { + testRemoveAndRebalanceRace(ATOMIC, false); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testRemoveAndRebalanceRaceTxWithPersistence() throws Exception { + persistence = true; + + testRemoveAndRebalanceRace(TRANSACTIONAL, true); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testRemoveAndRebalanceRaceTxMvccWithPersistence() throws Exception { + persistence = true; + + testRemoveAndRebalanceRace(TRANSACTIONAL_SNAPSHOT, false); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testRemoveAndRebalanceRaceAtomicWithPersistence() throws Exception { + persistence = true; + + testRemoveAndRebalanceRace(ATOMIC, false); + } + + /** + * @throws Exception If failed. + * @param expTombstone {@code True} if tombstones should be created. + */ + private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolean expTombstone) throws Exception { + IgniteEx ignite0 = startGrid(0); + + if (persistence) + ignite0.cluster().active(true); + + IgniteCache cache0 = ignite0.createCache(cacheConfiguration(atomicityMode)); + + LongMetric tombstoneMetric0 = (LongMetric)ignite0.context().metric().registry( + cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); + + Map map = new HashMap<>(); + + final int KEYS = 1024; + + for (int i = 0; i < KEYS; i++) + map.put(i, i); + + cache0.putAll(map); + + TestRecordingCommunicationSpi.spi(ignite0).blockMessages(GridDhtPartitionSupplyMessageV2.class, + getTestIgniteInstanceName(1)); + + IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { + @Override public Object call() throws Exception { + return startGrid(1); + } + }); + + IgniteEx ignite1 = (IgniteEx)fut.get(30_000); + + Set removed = new HashSet<>(); + + // Do removes while rebalance is in progress. + for (int i = 0; i < KEYS; i++) { + if (i % 2 == 0) { + removed.add(i); + + cache0.remove(i); + } + } + + final LongMetric tombstoneMetric1 = (LongMetric)ignite1.context().metric().registry( + cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); + + // On first node there should not be tombstones. + //assertEquals(0, tombstoneMetric0.get()); + + if (expTombstone) + assertEquals(removed.size(), tombstoneMetric1.get()); + else + assertEquals(0, tombstoneMetric1.get()); + + // Update some of removed keys, this should remove tombstones. + for (int i = 0; i < KEYS; i++) { + if (i % 4 == 0) { + removed.remove(i); + + cache0.put(i, i); + } + } + + assert !removed.isEmpty(); + + //assertEquals(0, tombstoneMetric0.get()); + + if (expTombstone) + assertEquals(removed.size(), tombstoneMetric1.get()); + else + assertEquals(0, tombstoneMetric1.get()); + + TestRecordingCommunicationSpi.spi(ignite0).stopBlock(); + + awaitPartitionMapExchange(); + + IgniteCache cache1 = ignite(1).cache(DEFAULT_CACHE_NAME); + + for (int i = 0; i < KEYS; i++) { + if (removed.contains(i)) + assertNull(cache1.get(i)); + else + assertEquals((Object)i, cache1.get(i)); + } + + // Tombstones should be removed after once rebalance is completed. + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + return tombstoneMetric1.get() == 0; + } + }, 30_000); + + assertEquals(0, tombstoneMetric1.get()); + } + + /** + * @param atomicityMode Cache atomicity mode. + * @return Cache configuration. + */ + private CacheConfiguration cacheConfiguration(CacheAtomicityMode atomicityMode) { + CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + + ccfg.setAtomicityMode(atomicityMode); + ccfg.setCacheMode(PARTITIONED); + ccfg.setBackups(2); + ccfg.setRebalanceMode(SYNC); + ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + + return ccfg; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWithTtlTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWithTtlTest.java index afe2b34050858..4461d947a46f9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWithTtlTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWithTtlTest.java @@ -316,7 +316,7 @@ protected void waitAndCheckExpired( IgniteCacheOffheapManager.CacheDataStore dataStore = ctx.cache().cacheGroup(CU.cacheId(GROUP_NAME)).offheap().dataStore(locPart); - GridCursor cur = dataStore.cursor(); + GridCursor cur = dataStore.cursor(false); assertFalse(cur.next()); assertEquals(0, locPart.fullSize()); diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java index 4263f9d386a36..3927a34c6e2e0 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheLoadRebalanceEvictionSelfTest; import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest; import org.apache.ignite.internal.processors.cache.distributed.CacheOperationsInterruptTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheRemoveWithTombstonesTest; import org.apache.ignite.internal.processors.cache.distributed.FailBackupOnAtomicOperationTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCachePrimarySyncTest; @@ -118,6 +119,8 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, FailBackupOnAtomicOperationTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheRemoveWithTombstonesTest.class, ignoredTests); + return suite; } } From e6a793bf4caf897453f40ad4df977ea7268dea4c Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 19 Jul 2019 21:20:51 +0300 Subject: [PATCH 05/30] ignite-11704 --- .../processors/cache/CacheGroupContext.java | 3 +- .../processors/cache/GridCacheMapEntry.java | 88 ++++++++++++++++- .../cache/IgniteCacheOffheapManager.java | 5 +- .../cache/IgniteCacheOffheapManagerImpl.java | 26 ++++- .../dht/topology/GridDhtLocalPartition.java | 99 ++++++++++++++++++- .../persistence/GridCacheOffheapManager.java | 8 +- .../CacheRemoveWithTombstonesTest.java | 39 ++++++-- 7 files changed, 250 insertions(+), 18 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index 4af5de54233fc..7963893696a47 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; @@ -1307,7 +1308,7 @@ public boolean supportsTombstone() { } public boolean createTombstone(@Nullable GridDhtLocalPartition part) { - return part != null && supportsTombstone(); + return part != null && supportsTombstone() && part.state() == GridDhtPartitionState.MOVING; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index adc86994e4b66..08986a9604c05 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -1717,8 +1717,12 @@ protected Object keyValue(boolean cpy) { } } - if (cctx.group().createTombstone(localPartition())) - cctx.offheap().removeWithTombstone(cctx, key, newVer, partition(), localPartition()); + if (cctx.group().createTombstone(localPartition())) { + cctx.offheap().removeWithTombstone(cctx, key, newVer, localPartition()); + + if (!cctx.group().createTombstone(localPartition())) + removeTombstone0(newVer); + } else removeValue(); @@ -2817,6 +2821,34 @@ protected void clearReader(UUID nodeId) throws GridCacheEntryRemovedException { return marked; } + /** + * @param tombstoneVer Tombstone version. + * @throws GridCacheEntryRemovedException If entry was removed. + * @throws IgniteCheckedException If failed. + */ + public void removeTombstone(GridCacheVersion tombstoneVer) throws GridCacheEntryRemovedException, IgniteCheckedException { + lockEntry(); + + try { + checkObsolete(); + + removeTombstone0(tombstoneVer); + } + finally { + unlockEntry(); + } + } + + /** + * @param tombstoneVer Tombstone version. + * @throws IgniteCheckedException If failed. + */ + private void removeTombstone0(GridCacheVersion tombstoneVer) throws IgniteCheckedException { + RemoveClosure closure = new RemoveClosure(this, tombstoneVer); + + cctx.offheap().invoke(cctx, key, localPartition(), closure); + } + /** * @return {@code True} if this entry should not be evicted from cache. */ @@ -5717,6 +5749,58 @@ private boolean checkRowExpired(CacheDataRow row) throws IgniteCheckedException return true; } + /** + * + */ + private static class RemoveClosure implements IgniteCacheOffheapManager.OffheapInvokeClosure { + /** */ + private final GridCacheMapEntry entry; + + /** */ + private final GridCacheVersion ver; + + /** */ + private IgniteTree.OperationType op; + + /** */ + private CacheDataRow oldRow; + + public RemoveClosure(GridCacheMapEntry entry, GridCacheVersion ver) { + this.entry = entry; + this.ver = ver; + } + + /** {@inheritDoc} */ + @Override public @Nullable CacheDataRow oldRow() { + return oldRow; + } + + /** {@inheritDoc} */ + @Override public void call(@Nullable CacheDataRow row) throws IgniteCheckedException { + if (row == null || !ver.equals(row.version())) { + op = IgniteTree.OperationType.NOOP; + + return; + } + + row.key(entry.key); + + oldRow = row; + + op = IgniteTree.OperationType.REMOVE; + } + + /** {@inheritDoc} */ + @Override public CacheDataRow newRow() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgniteTree.OperationType operationType() { + return op; + } + } + /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index c11e909bbcff0..c88334359e398 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -406,7 +406,6 @@ public void removeWithTombstone( GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, - int partId, GridDhtLocalPartition part ) throws IgniteCheckedException; @@ -454,6 +453,8 @@ public GridIterator cachePartitionIterator(int cacheId, final int */ public GridIterator partitionIterator(final int part, boolean withTombstones) throws IgniteCheckedException; + public GridIterator tombstonesIterator(final int part) throws IgniteCheckedException; + /** * @param part Partition number. * @param topVer Topology version. @@ -917,7 +918,7 @@ void mvccApplyUpdate(GridCacheContext cctx, * @param partId Partition number. * @throws IgniteCheckedException If failed. */ - public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, int partId) throws IgniteCheckedException; + public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, GridDhtLocalPartition part) throws IgniteCheckedException; /** * @param cctx Cache context. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index c45e3b19e2933..3597de7eba751 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -632,9 +632,10 @@ private Iterator cacheData(boolean primary, boolean backup, Affi GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, - int partId, GridDhtLocalPartition part) throws IgniteCheckedException { - dataStore(part).removeWithTombstone(cctx, key, ver, partId); + assert part != null; + + dataStore(part).removeWithTombstone(cctx, key, ver, part); } @Override public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { @@ -915,6 +916,19 @@ private Iterator cacheData(boolean primary, boolean backup, Affi return iterator(CU.UNDEFINED_CACHE_ID, singletonIterator(data), null, null, withTombstones); } + /** {@inheritDoc} */ + @Override public GridIterator tombstonesIterator(int part) { + assert locCacheDataStore == null; + + CacheDataStore data = partitionData(part); + + if (data == null) + return new GridEmptyCloseableIterator<>(); + + // TODO IGNITE-11704. + return iterator(CU.UNDEFINED_CACHE_ID, singletonIterator(data), null, null, true); + } + /** * * @param cacheId Cache ID. @@ -2730,7 +2744,11 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn } /** {@inheritDoc} */ - @Override public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, int partId) throws IgniteCheckedException { + @Override public void removeWithTombstone( + GridCacheContext cctx, + KeyCacheObject key, + GridCacheVersion ver, + GridDhtLocalPartition part) throws IgniteCheckedException { if (!busyLock.enterBusy()) throw new NodeStoppingException("Operation has been cancelled (node is stopping)."); @@ -2745,6 +2763,8 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn assert c.operationType() == PUT || c.operationType() == IN_PLACE : c.operationType(); + part.tombstoneCreated(); + if (!isTombstone(c.oldRow)) cctx.tombstoneCreated(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index e3e64350602f2..13b17619c3b03 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -44,6 +44,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheConcurrentMapImpl; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; +import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException; import org.apache.ignite.internal.processors.cache.GridCacheMapEntry; import org.apache.ignite.internal.processors.cache.GridCacheMapEntryFactory; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; @@ -173,6 +174,9 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements /** Set if topology update sequence should be updated on partition destroy. */ private boolean updateSeqOnDestroy; + /** */ + private volatile boolean tombstoneCreated; + /** * @param ctx Context. * @param grp Cache group. @@ -619,8 +623,12 @@ public boolean own() { assert partState == MOVING || partState == LOST; - if (casState(state, OWNING)) + if (casState(state, OWNING)) { + if (grp.supportsTombstone()) + clearTombstones(); + return true; + } } } @@ -1116,6 +1124,95 @@ public long fullSize() { return store.fullSize(); } + /** + * + */ + public void tombstoneCreated() { + tombstoneCreated = true; + } + + /** + * + */ + private void submitClearTombstones() { + if (tombstoneCreated) + grp.shared().kernalContext().closure().runLocalSafe(this::clearTombstones, true); + } + + /** + * + */ + private void clearTombstones() { + final int stopCheckingFreq = 1000; + + CacheMapHolder hld = grp.sharedGroup() ? null : singleCacheEntryMap; + + try { + GridIterator it0 = grp.offheap().tombstonesIterator(id); + + int cntr = 0; + + while (it0.hasNext()) { + CacheDataRow row = it0.next(); + + if (!grp.offheap().isTombstone(row)) + continue; + + assert row.key() != null; + assert row.version() != null; + + if (grp.sharedGroup() && (hld == null || hld.cctx.cacheId() != row.cacheId())) + hld = cacheMapHolder(ctx.cacheContext(row.cacheId())); + + assert hld != null; + + ctx.database().checkpointReadLock(); + + try { + while (true) { + GridCacheMapEntry cached = null; + + try { + cached = putEntryIfObsoleteOrAbsent( + hld, + hld.cctx, + grp.affinity().lastVersion(), + row.key(), + true, + false); + + cached.removeTombstone(row.version()); + + cached.touch(); + + break; + } + catch (GridCacheEntryRemovedException e) { + cached = null; + } + finally { + if (cached != null) + cached.touch(); + } + } + } + finally { + ctx.database().checkpointReadUnlock(); + } + + cntr++; + + if (cntr % stopCheckingFreq == 0) { + if (ctx.kernalContext().isStopping() || state() != OWNING) + break; + } + } + } + catch (IgniteCheckedException e) { + U.error(log, "Failed clear tombstone entries for partition: " + id, e); + } + } + /** * Removes all entries and rows from this partition. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index d4bcbd868e390..427c0b9df0cb3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -2423,12 +2423,16 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, int partId) throws IgniteCheckedException { + @Override public void removeWithTombstone( + GridCacheContext cctx, + KeyCacheObject key, + GridCacheVersion ver, + GridDhtLocalPartition part) throws IgniteCheckedException { assert ctx.database().checkpointLockIsHeldByThread(); CacheDataStore delegate = init0(false); - delegate.removeWithTombstone(cctx, key, ver, partId); + delegate.removeWithTombstone(cctx, key, ver, part); } /** {@inheritDoc} */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index 331fb64afdb1c..05962c4402f21 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -17,9 +17,11 @@ package org.apache.ignite.internal.processors.cache.distributed; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; @@ -28,8 +30,11 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2; +import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage; import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -45,7 +50,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; -import static org.apache.ignite.cache.CacheRebalanceMode.SYNC; +import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.cacheMetricsRegistryName; /** @@ -122,6 +127,8 @@ public void testRemoveAndRebalanceRaceAtomic() throws Exception { public void testRemoveAndRebalanceRaceTxWithPersistence() throws Exception { persistence = true; + cleanPersistenceDir(); + testRemoveAndRebalanceRace(TRANSACTIONAL, true); } @@ -169,8 +176,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea cache0.putAll(map); - TestRecordingCommunicationSpi.spi(ignite0).blockMessages(GridDhtPartitionSupplyMessageV2.class, - getTestIgniteInstanceName(1)); + blockRebalance(ignite0); IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() throws Exception { @@ -180,6 +186,12 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea IgniteEx ignite1 = (IgniteEx)fut.get(30_000); + if (persistence) { + ignite0.cluster().baselineAutoAdjustEnabled(false); + + ignite0.cluster().setBaselineTopology(2); + } + Set removed = new HashSet<>(); // Do removes while rebalance is in progress. @@ -195,7 +207,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); // On first node there should not be tombstones. - //assertEquals(0, tombstoneMetric0.get()); + assertEquals(0, tombstoneMetric0.get()); if (expTombstone) assertEquals(removed.size(), tombstoneMetric1.get()); @@ -213,7 +225,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea assert !removed.isEmpty(); - //assertEquals(0, tombstoneMetric0.get()); + assertEquals(0, tombstoneMetric0.get()); if (expTombstone) assertEquals(removed.size(), tombstoneMetric1.get()); @@ -242,6 +254,19 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea assertEquals(0, tombstoneMetric1.get()); } + /** + * + */ + private void blockRebalance(Ignite node) { + final int grpId = groupIdForCache(ignite(0), DEFAULT_CACHE_NAME); + + TestRecordingCommunicationSpi.spi(node).blockMessages(new IgniteBiPredicate() { + @Override public boolean apply(ClusterNode node, Message msg) { + return (msg instanceof GridDhtPartitionSupplyMessage) + && ((GridCacheGroupIdMessage)msg).groupId() == grpId; + } + }); + } /** * @param atomicityMode Cache atomicity mode. @@ -253,7 +278,7 @@ private CacheConfiguration cacheConfiguration(CacheAtomicityMo ccfg.setAtomicityMode(atomicityMode); ccfg.setCacheMode(PARTITIONED); ccfg.setBackups(2); - ccfg.setRebalanceMode(SYNC); + ccfg.setRebalanceMode(ASYNC); ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); return ccfg; From 2a0a0e0ea41f3f1aa9daca279546625b168faf81 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 19 Jul 2019 21:20:51 +0300 Subject: [PATCH 06/30] ignite-11704 --- .../processors/cache/CacheGroupContext.java | 3 +- .../processors/cache/GridCacheMapEntry.java | 88 ++++++++++++++- .../cache/IgniteCacheOffheapManager.java | 3 +- .../cache/IgniteCacheOffheapManagerImpl.java | 70 +++++++----- .../dht/topology/GridDhtLocalPartition.java | 101 +++++++++++++++++- .../persistence/CacheDataRowAdapter.java | 22 +++- .../GridCacheDatabaseSharedManager.java | 2 + .../persistence/GridCacheOffheapManager.java | 8 +- .../IgniteCacheDatabaseSharedManager.java | 59 ++++++++++ .../CacheRemoveWithTombstonesTest.java | 39 +++++-- 10 files changed, 350 insertions(+), 45 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index 4af5de54233fc..7963893696a47 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; @@ -1307,7 +1308,7 @@ public boolean supportsTombstone() { } public boolean createTombstone(@Nullable GridDhtLocalPartition part) { - return part != null && supportsTombstone(); + return part != null && supportsTombstone() && part.state() == GridDhtPartitionState.MOVING; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index adc86994e4b66..08986a9604c05 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -1717,8 +1717,12 @@ protected Object keyValue(boolean cpy) { } } - if (cctx.group().createTombstone(localPartition())) - cctx.offheap().removeWithTombstone(cctx, key, newVer, partition(), localPartition()); + if (cctx.group().createTombstone(localPartition())) { + cctx.offheap().removeWithTombstone(cctx, key, newVer, localPartition()); + + if (!cctx.group().createTombstone(localPartition())) + removeTombstone0(newVer); + } else removeValue(); @@ -2817,6 +2821,34 @@ protected void clearReader(UUID nodeId) throws GridCacheEntryRemovedException { return marked; } + /** + * @param tombstoneVer Tombstone version. + * @throws GridCacheEntryRemovedException If entry was removed. + * @throws IgniteCheckedException If failed. + */ + public void removeTombstone(GridCacheVersion tombstoneVer) throws GridCacheEntryRemovedException, IgniteCheckedException { + lockEntry(); + + try { + checkObsolete(); + + removeTombstone0(tombstoneVer); + } + finally { + unlockEntry(); + } + } + + /** + * @param tombstoneVer Tombstone version. + * @throws IgniteCheckedException If failed. + */ + private void removeTombstone0(GridCacheVersion tombstoneVer) throws IgniteCheckedException { + RemoveClosure closure = new RemoveClosure(this, tombstoneVer); + + cctx.offheap().invoke(cctx, key, localPartition(), closure); + } + /** * @return {@code True} if this entry should not be evicted from cache. */ @@ -5717,6 +5749,58 @@ private boolean checkRowExpired(CacheDataRow row) throws IgniteCheckedException return true; } + /** + * + */ + private static class RemoveClosure implements IgniteCacheOffheapManager.OffheapInvokeClosure { + /** */ + private final GridCacheMapEntry entry; + + /** */ + private final GridCacheVersion ver; + + /** */ + private IgniteTree.OperationType op; + + /** */ + private CacheDataRow oldRow; + + public RemoveClosure(GridCacheMapEntry entry, GridCacheVersion ver) { + this.entry = entry; + this.ver = ver; + } + + /** {@inheritDoc} */ + @Override public @Nullable CacheDataRow oldRow() { + return oldRow; + } + + /** {@inheritDoc} */ + @Override public void call(@Nullable CacheDataRow row) throws IgniteCheckedException { + if (row == null || !ver.equals(row.version())) { + op = IgniteTree.OperationType.NOOP; + + return; + } + + row.key(entry.key); + + oldRow = row; + + op = IgniteTree.OperationType.REMOVE; + } + + /** {@inheritDoc} */ + @Override public CacheDataRow newRow() { + return null; + } + + /** {@inheritDoc} */ + @Override public IgniteTree.OperationType operationType() { + return op; + } + } + /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index c11e909bbcff0..227243970e122 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -406,7 +406,6 @@ public void removeWithTombstone( GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, - int partId, GridDhtLocalPartition part ) throws IgniteCheckedException; @@ -917,7 +916,7 @@ void mvccApplyUpdate(GridCacheContext cctx, * @param partId Partition number. * @throws IgniteCheckedException If failed. */ - public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, int partId) throws IgniteCheckedException; + public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, GridDhtLocalPartition part) throws IgniteCheckedException; /** * @param cctx Cache context. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index c45e3b19e2933..7ae45b481d485 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.processors.cache; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -179,9 +178,6 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager /** */ protected GridStripedLock partStoreLock = new GridStripedLock(Runtime.getRuntime().availableProcessors()); - /** */ - private CacheObject NULL_VAL; - /** {@inheritDoc} */ @Override public GridAtomicLong globalRemoveId() { return globalRmvId; @@ -203,8 +199,6 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager if (grp.isLocal()) locCacheDataStore = createCacheDataStore(0); - - NULL_VAL = new CacheObjectImpl(null, ctx.marshaller().marshal(null)); } finally { ctx.database().checkpointReadUnlock(); @@ -632,28 +626,17 @@ private Iterator cacheData(boolean primary, boolean backup, Affi GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, - int partId, GridDhtLocalPartition part) throws IgniteCheckedException { - dataStore(part).removeWithTombstone(cctx, key, ver, partId); + assert part != null; + + dataStore(part).removeWithTombstone(cctx, key, ver, part); } @Override public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { - if (row == null || !grp.supportsTombstone()) + if (!grp.supportsTombstone()) return false; - CacheObject val = row.value(); - - assert val != null : row; - - if (val.cacheObjectType() == CacheObject.TYPE_REGULAR) { - byte[] null_bytes = NULL_VAL.valueBytes(null); - byte[] bytes = val.valueBytes(null); - - if (Arrays.equals(null_bytes, bytes)) - return true; - } - - return false; + return grp.shared().database().isTombstone(row); } /** {@inheritDoc} */ @@ -2712,7 +2695,7 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn this.oldRow = oldRow; - newRow = createRow(cctx, key, NULL_VAL, ver, 0, oldRow); + newRow = createRow(cctx, key, cctx.shared().database().tombstoneValue(), ver, 0, oldRow); } /** {@inheritDoc} */ @@ -2730,7 +2713,11 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn } /** {@inheritDoc} */ - @Override public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, int partId) throws IgniteCheckedException { + @Override public void removeWithTombstone( + GridCacheContext cctx, + KeyCacheObject key, + GridCacheVersion ver, + GridDhtLocalPartition part) throws IgniteCheckedException { if (!busyLock.enterBusy()) throw new NodeStoppingException("Operation has been cancelled (node is stopping)."); @@ -2745,6 +2732,8 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn assert c.operationType() == PUT || c.operationType() == IN_PLACE : c.operationType(); + part.tombstoneCreated(); + if (!isTombstone(c.oldRow)) cctx.tombstoneCreated(); @@ -2933,7 +2922,34 @@ private void afterRowFound(@Nullable CacheDataRow row, KeyCacheObject key) throw @Override public GridCursor cursor(boolean withTombstones) throws IgniteCheckedException { GridCursor cur = dataTree.find(null, null); - return withTombstones ? cur : cursorSkipTombstone(cur); + return withTombstones ? cursorSkipEmpty(cur) : cursorSkipTombstone(cur); + } + + private GridCursor cursorSkipEmpty(final GridCursor cur) { + if (!grp.supportsTombstone()) + return cur; + + return new GridCursor() { + CacheDataRow next; + + @Override public boolean next() throws IgniteCheckedException { + while (cur.next()) { + CacheDataRow next = cur.get(); + + if (next.version() != null) { + this.next = next; + + return true; + } + } + + return false; + } + + @Override public CacheDataRow get() { + return next; + } + }; } private GridCursor cursorSkipTombstone(final GridCursor cur) { @@ -2965,7 +2981,9 @@ private GridCursor cursorSkipTombstone(final GridCursor< /** {@inheritDoc} */ @Override public GridCursor cursor(Object x) throws IgniteCheckedException { - return cursorSkipTombstone(dataTree.find(null, null, x)); + GridCursor cur = dataTree.find(null, null, x); + + return x == CacheDataRowAdapter.RowData.TOMBSTONES ? cursorSkipEmpty(cur) : cursorSkipTombstone(cur); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index e3e64350602f2..b5a63e69d4497 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -44,6 +44,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheConcurrentMapImpl; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; +import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException; import org.apache.ignite.internal.processors.cache.GridCacheMapEntry; import org.apache.ignite.internal.processors.cache.GridCacheMapEntryFactory; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; @@ -54,12 +55,14 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader; import org.apache.ignite.internal.processors.cache.extras.GridCacheObsoleteEntryExtras; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.TxCounters; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.query.GridQueryRowCacheCleaner; import org.apache.ignite.internal.util.GridLongList; import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.lang.GridCursor; import org.apache.ignite.internal.util.lang.GridIterator; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.LT; @@ -173,6 +176,9 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements /** Set if topology update sequence should be updated on partition destroy. */ private boolean updateSeqOnDestroy; + /** */ + private volatile boolean tombstoneCreated; + /** * @param ctx Context. * @param grp Cache group. @@ -619,8 +625,12 @@ public boolean own() { assert partState == MOVING || partState == LOST; - if (casState(state, OWNING)) + if (casState(state, OWNING)) { + if (grp.supportsTombstone()) + submitClearTombstones(); + return true; + } } } @@ -1116,6 +1126,95 @@ public long fullSize() { return store.fullSize(); } + /** + * + */ + public void tombstoneCreated() { + tombstoneCreated = true; + } + + /** + * + */ + private void submitClearTombstones() { + if (tombstoneCreated) + grp.shared().kernalContext().closure().runLocalSafe(this::clearTombstones, true); + } + + /** + * + */ + private void clearTombstones() { + final int stopCheckingFreq = 1000; + + CacheMapHolder hld = grp.sharedGroup() ? null : singleCacheEntryMap; + + try { + GridCursor cur = store.cursor(CacheDataRowAdapter.RowData.TOMBSTONES); + + int cntr = 0; + + while (cur.next()) { + CacheDataRow row = cur.get(); + + if (!grp.offheap().isTombstone(row)) + continue; + + assert row.key() != null; + assert row.version() != null; + + if (grp.sharedGroup() && (hld == null || hld.cctx.cacheId() != row.cacheId())) + hld = cacheMapHolder(ctx.cacheContext(row.cacheId())); + + assert hld != null; + + ctx.database().checkpointReadLock(); + + try { + while (true) { + GridCacheMapEntry cached = null; + + try { + cached = putEntryIfObsoleteOrAbsent( + hld, + hld.cctx, + grp.affinity().lastVersion(), + row.key(), + true, + false); + + cached.removeTombstone(row.version()); + + cached.touch(); + + break; + } + catch (GridCacheEntryRemovedException e) { + cached = null; + } + finally { + if (cached != null) + cached.touch(); + } + } + } + finally { + ctx.database().checkpointReadUnlock(); + } + + cntr++; + + if (cntr % stopCheckingFreq == 0) { + if (ctx.kernalContext().isStopping() || state() != OWNING) + break; + } + } + } + catch (IgniteCheckedException e) { + U.error(log, "Failed clear tombstone entries for partition: " + id, e); + } + } + /** * Removes all entries and rows from this partition. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index f27a311f210fe..df46885057635 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -498,6 +498,14 @@ protected void readFullRow( int len = PageUtils.getInt(addr, off); off += 4; + boolean tombstones = rowData == RowData.TOMBSTONES; + + if (tombstones && !sharedCtx.database().isTombstone(addr + off + len + 1)) { + verReady = true; + + return; + } + if (rowData != RowData.NO_KEY && rowData != RowData.NO_KEY_WITH_HINTS) { byte type = PageUtils.getByte(addr, off); off++; @@ -519,10 +527,13 @@ protected void readFullRow( byte type = PageUtils.getByte(addr, off); off++; - byte[] bytes = PageUtils.getBytes(addr, off, len); - off += len; + if (!tombstones) { + byte[] bytes = PageUtils.getBytes(addr, off, len); + + val = coctx.kernalContext().cacheObjects().toCacheObject(coctx, type, bytes); + } - val = coctx.kernalContext().cacheObjects().toCacheObject(coctx, type, bytes); + off += len; int verLen; @@ -941,7 +952,10 @@ public enum RowData { FULL_WITH_HINTS, /** Force instant hints actualization for update operation with history (to avoid races with vacuum). */ - NO_KEY_WITH_HINTS + NO_KEY_WITH_HINTS, + + /** Do not read row data for non-tombstone entries. */ + TOMBSTONES } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 96cbe2d625c9b..8b7ef11aa399e 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -120,6 +120,8 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; +import org.apache.ignite.internal.processors.cache.CacheObject; +import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.ExchangeActions; import org.apache.ignite.internal.processors.cache.GridCacheContext; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index d4bcbd868e390..427c0b9df0cb3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -2423,12 +2423,16 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, int partId) throws IgniteCheckedException { + @Override public void removeWithTombstone( + GridCacheContext cctx, + KeyCacheObject key, + GridCacheVersion ver, + GridDhtLocalPartition part) throws IgniteCheckedException { assert ctx.database().checkpointLockIsHeldByThread(); CacheDataStore delegate = init0(false); - delegate.removeWithTombstone(cctx, key, ver, partId); + delegate.removeWithTombstone(cctx, key, ver, part); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 69a5d5032fb16..d1ece85ba9c55 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -19,6 +19,7 @@ import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -45,10 +46,13 @@ import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider; import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider; import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.pagemem.PageUtils; import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl; import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.CacheObject; +import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.GridCacheMapEntry; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; @@ -130,6 +134,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** First eviction was warned flag. */ private volatile boolean firstEvictWarn; + /** */ + private CacheObject TOMBSTONE_VAL; /** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { @@ -145,8 +151,61 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap pageSize = memCfg.getPageSize(); initDataRegions(memCfg); + + TOMBSTONE_VAL = new CacheObjectImpl(null, cctx.marshaller().marshal(null)); + } + + public CacheObject tombstoneValue() { + return TOMBSTONE_VAL; + } + + public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { + if (row == null) + return false; + + CacheObject val = row.value(); + + assert val != null : row; + + if (val.cacheObjectType() == CacheObject.TYPE_REGULAR) { + byte[] nullBytes = TOMBSTONE_VAL.valueBytes(null); + byte[] bytes = val.valueBytes(null); + + if (Arrays.equals(nullBytes, bytes)) + return true; + } + + return false; } + public boolean isTombstone(long addr) throws IgniteCheckedException { + int off = 0; + + byte type = PageUtils.getByte(addr, off + 4); + + if (type != CacheObject.TYPE_REGULAR) + return false; + + byte[] nullBytes = TOMBSTONE_VAL.valueBytes(null); + + int len = PageUtils.getInt(addr, off); + + if (len != nullBytes.length) + return false; + + off += 5; + + for (int i = 0; i < len; i++) { + byte b = PageUtils.getByte(addr, off++); + + if (nullBytes[i] != b) + return false; + } + + return true; + } + + /** * @param cfg Ignite configuration. * @param groupName Name of group. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index 331fb64afdb1c..05962c4402f21 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -17,9 +17,11 @@ package org.apache.ignite.internal.processors.cache.distributed; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; @@ -28,8 +30,11 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2; +import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage; import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.lang.IgniteBiPredicate; +import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -45,7 +50,7 @@ import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; -import static org.apache.ignite.cache.CacheRebalanceMode.SYNC; +import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.cacheMetricsRegistryName; /** @@ -122,6 +127,8 @@ public void testRemoveAndRebalanceRaceAtomic() throws Exception { public void testRemoveAndRebalanceRaceTxWithPersistence() throws Exception { persistence = true; + cleanPersistenceDir(); + testRemoveAndRebalanceRace(TRANSACTIONAL, true); } @@ -169,8 +176,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea cache0.putAll(map); - TestRecordingCommunicationSpi.spi(ignite0).blockMessages(GridDhtPartitionSupplyMessageV2.class, - getTestIgniteInstanceName(1)); + blockRebalance(ignite0); IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() throws Exception { @@ -180,6 +186,12 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea IgniteEx ignite1 = (IgniteEx)fut.get(30_000); + if (persistence) { + ignite0.cluster().baselineAutoAdjustEnabled(false); + + ignite0.cluster().setBaselineTopology(2); + } + Set removed = new HashSet<>(); // Do removes while rebalance is in progress. @@ -195,7 +207,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); // On first node there should not be tombstones. - //assertEquals(0, tombstoneMetric0.get()); + assertEquals(0, tombstoneMetric0.get()); if (expTombstone) assertEquals(removed.size(), tombstoneMetric1.get()); @@ -213,7 +225,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea assert !removed.isEmpty(); - //assertEquals(0, tombstoneMetric0.get()); + assertEquals(0, tombstoneMetric0.get()); if (expTombstone) assertEquals(removed.size(), tombstoneMetric1.get()); @@ -242,6 +254,19 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea assertEquals(0, tombstoneMetric1.get()); } + /** + * + */ + private void blockRebalance(Ignite node) { + final int grpId = groupIdForCache(ignite(0), DEFAULT_CACHE_NAME); + + TestRecordingCommunicationSpi.spi(node).blockMessages(new IgniteBiPredicate() { + @Override public boolean apply(ClusterNode node, Message msg) { + return (msg instanceof GridDhtPartitionSupplyMessage) + && ((GridCacheGroupIdMessage)msg).groupId() == grpId; + } + }); + } /** * @param atomicityMode Cache atomicity mode. @@ -253,7 +278,7 @@ private CacheConfiguration cacheConfiguration(CacheAtomicityMo ccfg.setAtomicityMode(atomicityMode); ccfg.setCacheMode(PARTITIONED); ccfg.setBackups(2); - ccfg.setRebalanceMode(SYNC); + ccfg.setRebalanceMode(ASYNC); ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); return ccfg; From 86bc43e24e2a5cec9060470b8958631605e50963 Mon Sep 17 00:00:00 2001 From: sboikov Date: Tue, 23 Jul 2019 07:23:40 +0300 Subject: [PATCH 07/30] ignite-11704 --- .../processors/cache/CacheGroupContext.java | 2 +- .../cache/IgniteCacheOffheapManager.java | 2 -- .../cache/IgniteCacheOffheapManagerImpl.java | 15 +-------------- .../dht/topology/GridDhtLocalPartition.java | 3 --- .../cache/persistence/CacheDataRowAdapter.java | 4 +++- 5 files changed, 5 insertions(+), 21 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index ffff1016231f6..cbdb3e8a62364 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -1293,7 +1293,7 @@ public boolean hasAtomicCaches() { } public boolean supportsTombstone() { - return !hasAtomicCaches && !mvccEnabled && !isLocal(); + return !sharedGroup() && !hasAtomicCaches && !mvccEnabled && !isLocal(); } public boolean createTombstone(@Nullable GridDhtLocalPartition part) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index c88334359e398..227243970e122 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -453,8 +453,6 @@ public GridIterator cachePartitionIterator(int cacheId, final int */ public GridIterator partitionIterator(final int part, boolean withTombstones) throws IgniteCheckedException; - public GridIterator tombstonesIterator(final int part) throws IgniteCheckedException; - /** * @param part Partition number. * @param topVer Topology version. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index f653342ba33fe..66832377e9506 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -898,19 +898,6 @@ private Iterator cacheData(boolean primary, boolean backup, Affi return iterator(CU.UNDEFINED_CACHE_ID, singletonIterator(data), null, null, withTombstones); } - /** {@inheritDoc} */ - @Override public GridIterator tombstonesIterator(int part) { - assert locCacheDataStore == null; - - CacheDataStore data = partitionData(part); - - if (data == null) - return new GridEmptyCloseableIterator<>(); - - // TODO IGNITE-11704. - return iterator(CU.UNDEFINED_CACHE_ID, singletonIterator(data), null, null, true); - } - /** * * @param cacheId Cache ID. @@ -2940,7 +2927,7 @@ private void afterRowFound(@Nullable CacheDataRow row, KeyCacheObject key) throw @Override public GridCursor cursor(boolean withTombstones) throws IgniteCheckedException { GridCursor cur = dataTree.find(null, null); - return withTombstones ? cursorSkipEmpty(cur) : cursorSkipTombstone(cur); + return withTombstones ? cur : cursorSkipTombstone(cur); } private GridCursor cursorSkipEmpty(final GridCursor cur) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index b5a63e69d4497..d24cfee1f7d8d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -1157,9 +1157,6 @@ private void clearTombstones() { while (cur.next()) { CacheDataRow row = cur.get(); - if (!grp.offheap().isTombstone(row)) - continue; - assert row.key() != null; assert row.version() != null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index f1689038aa833..1be2bb5339f5f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -272,8 +272,10 @@ private void doInitFromLink( else if (rowData == TOMBSTONES) { // TODO IGNITE-11704. if (val != null && !sharedCtx.database().isTombstone(this)) { - verReady = true; ver = null; + key = null; + val = null; + verReady = true; return; } From 0a574a68385cf250b928972654b0486e2efe67a4 Mon Sep 17 00:00:00 2001 From: sboikov Date: Tue, 23 Jul 2019 10:42:28 +0300 Subject: [PATCH 08/30] ignite-11704 --- .../persistence/CacheDataRowAdapter.java | 13 +- .../CacheRemoveWithTombstonesLoadTest.java | 347 ++++++++++++++++++ .../testsuites/IgniteCacheTestSuite9.java | 2 + 3 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index 1be2bb5339f5f..5216e21184148 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -260,10 +260,19 @@ private void doInitFromLink( int itemId = itemId(nextLink); incomplete = readIncomplete(incomplete, sharedCtx, coctx, pageMem, - grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); + grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); + + if (incomplete == null) { + if (rowData == TOMBSTONES && val != null && !sharedCtx.database().isTombstone(this)) { + // TODO IGNITE-11704. + ver = null; + key = null; + val = null; + verReady = true; + } - if (incomplete == null) return; + } if (rowData == KEY_ONLY) { if (key != null) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java new file mode 100644 index 0000000000000..2780132421d29 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.spi.metric.LongMetric; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; +import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.cacheMetricsRegistryName; + +/** + * + */ +public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { + /** */ + private boolean persistence; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration(); + + if (persistence) { + dsCfg.setDefaultDataRegionConfiguration( + new DataRegionConfiguration().setMaxSize(100L * 1024 * 1024).setPersistenceEnabled(true)) + .setWalMode(WALMode.LOG_ONLY); + } + + dsCfg.setPageSize(1024); + + cfg.setDataStorageConfiguration(dsCfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(); + + cleanPersistenceDir(); + + super.afterTest(); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testRemoveAndRebalance() throws Exception { + removeAndRebalance(); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testRemoveAndRebalanceWithPersistence() throws Exception { + persistence = true; + + testRemoveAndRebalance(); + } + + /** + * @throws Exception If failed. + */ + private void removeAndRebalance() throws Exception { + IgniteEx ignite0 = startGrid(0); + + if (persistence) + ignite0.cluster().active(true); + + final int pageSize = ignite0.configuration().getDataStorageConfiguration().getPageSize(); + + assert pageSize > 0; + + IgniteCache cache0 = ignite0.createCache(cacheConfiguration()); + + ThreadLocalRandom rnd = ThreadLocalRandom.current(); + + List keys = new ArrayList<>(); + + Map data = new HashMap<>(); + + final int KEYS = 10_000; + final int ADD_NODES = 3; + + for (int i = 0; i < KEYS; i++) { + TestKey key = new TestKey(i, new byte[rnd.nextInt(pageSize * 3)]); + + keys.add(key); + + data.put(key, new TestValue(new byte[rnd.nextInt(pageSize * 3)])); + } + + cache0.putAll(data); + + AtomicInteger nodeIdx = new AtomicInteger(); + + for (int iter = 0; iter < ADD_NODES; iter++) { + IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { + @Override public Object call() throws Exception { + int idx = nodeIdx.incrementAndGet(); + + info("Start node: " + idx); + + return startGrid(idx); + } + }); + + long endTime = System.currentTimeMillis() + 5000; + + while (System.currentTimeMillis() < endTime) { + for (int i = 0; i < 100; i++) { + TestKey key = keys.get(rnd.nextInt(keys.size())); + + if (rnd.nextBoolean()) { + cache0.remove(key); + + data.remove(key); + } else { + TestValue val = new TestValue(new byte[rnd.nextInt(pageSize * 3)]); + + cache0.put(key, val); + data.put(key, val); + } + } + } + + fut.get(30_000); + + checkData(keys, data); + + waitTombstoneCleanup(); + + checkData(keys, data); + } + + awaitPartitionMapExchange(); + + for (int iter = 0; iter < ADD_NODES; iter++) { + IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { + @Override public Object call() throws Exception { + int idx = nodeIdx.getAndDecrement(); + + info("Stop node: " + idx); + + stopGrid(idx); + + awaitPartitionMapExchange(); + + return null; + } + }); + + long endTime = System.currentTimeMillis() + 5000; + + while (System.currentTimeMillis() < endTime) { + for (int i = 0; i < 100; i++) { + TestKey key = keys.get(rnd.nextInt(keys.size())); + + if (rnd.nextBoolean()) { + cache0.remove(key); + + data.remove(key); + } else { + TestValue val = new TestValue(new byte[rnd.nextInt(pageSize * 3)]); + + cache0.put(key, val); + data.put(key, val); + } + } + } + + fut.get(30_000); + + checkData(keys, data); + + waitTombstoneCleanup(); + + checkData(keys, data); + } + } + + private void checkData(List keys, Map data) { + for (Ignite node : Ignition.allGrids()) { + if (!node.name().endsWith("CacheRemoveWithTombstonesLoadTest1")) + continue; + + info("Check node: " + node.name()); + + IgniteCache cache = node.cache(DEFAULT_CACHE_NAME); + + for (TestKey key : keys) { + TestValue expVal = data.get(key); + TestValue val = cache.get(key); + + if (expVal == null) + assertNull(val); + else { + assertNotNull(val); + assertTrue(Arrays.equals(expVal.dummyData, val.dummyData)); + } + } + } + } + + /** + * @throws Exception If failed. + */ + private void waitTombstoneCleanup() throws Exception { + for (Ignite node : Ignition.allGrids()) { + final LongMetric tombstones = ((IgniteEx)node).context().metric().registry( + cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); + + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + return tombstones.get() == 0; + } + }, 30_000); + + assertEquals("Failed to wait for tombstone cleanup: " + node.name(), 0, tombstones.get()); + } + } + + /** + * @return Cache configuration. + */ + private CacheConfiguration cacheConfiguration() { + CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); + + ccfg.setAtomicityMode(TRANSACTIONAL); + ccfg.setCacheMode(PARTITIONED); + ccfg.setBackups(1); + ccfg.setRebalanceMode(ASYNC); + ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + + return ccfg; + } + + /** + * + */ + static class TestKey { + /** */ + private final int id; + + /** */ + private final byte[] dummyData; + + /** + * @param id ID. + * @param dummyData Dummy byte array (to test with various key sizes). + */ + public TestKey(int id, byte[] dummyData) { + this.id = id; + this.dummyData = dummyData; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + TestKey testKey = (TestKey) o; + + return id == testKey.id && Arrays.equals(dummyData, testKey.dummyData); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int result = Objects.hash(id); + result = 31 * result + Arrays.hashCode(dummyData); + return result; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "TestKey [id=" + id + "]"; + } + } + + /** + * + */ + static class TestValue { + /** */ + private final byte[] dummyData; + + /** + * @param dummyData Dummy byte array (to test with various value sizes). + */ + public TestValue(byte[] dummyData) { + this.dummyData = dummyData; + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java index 3927a34c6e2e0..708d4d091108a 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheLoadRebalanceEvictionSelfTest; import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest; import org.apache.ignite.internal.processors.cache.distributed.CacheOperationsInterruptTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheRemoveWithTombstonesLoadTest; import org.apache.ignite.internal.processors.cache.distributed.CacheRemoveWithTombstonesTest; import org.apache.ignite.internal.processors.cache.distributed.FailBackupOnAtomicOperationTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePrimarySyncTest; @@ -120,6 +121,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, FailBackupOnAtomicOperationTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheRemoveWithTombstonesTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheRemoveWithTombstonesLoadTest.class, ignoredTests); return suite; } From db1625276b61cba1b87d287a133b979031728c11 Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 24 Jul 2019 09:58:40 +0300 Subject: [PATCH 09/30] ignite-11704 --- .../persistence/CacheDataRowAdapter.java | 58 +++++++++---------- .../IgniteCacheDatabaseSharedManager.java | 11 +++- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index 5216e21184148..c0a421e9b54e5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -262,33 +262,8 @@ private void doInitFromLink( incomplete = readIncomplete(incomplete, sharedCtx, coctx, pageMem, grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); - if (incomplete == null) { - if (rowData == TOMBSTONES && val != null && !sharedCtx.database().isTombstone(this)) { - // TODO IGNITE-11704. - ver = null; - key = null; - val = null; - verReady = true; - } - + if (incomplete == null || (rowData == KEY_ONLY && key != null)) return; - } - - if (rowData == KEY_ONLY) { - if (key != null) - return; - } - else if (rowData == TOMBSTONES) { - // TODO IGNITE-11704. - if (val != null && !sharedCtx.database().isTombstone(this)) { - ver = null; - key = null; - val = null; - verReady = true; - - return; - } - } nextLink = incomplete.getNextLink(); } @@ -377,9 +352,7 @@ private IncompleteObject readIncomplete( buf.position(off); buf.limit(off + payloadSize); - boolean keyOnly = rowData == RowData.KEY_ONLY; - - incomplete = readFragment(sharedCtx, coctx, buf, keyOnly, readCacheId, incomplete, skipVer); + incomplete = readFragment(sharedCtx, coctx, buf, rowData, readCacheId, incomplete, skipVer); if (incomplete != null) incomplete.setNextLink(nextLink); @@ -416,11 +389,13 @@ protected IncompleteObject readFragment( GridCacheSharedContext sharedCtx, CacheObjectContext coctx, ByteBuffer buf, - boolean keyOnly, + RowData rowData, boolean readCacheId, IncompleteObject incomplete, boolean skipVer ) throws IgniteCheckedException { + boolean tombstones = rowData == TOMBSTONES; + if (readCacheId && cacheId == 0) { incomplete = readIncompleteCacheId(buf, incomplete); @@ -442,6 +417,12 @@ protected IncompleteObject readFragment( // Read key. if (key == null) { + if (tombstones && sharedCtx.database().isTombstone(buf, key, (IncompleteCacheObject)incomplete) == Boolean.FALSE) { + verReady = true; + + return null; + } + incomplete = readIncompleteKey(coctx, buf, (IncompleteCacheObject)incomplete); if (key == null) { @@ -449,7 +430,7 @@ protected IncompleteObject readFragment( return incomplete; // Need to finish reading the key. } - if (keyOnly) + if (rowData == RowData.KEY_ONLY) return null; // Key is ready - we are done! incomplete = null; @@ -468,6 +449,13 @@ protected IncompleteObject readFragment( // Read value. if (val == null) { + if (tombstones && sharedCtx.database().isTombstone(buf, key, (IncompleteCacheObject)incomplete) == Boolean.FALSE) { + key = null; + verReady = true; + + return null; + } + incomplete = readIncompleteValue(coctx, buf, (IncompleteCacheObject)incomplete); if (val == null) { @@ -478,6 +466,14 @@ protected IncompleteObject readFragment( incomplete = null; } + if (tombstones && !sharedCtx.database().isTombstone(this)) { + key = null; + val = null; + verReady = true; + + return null; + } + // Read version. if (!verReady) { incomplete = readIncompleteVersion(buf, incomplete, skipVer); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 16c43ef3f64f4..feb2e78c7152d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.cache.persistence; import java.io.File; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -55,6 +56,8 @@ import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.GridCacheMapEntry; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; +import org.apache.ignite.internal.processors.cache.IncompleteCacheObject; +import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.persistence.evict.FairFifoPageEvictionTracker; import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker; @@ -179,6 +182,13 @@ public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { return false; } + public Boolean isTombstone(ByteBuffer buf, + @Nullable KeyCacheObject key, + @Nullable IncompleteCacheObject incomplete) { + // TODO IGNITE-11704 + return null; + } + public boolean isTombstone(long addr) throws IgniteCheckedException { int off = 0; @@ -206,7 +216,6 @@ public boolean isTombstone(long addr) throws IgniteCheckedException { return true; } - /** * @param cfg Ignite configuration. * @param groupName Name of group. From 08b59ddcfb0739e9233ff46b0f438cc099604cc3 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 26 Jul 2019 10:29:12 +0300 Subject: [PATCH 10/30] ignite-11704 --- .../cache/IncompleteCacheObject.java | 4 + .../processors/cache/IncompleteObject.java | 2 +- .../IgniteCacheDatabaseSharedManager.java | 112 ++++++++++++++++-- 3 files changed, 105 insertions(+), 13 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java index dedb3bd98ef24..5f751257fc85e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java @@ -75,6 +75,10 @@ public IncompleteCacheObject(final ByteBuffer buf) { super.readData(buf); } + public int dataOffset() { + return off; + } + /** * @return Data type. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteObject.java index 7c24c12bcf5de..27c9def992545 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteObject.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteObject.java @@ -33,7 +33,7 @@ public class IncompleteObject { private T obj; /** */ - private int off; + protected int off; /** * @param data Data bytes. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index feb2e78c7152d..567c10455884f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -139,7 +139,10 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap private volatile boolean firstEvictWarn; /** */ - private CacheObject TOMBSTONE_VAL; + private byte[] tombstoneBytes; + + /** */ + private CacheObject tombstoneVal; /** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { @@ -156,11 +159,13 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap initDataRegions(memCfg); - TOMBSTONE_VAL = new CacheObjectImpl(null, cctx.marshaller().marshal(null)); + tombstoneBytes = cctx.marshaller().marshal(null); + + tombstoneVal = new CacheObjectImpl(null, tombstoneBytes); } public CacheObject tombstoneValue() { - return TOMBSTONE_VAL; + return tombstoneVal; } public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { @@ -172,22 +177,107 @@ public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { assert val != null : row; if (val.cacheObjectType() == CacheObject.TYPE_REGULAR) { - byte[] nullBytes = TOMBSTONE_VAL.valueBytes(null); byte[] bytes = val.valueBytes(null); - if (Arrays.equals(nullBytes, bytes)) + if (Arrays.equals(tombstoneBytes, bytes)) return true; } return false; } + /** + * @param buf Buffer. + * @param key Row key. + * @param incomplete Incomplete object. + * @return Tombstone flag or {@code null} if there is no enough data. + * @throws IgniteCheckedException If failed. + */ public Boolean isTombstone(ByteBuffer buf, @Nullable KeyCacheObject key, - @Nullable IncompleteCacheObject incomplete) { - // TODO IGNITE-11704 + @Nullable IncompleteCacheObject incomplete) throws IgniteCheckedException { + if (key == null) { + if (incomplete == null) { // Did not start read key yet. + if (buf.remaining() < IncompleteCacheObject.HEAD_LEN) { + return null; + } + + int keySize = buf.getInt(buf.position()); + + int headOffset = (IncompleteCacheObject.HEAD_LEN + keySize) /* key */ + + 8 /* expire time */; + + int requiredSize = headOffset + IncompleteCacheObject.HEAD_LEN; // Value header. + + if (buf.remaining() < requiredSize) + return null; + + return isTombstone(buf, headOffset); + } + else { // Reading key, check if there is enogh data to check value header. + byte[] data = incomplete.data(); + + if (data == null) // Header is not available yet. + return null; + + int keyRemaining = data.length - incomplete.dataOffset(); + + assert keyRemaining > 0 : keyRemaining; + + int headOffset = keyRemaining + 8 /* expire time */; + + int requiredSize = headOffset + IncompleteCacheObject.HEAD_LEN; // Value header. + + if (buf.remaining() < requiredSize) + return null; + + return isTombstone(buf, headOffset); + } + } + + if (incomplete == null) { // Did not start read value yet. + if (buf.remaining() < IncompleteCacheObject.HEAD_LEN) + return null; + + return isTombstone(buf, 0); + } + + byte[] data = incomplete.data(); + + if (data == null) // Header is not available yet. + return null; + + if (incomplete.type() != CacheObject.TYPE_REGULAR || data.length != tombstoneBytes.length) + return Boolean.FALSE; + return null; - } + } + + /** + * @param buf Buffer. + * @param offset Value offset. + * @return Tombstone flag or {@code null} if there is no enough data. + * @throws IgniteCheckedException If failed. + */ + private Boolean isTombstone(ByteBuffer buf, int offset) throws IgniteCheckedException { + int valLen = buf.getInt(buf.position() + offset); + if (valLen != tombstoneBytes.length) + return Boolean.FALSE; + + byte valType = buf.get(buf.position() + offset + 4); + if (valType != CacheObject.TYPE_REGULAR) + return Boolean.FALSE; + + if (buf.remaining() < (offset + 5 + tombstoneBytes.length)) + return null; + + for (int i = 0; i < tombstoneBytes.length; i++) { + if (tombstoneBytes[i] != buf.get(buf.position() + offset + 5 + i)) + return Boolean.FALSE; + } + + return Boolean.TRUE; + } public boolean isTombstone(long addr) throws IgniteCheckedException { int off = 0; @@ -197,11 +287,9 @@ public boolean isTombstone(long addr) throws IgniteCheckedException { if (type != CacheObject.TYPE_REGULAR) return false; - byte[] nullBytes = TOMBSTONE_VAL.valueBytes(null); - int len = PageUtils.getInt(addr, off); - if (len != nullBytes.length) + if (len != tombstoneBytes.length) return false; off += 5; @@ -209,7 +297,7 @@ public boolean isTombstone(long addr) throws IgniteCheckedException { for (int i = 0; i < len; i++) { byte b = PageUtils.getByte(addr, off++); - if (nullBytes[i] != b) + if (tombstoneBytes[i] != b) return false; } From 829449525fffdc4e0bac14c6b1905abe164f96d0 Mon Sep 17 00:00:00 2001 From: sboikov Date: Fri, 26 Jul 2019 11:13:19 +0300 Subject: [PATCH 11/30] ignite-11704 --- .../dht/topology/GridDhtLocalPartition.java | 6 +- .../dht/topology/PartitionsEvictManager.java | 344 ++++++++++++++---- .../IgniteCacheDatabaseSharedManager.java | 22 +- .../CacheRemoveWithTombstonesTest.java | 12 +- 4 files changed, 300 insertions(+), 84 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index d24cfee1f7d8d..90974633f5039 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -1138,13 +1138,13 @@ public void tombstoneCreated() { */ private void submitClearTombstones() { if (tombstoneCreated) - grp.shared().kernalContext().closure().runLocalSafe(this::clearTombstones, true); + grp.shared().evict().clearTombstonesAsync(grp, this); } /** * */ - private void clearTombstones() { + public void clearTombstones(EvictionContext evictionCtx) { final int stopCheckingFreq = 1000; CacheMapHolder hld = grp.sharedGroup() ? null : singleCacheEntryMap; @@ -1202,7 +1202,7 @@ private void clearTombstones() { cntr++; if (cntr % stopCheckingFreq == 0) { - if (ctx.kernalContext().isStopping() || state() != OWNING) + if (evictionCtx.shouldStop() || state() != OWNING) break; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java index 826902cee1a87..b2fd276a063d0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java @@ -21,6 +21,7 @@ import java.util.Comparator; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -104,6 +105,14 @@ public void onCacheGroupStopped(CacheGroupContext grp){ } } + public void clearTombstonesAsync(CacheGroupContext grp, GridDhtLocalPartition part) { + if (addAsyncTask(grp, part, TaskType.CLEAR_TOMBSTONES)) { + if (log.isDebugEnabled()) + log.debug("Partition has been scheduled for tomstones cleanup [grp=" + grp.cacheOrGroupName() + + ", p=" + part.id() + ", state=" + part.state() + "]"); + } + } + /** * Adds partition to eviction queue and starts eviction process if permit available. * @@ -111,29 +120,56 @@ public void onCacheGroupStopped(CacheGroupContext grp){ * @param part Partition to evict. */ public void evictPartitionAsync(CacheGroupContext grp, GridDhtLocalPartition part) { + if (addAsyncTask(grp, part, TaskType.EVICT)) { + if (log.isDebugEnabled()) + log.debug("Partition has been scheduled for eviction [grp=" + grp.cacheOrGroupName() + + ", p=" + part.id() + ", state=" + part.state() + "]"); + } + } + + /** + * @param grp Group context. + * @param part Partition. + * @param type Task type. + * @return {@code True} if task was added. + */ + private boolean addAsyncTask(CacheGroupContext grp, GridDhtLocalPartition part, TaskType type) { GroupEvictionContext grpEvictionCtx = evictionGroupsMap.computeIfAbsent( grp.groupId(), (k) -> new GroupEvictionContext(grp)); // Check node stop. if (grpEvictionCtx.shouldStop()) - return; + return false; int bucket; + AbstractEvictionTask task; + + switch (type) { + case EVICT: + task = new PartitionEvictionTask(part, grpEvictionCtx); + break; + + case CLEAR_TOMBSTONES: + task = new ClearTombstonesTask(part, grpEvictionCtx); + break; + + default: + throw new UnsupportedOperationException("Unsupported task type: " + type); + } + synchronized (mux) { - if (!grpEvictionCtx.partIds.add(part.id())) - return; + if (!grpEvictionCtx.taskIds.add(task.id)) + return false; - bucket = evictionQueue.offer(new PartitionEvictionTask(part, grpEvictionCtx)); + bucket = evictionQueue.offer(task); } - grpEvictionCtx.totalTasks.incrementAndGet(); + grpEvictionCtx.taskAdded(task); - if (log.isDebugEnabled()) - log.debug("Partition has been scheduled for eviction [grp=" + grp.cacheOrGroupName() - + ", p=" + part.id() + ", state=" + part.state() + "]"); + scheduleNextTask(bucket); - scheduleNextPartitionEviction(bucket); + return true; } /** @@ -141,7 +177,7 @@ public void evictPartitionAsync(CacheGroupContext grp, GridDhtLocalPartition par * * @param bucket Bucket. */ - private void scheduleNextPartitionEviction(int bucket) { + private void scheduleNextTask(int bucket) { // Check node stop. if (sharedEvictionCtx.shouldStop()) return; @@ -156,7 +192,7 @@ private void scheduleNextPartitionEviction(int bucket) { // Get task until we have permits. while (permits >= 0) { // Get task from bucket. - PartitionEvictionTask evictionTask = evictionQueue.poll(bucket); + AbstractEvictionTask evictionTask = evictionQueue.poll(bucket); // If bucket empty try get from another. if (evictionTask == null) { @@ -196,8 +232,8 @@ private void scheduleNextPartitionEviction(int bucket) { permits++; } - // Re-schedule new one task form same bucket. - scheduleNextPartitionEviction(bucket); + // Re-schedule new one task for same bucket. + scheduleNextTask(bucket); }); // Submit task to executor. @@ -217,10 +253,10 @@ private void showProgress() { int size = evictionQueue.size() + 1; // Queue size plus current partition. if (log.isInfoEnabled()) - log.info("Eviction in progress [permits=" + permits+ + log.info("Partition cleanup in progress [permits=" + permits+ ", threads=" + threads + ", groups=" + evictionGroupsMap.keySet().size() + - ", remainingPartsToEvict=" + size + "]"); + ", remainingTasks=" + size + "]"); evictionGroupsMap.values().forEach(GroupEvictionContext::showProgress); @@ -263,6 +299,28 @@ private void showProgress() { evictionGrps.forEach(GroupEvictionContext::awaitFinishAll); } + /** + * + */ + private class TasksStatistics { + private int total; + + private int inProgress; + + void taskAdded() { + total++; + } + + void taskStarted() { + inProgress++; + } + + void taskFinished() { + total--; + inProgress--; + } + } + /** * */ @@ -270,26 +328,29 @@ private class GroupEvictionContext implements EvictionContext { /** */ private final CacheGroupContext grp; - /** Deduplicate set partition ids. */ - private final Set partIds = new HashSet<>(); + /** Deduplicate set partition tasks. */ + private final Set taskIds = new HashSet<>(); /** Future for currently running partition eviction task. */ - private final Map> partsEvictFutures = new ConcurrentHashMap<>(); + private final Map> taskFutures = new ConcurrentHashMap<>(); /** Flag indicates that eviction process has stopped for this group. */ private volatile boolean stop; - /** Total partition to evict. */ + /** Total tasks. */ private AtomicInteger totalTasks = new AtomicInteger(); - /** Total partition evict in progress. */ - private int taskInProgress; + /** */ + private Map stats = U.newHashMap(2); /** * @param grp Group context. */ private GroupEvictionContext(CacheGroupContext grp) { this.grp = grp; + + for (TaskType type : TaskType.VALS) + stats.put(type, new TasksStatistics()); } /** {@inheritDoc} */ @@ -297,29 +358,35 @@ private GroupEvictionContext(CacheGroupContext grp) { return stop || sharedEvictionCtx.shouldStop(); } + void taskAdded(AbstractEvictionTask task) { + totalTasks.incrementAndGet(); + + synchronized (this) { + stats.get(task.id.type).taskAdded(); + } + } + /** * * @param task Partition eviction task. */ - private synchronized void taskScheduled(PartitionEvictionTask task) { + private synchronized void taskScheduled(AbstractEvictionTask task) { if (shouldStop()) return; - taskInProgress++; + stats.get(task.id.type).taskStarted(); GridFutureAdapter fut = task.finishFut; - int partId = task.part.id(); - - partIds.remove(partId); + taskIds.remove(task.id); - partsEvictFutures.put(partId, fut); + taskFutures.put(task.id, fut); fut.listen(f -> { synchronized (this) { - taskInProgress--; + stats.get(task.id.type).taskFinished(); - partsEvictFutures.remove(partId, f); + taskFutures.remove(task.id, f); if (totalTasks.decrementAndGet() == 0) evictionGroupsMap.remove(grp.groupId()); @@ -338,7 +405,7 @@ private void stop() { * Await evict finish. */ private void awaitFinishAll(){ - partsEvictFutures.forEach(this::awaitFinish); + taskFutures.forEach(this::awaitFinish); evictionGroupsMap.remove(grp.groupId()); } @@ -346,17 +413,17 @@ private void awaitFinishAll(){ /** * Await evict finish partition. */ - private void awaitFinish(Integer part, IgniteInternalFuture fut) { + private void awaitFinish(TaskId taskId, IgniteInternalFuture fut) { // Wait for last offered partition eviction completion try { - log.info("Await partition evict, grpName=" + grp.cacheOrGroupName() + - ", grpId=" + grp.groupId() + ", partId=" + part); + log.info("Await partition cleanup [grpName=" + grp.cacheOrGroupName() + + ", grpId=" + grp.groupId() + ", task=" + taskId.type + ", partId=" + taskId.part + ']'); fut.get(); } catch (IgniteCheckedException e) { if (log.isDebugEnabled()) - log.warning("Failed to await partition eviction during stopping.", e); + log.warning("Failed to await partition cleanup during stopping.", e); } } @@ -364,47 +431,132 @@ private void awaitFinish(Integer part, IgniteInternalFuture fut) { * Shows progress group of eviction. */ private void showProgress() { - if (log.isInfoEnabled()) - log.info("Group eviction in progress [grpName=" + grp.cacheOrGroupName()+ - ", grpId=" + grp.groupId() + - ", remainingPartsToEvict=" + (totalTasks.get() - taskInProgress) + - ", partsEvictInProgress=" + taskInProgress + - ", totalParts= " + grp.topology().localPartitions().size() + "]"); + if (log.isInfoEnabled()) { + StringBuilder msg = new StringBuilder( + "Group cleanup in progress [grpName=" + grp.cacheOrGroupName() + ", grpId=" + grp.groupId()); + + synchronized (this) { + TasksStatistics evicts = stats.get(TaskType.EVICT); + if (evicts.total > 0) { + msg.append(", remainingPartsToEvict=" + (evicts.total - evicts.inProgress)). + append(", partsEvictInProgress=" + evicts.inProgress); + } + + TasksStatistics tombstones = stats.get(TaskType.CLEAR_TOMBSTONES); + if (tombstones.total > 0) { + msg.append(", remainingPartsToClearTombstones=" + (tombstones.total - tombstones.inProgress)). + append(", tombstoneClearInProgress=" + tombstones.inProgress); + } + } + + msg.append(", totalParts= " + grp.topology().localPartitions().size() + "]"); + + log.info(msg.toString()); + } } } /** - * Task for self-scheduled partition eviction / clearing. + * */ - private class PartitionEvictionTask implements Runnable { + private enum TaskType { + /** */ + EVICT, + + /** */ + CLEAR_TOMBSTONES; + + /** */ + private static TaskType[] VALS = values(); + } + + /** + * + */ + private static class TaskId { + /** */ + final int part; + + /** */ + final TaskType type; + + /** + * @param part Partiotion id. + * @param type Task type. + */ + TaskId(int part, TaskType type) { + this.part = part; + this.type = type; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + TaskId taskKey = (TaskId)o; + + return part == taskKey.part && type == taskKey.type; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(part, type); + } + } + + /** + * + */ + private abstract class AbstractEvictionTask implements Runnable { /** Partition to evict. */ - private final GridDhtLocalPartition part; + protected final GridDhtLocalPartition part; /** */ - private final long size; + protected final long size; /** Eviction context. */ - private final GroupEvictionContext grpEvictionCtx; + protected final GroupEvictionContext grpEvictionCtx; /** */ - private final GridFutureAdapter finishFut = new GridFutureAdapter<>(); + protected final GridFutureAdapter finishFut = new GridFutureAdapter<>(); + + /** */ + private final TaskId id; /** * @param part Partition. * @param grpEvictionCtx Eviction context. */ - private PartitionEvictionTask( - GridDhtLocalPartition part, - GroupEvictionContext grpEvictionCtx + private AbstractEvictionTask( + GridDhtLocalPartition part, + GroupEvictionContext grpEvictionCtx, + TaskType type ) { this.part = part; this.grpEvictionCtx = grpEvictionCtx; + id = new TaskId(part.id(), type); + size = part.fullSize(); } + /** + * @return {@code False} if need retry task later. + * @throws IgniteCheckedException If failed. + */ + abstract boolean run0() throws IgniteCheckedException; + + /** + * + */ + abstract void scheduleRetry(); + /** {@inheritDoc} */ - @Override public void run() { + @Override public final void run() { if (grpEvictionCtx.shouldStop()) { finishFut.onDone(); @@ -412,16 +564,7 @@ private PartitionEvictionTask( } try { - assert part.state() != GridDhtPartitionState.OWNING : part; - - boolean success = part.tryClear(grpEvictionCtx); - - assert part.state() != GridDhtPartitionState.OWNING : part; - - if (success) { - if (part.state() == GridDhtPartitionState.EVICTED && part.markForDestroy()) - part.destroy(); - } + boolean success = run0(); // Complete eviction future before schedule new to prevent deadlock with // simultaneous eviction stopping and scheduling new eviction. @@ -429,7 +572,7 @@ private PartitionEvictionTask( // Re-offer partition if clear was unsuccessful due to partition reservation. if (!success) - evictPartitionAsync(grpEvictionCtx.grp, part); + scheduleRetry(); } catch (Throwable ex) { finishFut.onDone(ex); @@ -445,12 +588,77 @@ private PartitionEvictionTask( } } + /** + * Task for self-scheduled partition eviction / clearing. + */ + private class PartitionEvictionTask extends AbstractEvictionTask { + /** + * @param part Partition. + * @param grpEvictionCtx Eviction context. + */ + private PartitionEvictionTask( + GridDhtLocalPartition part, + GroupEvictionContext grpEvictionCtx + ) { + super(part, grpEvictionCtx, TaskType.EVICT); + } + + /** {@inheritDoc} */ + @Override void scheduleRetry() { + evictPartitionAsync(grpEvictionCtx.grp, part); + } + + /** {@inheritDoc} */ + @Override public boolean run0() throws IgniteCheckedException { + assert part.state() != GridDhtPartitionState.OWNING : part; + + boolean success = part.tryClear(grpEvictionCtx); + + assert part.state() != GridDhtPartitionState.OWNING : part; + + if (success) { + if (part.state() == GridDhtPartitionState.EVICTED && part.markForDestroy()) + part.destroy(); + } + + return success; + } + } + + /** + * + */ + private class ClearTombstonesTask extends AbstractEvictionTask { + /** + * @param part Partition. + * @param grpEvictionCtx Eviction context. + */ + private ClearTombstonesTask( + GridDhtLocalPartition part, + GroupEvictionContext grpEvictionCtx + ) { + super(part, grpEvictionCtx, TaskType.CLEAR_TOMBSTONES); + } + + /** {@inheritDoc} */ + @Override void scheduleRetry() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public boolean run0() throws IgniteCheckedException { + part.clearTombstones(grpEvictionCtx); + + return true; + } + } + /** * */ private class BucketQueue { /** Queues contains partitions scheduled for eviction. */ - private final Queue[] buckets; + private final Queue[] buckets; /** */ private final long[] bucketSizes; @@ -473,8 +681,8 @@ private class BucketQueue { * @param bucket Bucket index. * @return Partition evict task, or {@code null} if bucket queue is empty. */ - PartitionEvictionTask poll(int bucket) { - PartitionEvictionTask task = buckets[bucket].poll(); + AbstractEvictionTask poll(int bucket) { + AbstractEvictionTask task = buckets[bucket].poll(); if (task != null) bucketSizes[bucket] -= task.size; @@ -487,7 +695,7 @@ PartitionEvictionTask poll(int bucket) { * * @return Partition evict task. */ - PartitionEvictionTask pollAny() { + AbstractEvictionTask pollAny() { for (int bucket = 0; bucket < bucketSizes.length; bucket++){ if (!buckets[bucket].isEmpty()) return poll(bucket); @@ -502,7 +710,7 @@ PartitionEvictionTask pollAny() { * @param task Eviction task. * @return Bucket index. */ - int offer(PartitionEvictionTask task) { + int offer(AbstractEvictionTask task) { int bucket = calculateBucket(); buckets[bucket].offer(task); @@ -526,7 +734,7 @@ boolean isEmpty(){ int size(){ int size = 0; - for (Queue queue : buckets) + for (Queue queue : buckets) size += queue.size(); return size; @@ -556,7 +764,7 @@ private int calculateBucket() { * * @return Queue for evict partitions. */ - private Queue createEvictPartitionQueue() { + private Queue createEvictPartitionQueue() { switch (QUEUE_TYPE) { case 1: return new PriorityBlockingQueue<>( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 567c10455884f..88521f82c3071 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -164,11 +164,19 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap tombstoneVal = new CacheObjectImpl(null, tombstoneBytes); } + /** + * @return Value to be stored for removed entry. + */ public CacheObject tombstoneValue() { return tombstoneVal; } - public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { + /** + * @param row Row. + * @return {@code True} if given row is tombstone. + * @throws IgniteCheckedException If failed. + */ + public boolean isTombstone(@Nullable CacheDataRow row) throws IgniteCheckedException { if (row == null) return false; @@ -191,11 +199,10 @@ public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { * @param key Row key. * @param incomplete Incomplete object. * @return Tombstone flag or {@code null} if there is no enough data. - * @throws IgniteCheckedException If failed. */ public Boolean isTombstone(ByteBuffer buf, @Nullable KeyCacheObject key, - @Nullable IncompleteCacheObject incomplete) throws IgniteCheckedException { + @Nullable IncompleteCacheObject incomplete) { if (key == null) { if (incomplete == null) { // Did not start read key yet. if (buf.remaining() < IncompleteCacheObject.HEAD_LEN) { @@ -257,9 +264,8 @@ public Boolean isTombstone(ByteBuffer buf, * @param buf Buffer. * @param offset Value offset. * @return Tombstone flag or {@code null} if there is no enough data. - * @throws IgniteCheckedException If failed. */ - private Boolean isTombstone(ByteBuffer buf, int offset) throws IgniteCheckedException { + private Boolean isTombstone(ByteBuffer buf, int offset) { int valLen = buf.getInt(buf.position() + offset); if (valLen != tombstoneBytes.length) return Boolean.FALSE; @@ -279,7 +285,11 @@ private Boolean isTombstone(ByteBuffer buf, int offset) throws IgniteCheckedExce return Boolean.TRUE; } - public boolean isTombstone(long addr) throws IgniteCheckedException { + /** + * @param addr Row address. + * @return {@code True} if stored value is tombstone. + */ + public boolean isTombstone(long addr) { int off = 0; byte type = PageUtils.getByte(addr, off + 4); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index 05962c4402f21..dcd7f6c020604 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -127,9 +127,7 @@ public void testRemoveAndRebalanceRaceAtomic() throws Exception { public void testRemoveAndRebalanceRaceTxWithPersistence() throws Exception { persistence = true; - cleanPersistenceDir(); - - testRemoveAndRebalanceRace(TRANSACTIONAL, true); + testRemoveAndRebalanceRaceTx(); } /** @@ -139,7 +137,7 @@ public void testRemoveAndRebalanceRaceTxWithPersistence() throws Exception { public void testRemoveAndRebalanceRaceTxMvccWithPersistence() throws Exception { persistence = true; - testRemoveAndRebalanceRace(TRANSACTIONAL_SNAPSHOT, false); + testRemoveAndRebalanceRaceTxMvcc(); } /** @@ -149,7 +147,7 @@ public void testRemoveAndRebalanceRaceTxMvccWithPersistence() throws Exception { public void testRemoveAndRebalanceRaceAtomicWithPersistence() throws Exception { persistence = true; - testRemoveAndRebalanceRace(ATOMIC, false); + testRemoveAndRebalanceRaceAtomic(); } /** @@ -164,7 +162,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea IgniteCache cache0 = ignite0.createCache(cacheConfiguration(atomicityMode)); - LongMetric tombstoneMetric0 = (LongMetric)ignite0.context().metric().registry( + LongMetric tombstoneMetric0 = ignite0.context().metric().registry( cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); Map map = new HashMap<>(); @@ -203,7 +201,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea } } - final LongMetric tombstoneMetric1 = (LongMetric)ignite1.context().metric().registry( + final LongMetric tombstoneMetric1 = ignite1.context().metric().registry( cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); // On first node there should not be tombstones. From 8299a2cce28247a1ccb344e314884ea92f1e1b3c Mon Sep 17 00:00:00 2001 From: sboikov Date: Sun, 28 Jul 2019 18:10:02 +0300 Subject: [PATCH 12/30] ignite-11704 --- .../processors/cache/CacheGroupContext.java | 7 +++++ .../processors/cache/CacheMetricsImpl.java | 6 ++++ .../processors/cache/GridCacheContext.java | 6 ++++ .../processors/cache/GridCacheMapEntry.java | 5 ++++ .../cache/IgniteCacheOffheapManager.java | 9 ++++++ .../cache/IgniteCacheOffheapManagerImpl.java | 29 +++++++++++++++++++ .../cache/IncompleteCacheObject.java | 3 ++ .../dht/topology/GridDhtLocalPartition.java | 14 ++------- .../dht/topology/PartitionsEvictManager.java | 18 ++++++++++++ .../persistence/CacheDataRowAdapter.java | 4 +-- .../CacheDeferredDeleteSanitySelfTest.java | 6 ++-- .../visor/verify/ValidateIndexesClosure.java | 2 +- ...teCacheLockPartitionOnAffinityRunTest.java | 4 +-- .../query/h2/GridIndexRebuildSelfTest.java | 2 +- ...idIndexRebuildWithMvccEnabledSelfTest.java | 2 +- .../util/GridCommandHandlerIndexingTest.java | 2 +- 16 files changed, 97 insertions(+), 22 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index cbdb3e8a62364..3a7a445de901d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -1292,10 +1292,17 @@ public boolean hasAtomicCaches() { return hasAtomicCaches; } + /** + * @return {@code True} if need create temporary tombstones entries for removed data. + */ public boolean supportsTombstone() { return !sharedGroup() && !hasAtomicCaches && !mvccEnabled && !isLocal(); } + /** + * @param part Partition. + * @return {@code True} if need create tombstone for remove in given partition. + */ public boolean createTombstone(@Nullable GridDhtLocalPartition part) { return part != null && supportsTombstone() && part.state() == GridDhtPartitionState.MOVING; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java index e0020374b3b38..4e993ef70f570 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java @@ -1013,10 +1013,16 @@ public void addPutAndGetTimeNanos(long duration) { delegate.addPutAndGetTimeNanos(duration); } + /** + * Increments tombstones counter. + */ public void tombstoneCreated() { tombstones.increment(); } + /** + * Decrements tombstones counter. + */ public void tombstoneRemoved() { tombstones.decrement(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 5ac56db0dcfa5..959095c3189fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -2372,6 +2372,9 @@ protected Object readResolve() throws ObjectStreamException { } } + /** + * Increments tombstones counter. + */ public void tombstoneCreated() { GridCacheAdapter cache = this.cache; @@ -2379,6 +2382,9 @@ public void tombstoneCreated() { cache.metrics0().tombstoneCreated(); } + /** + * Decrements tombstones counter. + */ public void tombstoneRemoved() { GridCacheAdapter cache = this.cache; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 08986a9604c05..1133085b16d77 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -5711,6 +5711,11 @@ private LazyValueEntry(KeyCacheObject key, boolean keepBinary) { } } + /** + * @param row Data row. + * @return {@code True} if row expired. + * @throws IgniteCheckedException If failed. + */ private boolean checkRowExpired(CacheDataRow row) throws IgniteCheckedException { assert row != null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index 227243970e122..e06805983299a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -409,6 +409,11 @@ public void removeWithTombstone( GridDhtLocalPartition part ) throws IgniteCheckedException; + /** + * @param row Data row. + * @return {@code True} if give row is tombstone. + * @throws IgniteCheckedException If failed. + */ public boolean isTombstone(@Nullable CacheDataRow row) throws IgniteCheckedException; /** @@ -448,6 +453,7 @@ public GridIterator cachePartitionIterator(int cacheId, final int /** * @param part Partition number. + * @param withTombstones {@code True} if should return tombstone entries. * @return Iterator for given partition. * @throws IgniteCheckedException If failed. */ @@ -959,6 +965,7 @@ List> mvccFindAllVersions(GridCacheContext cc throws IgniteCheckedException; /** + * @param withTombstones {@code True} if should return tombstone entries. * @return Data cursor. * @throws IgniteCheckedException If failed. */ @@ -980,6 +987,7 @@ List> mvccFindAllVersions(GridCacheContext cc /** * @param cacheId Cache ID. + * @param withTombstones {@code True} if should return tombstone entries. * @return Data cursor. * @throws IgniteCheckedException If failed. */ @@ -1021,6 +1029,7 @@ public GridCursor cursor(int cacheId, KeyCacheObject low * @param upper Upper bound. * @param x Implementation specific argument, {@code null} always means that we need to return full detached data row. * @param snapshot Mvcc snapshot. + * @param withTombstones {@code True} if should return tombstone entries. * @return Data cursor. * @throws IgniteCheckedException If failed. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 66832377e9506..8aa5514918d9e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -632,6 +632,7 @@ private Iterator cacheData(boolean primary, boolean backup, Affi dataStore(part).removeWithTombstone(cctx, key, ver, part); } + /** {@inheritDoc} */ @Override public boolean isTombstone(CacheDataRow row) throws IgniteCheckedException { if (!grp.supportsTombstone()) return false; @@ -904,6 +905,7 @@ private Iterator cacheData(boolean primary, boolean backup, Affi * @param dataIt Data store iterator. * @param mvccSnapshot Mvcc snapshot. * @param dataPageScanEnabled Flag to enable data page scan. + * @param withTombstones {@code True} if should return tombstone entries. * @return Rows iterator */ private GridCloseableIterator iterator(final int cacheId, @@ -979,6 +981,7 @@ private GridCloseableIterator iterator(final int cacheId, /** * @param cacheId Cache ID. * @param dataIt Data store iterator. + * @param withTombstones {@code True} if should return tombstone entries. * @return Rows iterator */ private GridCloseableIterator evictionSafeIterator( @@ -2682,6 +2685,11 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn /** */ private CacheDataRow newRow; + /** + * @param cctx Context. + * @param key Key. + * @param ver Version. + */ RemoveWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver) { this.cctx = cctx; this.key = key; @@ -2753,6 +2761,7 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn * @param cctx Cache context. * @param key Key. * @param oldRow Removed row. + * @tombstoneRow Tombstone row (if tombstone was created for remove). * @throws IgniteCheckedException If failed. */ private void finishRemove(GridCacheContext cctx, @@ -2799,6 +2808,11 @@ private void clearPendingEntries(GridCacheContext cctx, CacheDataRow oldRow) pendingTree().removex(new PendingRow(cacheId, oldRow.expireTime(), oldRow.link())); } + /** + * @param row Data row. + * @return {@code Null} if given row is tombstone, otherwise row itself. + * @throws IgniteCheckedException If null. + */ @Nullable private CacheDataRow checkTombstone(@Nullable CacheDataRow row) throws IgniteCheckedException { return grp.offheap().isTombstone(row) ? null : row; } @@ -2930,17 +2944,24 @@ private void afterRowFound(@Nullable CacheDataRow row, KeyCacheObject key) throw return withTombstones ? cur : cursorSkipTombstone(cur); } + /** + * @param cur Cursor. + * @return Cursor skipping non-tombstone entries. + */ private GridCursor cursorSkipEmpty(final GridCursor cur) { if (!grp.supportsTombstone()) return cur; return new GridCursor() { + /** */ CacheDataRow next; + /** {@inheritDoc} */ @Override public boolean next() throws IgniteCheckedException { while (cur.next()) { CacheDataRow next = cur.get(); + // If request cursor with RowData.TOMBSTONES, then for non-tombtones all fields are null. if (next.version() != null) { this.next = next; @@ -2951,19 +2972,26 @@ private GridCursor cursorSkipEmpty(final GridCursor cursorSkipTombstone(final GridCursor cur) { if (!grp.supportsTombstone()) return cur; return new GridCursor() { + /** */ CacheDataRow next; + /** {@inheritDoc} */ @Override public boolean next() throws IgniteCheckedException { while (cur.next()) { CacheDataRow next = cur.get(); @@ -2978,6 +3006,7 @@ private GridCursor cursorSkipTombstone(final GridCursor< return false; } + /** {@inheritDoc} */ @Override public CacheDataRow get() { return next; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java index 5f751257fc85e..6b21145ba7a12 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java @@ -75,6 +75,9 @@ public IncompleteCacheObject(final ByteBuffer buf) { super.readData(buf); } + /** + * @return Size of already read data. + */ public int dataOffset() { return off; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 90974633f5039..d9b34c61254bd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -176,7 +176,7 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements /** Set if topology update sequence should be updated on partition destroy. */ private boolean updateSeqOnDestroy; - /** */ + /** Set if tombstone was created in partition. */ private volatile boolean tombstoneCreated; /** @@ -626,8 +626,8 @@ public boolean own() { assert partState == MOVING || partState == LOST; if (casState(state, OWNING)) { - if (grp.supportsTombstone()) - submitClearTombstones(); + if (grp.supportsTombstone() && tombstoneCreated) + grp.shared().evict().clearTombstonesAsync(grp, this); return true; } @@ -1133,14 +1133,6 @@ public void tombstoneCreated() { tombstoneCreated = true; } - /** - * - */ - private void submitClearTombstones() { - if (tombstoneCreated) - grp.shared().evict().clearTombstonesAsync(grp, this); - } - /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java index b2fd276a063d0..58db3c372cd6d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java @@ -105,6 +105,10 @@ public void onCacheGroupStopped(CacheGroupContext grp){ } } + /** + * @param grp Group context. + * @param part Partition to clear tombstones. + */ public void clearTombstonesAsync(CacheGroupContext grp, GridDhtLocalPartition part) { if (addAsyncTask(grp, part, TaskType.CLEAR_TOMBSTONES)) { if (log.isDebugEnabled()) @@ -303,18 +307,29 @@ private void showProgress() { * */ private class TasksStatistics { + /** */ private int total; + /** */ private int inProgress; + /** + * + */ void taskAdded() { total++; } + /** + * + */ void taskStarted() { inProgress++; } + /** + * + */ void taskFinished() { total--; inProgress--; @@ -358,6 +373,9 @@ private GroupEvictionContext(CacheGroupContext grp) { return stop || sharedEvictionCtx.shouldStop(); } + /** + * @param task Task. + */ void taskAdded(AbstractEvictionTask task) { totalTasks.incrementAndGet(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index c0a421e9b54e5..a3b876d76510b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -260,7 +260,7 @@ private void doInitFromLink( int itemId = itemId(nextLink); incomplete = readIncomplete(incomplete, sharedCtx, coctx, pageMem, - grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); + grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); if (incomplete == null || (rowData == KEY_ONLY && key != null)) return; @@ -378,7 +378,7 @@ protected int readHeader(GridCacheSharedContext sharedCtx, long addr, int * @param sharedCtx Cache shared context. * @param coctx Cache object context. * @param buf Buffer. - * @param keyOnly {@code true} If need to read only key object. + * @param rowData Required row data. * @param readCacheId {@code true} If need to read cache ID. * @param incomplete Incomplete object. * @param skipVer Whether version read should be skipped. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteSanitySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteSanitySelfTest.java index 69a19f47816e0..8c2bd1db35fcf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteSanitySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteSanitySelfTest.java @@ -51,10 +51,10 @@ public void testDeferredDelete() throws Exception { testDeferredDelete(LOCAL, TRANSACTIONAL, false, false); testDeferredDelete(PARTITIONED, ATOMIC, false, true); - testDeferredDelete(PARTITIONED, TRANSACTIONAL, false, true); + testDeferredDelete(PARTITIONED, TRANSACTIONAL, false, false); testDeferredDelete(REPLICATED, ATOMIC, false, true); - testDeferredDelete(REPLICATED, TRANSACTIONAL, false, true); + testDeferredDelete(REPLICATED, TRANSACTIONAL, false, false); // Near testDeferredDelete(LOCAL, ATOMIC, true, false); @@ -64,7 +64,7 @@ public void testDeferredDelete() throws Exception { testDeferredDelete(PARTITIONED, TRANSACTIONAL, true, false); testDeferredDelete(REPLICATED, ATOMIC, true, true); - testDeferredDelete(REPLICATED, TRANSACTIONAL, true, true); + testDeferredDelete(REPLICATED, TRANSACTIONAL, true, false); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java index c7fa349407f28..fd4fbc6176de9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java @@ -446,7 +446,7 @@ private Map processPartition( long partSize = part.dataStore().fullSize(); - GridIterator it = grpCtx.offheap().partitionIterator(part.id()); + GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); Object consId = ignite.context().discovery().localNode().consistentId(); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java index 23fe4bae6dc6b..c6e0575843239 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java @@ -85,7 +85,7 @@ private static int getOrganizationCountFromPartitionMap(final IgniteEx ignite, int cnt = 0; - GridCursor c = pOrgs.dataStore().cursor(); + GridCursor c = pOrgs.dataStore().cursor(false); CacheObjectContext ctx = cacheAdapterOrg.context().cacheObjectContext(); @@ -120,7 +120,7 @@ private static int getPersonsCountFromPartitionMap(final IgniteEx ignite, int or int cnt = 0; - GridCursor c = pPers.dataStore().cursor(); + GridCursor c = pPers.dataStore().cursor(false); CacheObjectContext ctx = cacheAdapterPers.context().cacheObjectContext(); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java index b4bb9e9112150..71588d5cc465a 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildSelfTest.java @@ -160,7 +160,7 @@ protected void checkDataState(IgniteEx srv, boolean afterRebuild) throws IgniteC assertNotNull(icache); for (IgniteCacheOffheapManager.CacheDataStore store : icache.context().offheap().cacheDataStores()) { - GridCursor cur = store.cursor(); + GridCursor cur = store.cursor(false); while (cur.next()) { CacheDataRow row = cur.get(); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildWithMvccEnabledSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildWithMvccEnabledSelfTest.java index 72118433badf6..528e5b5963322 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildWithMvccEnabledSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexRebuildWithMvccEnabledSelfTest.java @@ -98,7 +98,7 @@ private static void lockVersion(IgniteEx node) throws IgniteCheckedException { CacheObjectContext coCtx = icache.context().cacheObjectContext(); for (IgniteCacheOffheapManager.CacheDataStore store : icache.context().offheap().cacheDataStores()) { - GridCursor cur = store.cursor(); + GridCursor cur = store.cursor(false); while (cur.next()) { CacheDataRow row = cur.get(); diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java index a6f545614f781..2752617a407a2 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java @@ -374,7 +374,7 @@ private void breakSqlIndex(Ignite ig, String cacheName) throws Exception { GridDhtLocalPartition locPart = ctx.topology().localPartitions().get(0); - GridIterator it = ctx.group().offheap().partitionIterator(locPart.id()); + GridIterator it = ctx.group().offheap().partitionIterator(locPart.id(), false); for (int i = 0; i < 500; i++) { if (!it.hasNextX()) { From 10d24bdc128ee7472e4d5a320086643ccb88b3f5 Mon Sep 17 00:00:00 2001 From: sboikov Date: Tue, 30 Jul 2019 07:29:26 +0300 Subject: [PATCH 13/30] ignite-11704 --- .../CacheRemoveWithTombstonesLoadTest.java | 8 +++++++ .../CacheRemoveWithTombstonesTest.java | 22 ++++--------------- .../testframework/MvccFeatureChecker.java | 3 ++- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java index 2780132421d29..cb144b9eea383 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java @@ -31,6 +31,7 @@ import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -75,6 +76,13 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { return cfg; } + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + MvccFeatureChecker.skipIfNotSupported(MvccFeatureChecker.Feature.TOMBSTONES); + } + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index dcd7f6c020604..cb4ad69f5c265 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -37,6 +37,7 @@ import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -104,14 +105,6 @@ public void testRemoveAndRebalanceRaceTx() throws Exception { testRemoveAndRebalanceRace(TRANSACTIONAL, true); } - /** - * @throws Exception If failed. - */ - @Test - public void testRemoveAndRebalanceRaceTxMvcc() throws Exception { - testRemoveAndRebalanceRace(TRANSACTIONAL_SNAPSHOT, false); - } - /** * @throws Exception If failed. */ @@ -130,16 +123,6 @@ public void testRemoveAndRebalanceRaceTxWithPersistence() throws Exception { testRemoveAndRebalanceRaceTx(); } - /** - * @throws Exception If failed. - */ - @Test - public void testRemoveAndRebalanceRaceTxMvccWithPersistence() throws Exception { - persistence = true; - - testRemoveAndRebalanceRaceTxMvcc(); - } - /** * @throws Exception If failed. */ @@ -162,6 +145,9 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea IgniteCache cache0 = ignite0.createCache(cacheConfiguration(atomicityMode)); + if (MvccFeatureChecker.forcedMvcc()) + expTombstone = false; + LongMetric tombstoneMetric0 = ignite0.context().metric().registry( cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java index 1daeab46130ed..dce8c8dd9a502 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java @@ -47,7 +47,8 @@ public enum Feature { EVICTION, EXPIRATION, METRICS, - INTERCEPTOR + INTERCEPTOR, + TOMBSTONES } /** From 5010c38efd35d6a900159a85b82360f99f24edb8 Mon Sep 17 00:00:00 2001 From: sboikov Date: Tue, 30 Jul 2019 08:05:09 +0300 Subject: [PATCH 14/30] ignite-11704 --- .../processors/cache/IgniteCacheOffheapManager.java | 10 +++++++--- .../persistence/GridCacheDatabaseSharedManager.java | 2 -- .../distributed/CacheRemoveWithTombstonesTest.java | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index e06805983299a..aca75b69af329 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -398,7 +398,7 @@ public void remove( /** * @param cctx Cache context. * @param key Key. - * @param partId Partition number. + * @param ver Version. * @param part Partition. * @throws IgniteCheckedException If failed. */ @@ -919,10 +919,14 @@ void mvccApplyUpdate(GridCacheContext cctx, * @param cctx Cache context. * @param key Key. * @param ver Version. - * @param partId Partition number. + * @param part Partition. * @throws IgniteCheckedException If failed. */ - public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, GridDhtLocalPartition part) throws IgniteCheckedException; + public void removeWithTombstone( + GridCacheContext cctx, + KeyCacheObject key, + GridCacheVersion ver, + GridDhtLocalPartition part) throws IgniteCheckedException; /** * @param cctx Cache context. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 3fd36a0244d91..b1a21de6b1376 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -121,8 +121,6 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; -import org.apache.ignite.internal.processors.cache.CacheObject; -import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.ExchangeActions; import org.apache.ignite.internal.processors.cache.GridCacheContext; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index cb4ad69f5c265..b098905dc48aa 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -238,6 +238,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea assertEquals(0, tombstoneMetric1.get()); } + /** * */ From 8035686072bbe34af3c60e71d3c454def9cab0c5 Mon Sep 17 00:00:00 2001 From: sboikov Date: Tue, 30 Jul 2019 08:06:15 +0300 Subject: [PATCH 15/30] ignite-11704 --- .../cache/distributed/CacheRemoveWithTombstonesTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index b098905dc48aa..daab924c925f0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -49,7 +49,6 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.cacheMetricsRegistryName; From ea425b4306702abf27dd84e1370a4bc273df874a Mon Sep 17 00:00:00 2001 From: sboikov Date: Tue, 30 Jul 2019 09:43:39 +0300 Subject: [PATCH 16/30] ignite-11704 --- .../processors/cache/GridCacheAbstractFullApiSelfTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 7a823d44323cd..ef7463edbe8bf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -6728,7 +6728,10 @@ public CheckEntriesDeletedTask(int cnt) { GridCacheContext ctx = ((IgniteKernal)ignite).internalCache(DEFAULT_CACHE_NAME).context(); - GridCacheEntryEx entry = ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key); + if (ctx.isNear()) + ctx = ctx.near().dht().context(); + + GridCacheEntryEx entry = ctx.cache().peekEx(key); if (ctx.deferredDelete() && ignite.affinity(DEFAULT_CACHE_NAME).mapKeyToPrimaryAndBackups(key).contains(((IgniteKernal)ignite).localNode())) { assertNotNull(entry); From 7d3fcdc3b6b3235f69c39074afa3189776865402 Mon Sep 17 00:00:00 2001 From: sboikov Date: Tue, 30 Jul 2019 10:54:01 +0300 Subject: [PATCH 17/30] ignite-11704 --- .../internal/processors/query/h2/database/H2PkHashIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java index d3bff2997fa8f..db85a7b74deab 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java @@ -209,7 +209,7 @@ public H2PkHashIndex( int part = store.partId(); if (partsFilter == null || partsFilter.applyPartition(part)) - cursors.add(store.cursor(cctx.cacheId())); + cursors.add(store.cursor(cctx.cacheId(), false)); } Cursor pkHashCursor = new H2PkHashIndexCursor(cursors.iterator()); From 901661ad40815aaeaf25470c67e575fade0c9ade Mon Sep 17 00:00:00 2001 From: sboikov Date: Tue, 30 Jul 2019 10:57:47 +0300 Subject: [PATCH 18/30] ignite-11704 --- .../processors/cache/IgniteCacheOffheapManager.java | 11 ++++++----- .../cache/IgniteCacheOffheapManagerImpl.java | 10 +++++----- .../cache/persistence/GridCacheOffheapManager.java | 8 ++++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index aca75b69af329..8b6b6232d14db 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter; import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow; import org.apache.ignite.internal.processors.cache.persistence.RootPage; import org.apache.ignite.internal.processors.cache.persistence.RowStore; @@ -215,7 +216,7 @@ public List> mvccAllVersions(GridCacheContext * @return Iterator over all versions. * @throws IgniteCheckedException If failed. */ - GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, Object x) + GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, CacheDataRowAdapter.RowData x) throws IgniteCheckedException; /** @@ -945,7 +946,7 @@ public void removeWithTombstone( * @return Iterator over all versions. * @throws IgniteCheckedException If failed. */ - GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, Object x) + GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, CacheDataRowAdapter.RowData x) throws IgniteCheckedException; /** @@ -980,7 +981,7 @@ List> mvccFindAllVersions(GridCacheContext cc * @return Data cursor. * @throws IgniteCheckedException If failed. */ - public GridCursor cursor(Object x) throws IgniteCheckedException; + public GridCursor cursor(CacheDataRowAdapter.RowData x) throws IgniteCheckedException; /** * @param mvccSnapshot MVCC snapshot. @@ -1025,7 +1026,7 @@ public GridCursor cursor(int cacheId, KeyCacheObject low * @throws IgniteCheckedException If failed. */ public GridCursor cursor(int cacheId, KeyCacheObject lower, - KeyCacheObject upper, Object x) throws IgniteCheckedException; + KeyCacheObject upper, CacheDataRowAdapter.RowData x) throws IgniteCheckedException; /** * @param cacheId Cache ID. @@ -1040,7 +1041,7 @@ public GridCursor cursor(int cacheId, KeyCacheObject low public GridCursor cursor(int cacheId, KeyCacheObject lower, KeyCacheObject upper, - Object x, + CacheDataRowAdapter.RowData x, MvccSnapshot snapshot, boolean withTombstones) throws IgniteCheckedException; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 8aa5514918d9e..1f3a22d652784 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -687,7 +687,7 @@ private Iterator cacheData(boolean primary, boolean backup, Affi /** {@inheritDoc} */ @Override public GridCursor mvccAllVersionsCursor(GridCacheContext cctx, - KeyCacheObject key, Object x) throws IgniteCheckedException { + KeyCacheObject key, CacheDataRowAdapter.RowData x) throws IgniteCheckedException { CacheDataStore dataStore = dataStore(cctx, key); return dataStore != null ? dataStore.mvccAllVersionsCursor(cctx, key, x) : EMPTY_CURSOR; @@ -2889,7 +2889,7 @@ private void clearPendingEntries(GridCacheContext cctx, CacheDataRow oldRow) } /** {@inheritDoc} */ - @Override public GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, Object x) + @Override public GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, CacheDataRowAdapter.RowData x) throws IgniteCheckedException { int cacheId = cctx.cacheId(); @@ -3014,7 +3014,7 @@ private GridCursor cursorSkipTombstone(final GridCursor< } /** {@inheritDoc} */ - @Override public GridCursor cursor(Object x) throws IgniteCheckedException { + @Override public GridCursor cursor(CacheDataRowAdapter.RowData x) throws IgniteCheckedException { GridCursor cur = dataTree.find(null, null, x); return x == CacheDataRowAdapter.RowData.TOMBSTONES ? cursorSkipEmpty(cur) : cursorSkipTombstone(cur); @@ -3055,7 +3055,7 @@ private GridCursor cursorSkipTombstone(final GridCursor< /** {@inheritDoc} */ @Override public GridCursor cursor(int cacheId, KeyCacheObject lower, - KeyCacheObject upper, Object x) throws IgniteCheckedException { + KeyCacheObject upper, CacheDataRowAdapter.RowData x) throws IgniteCheckedException { return cursor(cacheId, lower, upper, null, null, false); } @@ -3063,7 +3063,7 @@ private GridCursor cursorSkipTombstone(final GridCursor< @Override public GridCursor cursor(int cacheId, KeyCacheObject lower, KeyCacheObject upper, - Object x, + CacheDataRowAdapter.RowData x, MvccSnapshot snapshot, boolean withTombstones) throws IgniteCheckedException { SearchRow lowerRow; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 427c0b9df0cb3..8470e49cccd02 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -2469,7 +2469,7 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { /** {@inheritDoc} */ @Override public GridCursor mvccAllVersionsCursor(GridCacheContext cctx, - KeyCacheObject key, Object x) throws IgniteCheckedException { + KeyCacheObject key, CacheDataRowAdapter.RowData x) throws IgniteCheckedException { CacheDataStore delegate = init0(true); if (delegate != null) @@ -2490,7 +2490,7 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { } /** {@inheritDoc} */ - @Override public GridCursor cursor(Object x) throws IgniteCheckedException { + @Override public GridCursor cursor(CacheDataRowAdapter.RowData x) throws IgniteCheckedException { CacheDataStore delegate = init0(true); if (delegate != null) @@ -2527,7 +2527,7 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { @Override public GridCursor cursor(int cacheId, KeyCacheObject lower, KeyCacheObject upper, - Object x) + CacheDataRowAdapter.RowData x) throws IgniteCheckedException { CacheDataStore delegate = init0(true); @@ -2541,7 +2541,7 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { @Override public GridCursor cursor(int cacheId, KeyCacheObject lower, KeyCacheObject upper, - Object x, + CacheDataRowAdapter.RowData x, MvccSnapshot mvccSnapshot, boolean withTombstones) throws IgniteCheckedException { From a6cd27dbd0ec5a7e9d82d4904f6b30c2cb5d7b6e Mon Sep 17 00:00:00 2001 From: sboikov Date: Wed, 31 Jul 2019 10:58:06 +0300 Subject: [PATCH 19/30] ignite-11704 --- .../CacheRemoveWithTombstonesLoadTest.java | 19 ++++++++++++++----- .../testframework/MvccFeatureChecker.java | 3 +-- .../testsuites/IgniteCacheMvccTestSuite9.java | 4 ++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java index cb144b9eea383..6b0aefdfe1616 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Assume; import org.junit.Test; import java.util.ArrayList; @@ -80,7 +81,7 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { @Override protected void beforeTestsStarted() throws Exception { super.beforeTestsStarted(); - MvccFeatureChecker.skipIfNotSupported(MvccFeatureChecker.Feature.TOMBSTONES); + Assume.assumeFalse(MvccFeatureChecker.forcedMvcc()); } /** {@inheritDoc} */ @@ -138,8 +139,8 @@ private void removeAndRebalance() throws Exception { Map data = new HashMap<>(); - final int KEYS = 10_000; - final int ADD_NODES = 3; + final int KEYS = persistence ? 5_000 : 10_000; + final int ADD_NODES = persistence ? 2 : 3; for (int i = 0; i < KEYS; i++) { TestKey key = new TestKey(i, new byte[rnd.nextInt(pageSize * 3)]); @@ -164,7 +165,7 @@ private void removeAndRebalance() throws Exception { } }); - long endTime = System.currentTimeMillis() + 5000; + long endTime = System.currentTimeMillis() + 2500; while (System.currentTimeMillis() < endTime) { for (int i = 0; i < 100; i++) { @@ -180,6 +181,8 @@ private void removeAndRebalance() throws Exception { cache0.put(key, val); data.put(key, val); } + + Thread.sleep(10); } } @@ -209,7 +212,7 @@ private void removeAndRebalance() throws Exception { } }); - long endTime = System.currentTimeMillis() + 5000; + long endTime = System.currentTimeMillis() + 2500; while (System.currentTimeMillis() < endTime) { for (int i = 0; i < 100; i++) { @@ -226,6 +229,8 @@ private void removeAndRebalance() throws Exception { data.put(key, val); } } + + Thread.sleep(10); } fut.get(30_000); @@ -238,6 +243,10 @@ private void removeAndRebalance() throws Exception { } } + /** + * @param keys Keys to check. + * @param data Expected data. + */ private void checkData(List keys, Map data) { for (Ignite node : Ignition.allGrids()) { if (!node.name().endsWith("CacheRemoveWithTombstonesLoadTest1")) diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java index dce8c8dd9a502..1daeab46130ed 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java @@ -47,8 +47,7 @@ public enum Feature { EVICTION, EXPIRATION, METRICS, - INTERCEPTOR, - TOMBSTONES + INTERCEPTOR } /** diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java index 243fb4b344838..e5dff20575252 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java @@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheGetCustomCollectionsSelfTest; import org.apache.ignite.internal.processors.cache.IgniteCacheLoadRebalanceEvictionSelfTest; import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheRemoveWithTombstonesLoadTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxConcurrentRemoveObjectsTest; @@ -88,6 +89,9 @@ public static List> suite() { ignoredTests.add(IoStatisticsCacheSelfTest.class); ignoredTests.add(IoStatisticsCachePersistenceSelfTest.class); + // Tombstones are not created with mvcc. + ignoredTests.add(CacheRemoveWithTombstonesLoadTest.class); + return new ArrayList<>(IgniteCacheTestSuite9.suite(ignoredTests)); } } From 1da9dc31b51718f595334a32e60e11f571676dd5 Mon Sep 17 00:00:00 2001 From: sboikov Date: Thu, 1 Aug 2019 09:43:33 +0300 Subject: [PATCH 20/30] ignite-11704 --- .../internal/processors/cache/CacheGroupContext.java | 2 +- .../internal/processors/cache/GridCacheContext.java | 8 ++++++-- .../internal/processors/cache/GridCacheMapEntry.java | 3 +++ .../processors/cache/IgniteCacheOffheapManagerImpl.java | 2 ++ .../distributed/dht/topology/GridDhtLocalPartition.java | 2 -- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index 3a7a445de901d..c2cba6b5c686b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -1296,7 +1296,7 @@ public boolean hasAtomicCaches() { * @return {@code True} if need create temporary tombstones entries for removed data. */ public boolean supportsTombstone() { - return !sharedGroup() && !hasAtomicCaches && !mvccEnabled && !isLocal(); + return !mvccEnabled && !isLocal(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 959095c3189fe..4c519811a2cb9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -625,8 +625,12 @@ public byte ioPolicy() { public void cache(GridCacheAdapter cache) { this.cache = cache; - deferredDel = !grp.supportsTombstone() && (cache.isDht() || cache.isDhtAtomic() || cache.isColocated() || - (cache.isNear() && cache.configuration().getAtomicityMode() == ATOMIC)); + if (grp.supportsTombstone() && cache.configuration().getAtomicityMode() == TRANSACTIONAL) + deferredDel = false; + else { + deferredDel = (cache.isDht() || cache.isDhtAtomic() || cache.isColocated() || + (cache.isNear() && cache.configuration().getAtomicityMode() == ATOMIC)); + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 1133085b16d77..db066b6eb8b61 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -4505,6 +4505,9 @@ protected void removeValue() throws IgniteCheckedException { CacheDataRow row = cctx.offheap().read(this); + if (cctx.offheap().isTombstone(row)) + return; + if (row != null && (filter == null || filter.apply(row))) clo.apply(row); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 1f3a22d652784..9d153587a2355 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -628,6 +628,8 @@ private Iterator cacheData(boolean primary, boolean backup, Affi GridCacheVersion ver, GridDhtLocalPartition part) throws IgniteCheckedException { assert part != null; + assert !cctx.isNear(); + assert !cctx.isLocal(); dataStore(part).removeWithTombstone(cctx, key, ver, part); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index d9b34c61254bd..f098e995fee42 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -1174,8 +1174,6 @@ public void clearTombstones(EvictionContext evictionCtx) { cached.removeTombstone(row.version()); - cached.touch(); - break; } catch (GridCacheEntryRemovedException e) { From ee2ddedc3282921d54c45f3fc3fdff5f423002bf Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 25 Sep 2019 18:23:59 +0300 Subject: [PATCH 21/30] IGNITE-11704 Cleanup --- .../processors/cache/CacheGroupContext.java | 2 +- .../processors/cache/GridCacheMapEntry.java | 52 +++++++++++-------- .../cache/IgniteCacheOffheapManager.java | 12 ++++- .../cache/IgniteCacheOffheapManagerImpl.java | 24 +++++---- .../dht/topology/PartitionsEvictManager.java | 12 ++--- .../persistence/GridCacheOffheapManager.java | 9 ++-- .../IgniteCacheDatabaseSharedManager.java | 5 +- .../CollectConflictPartitionKeysTask.java | 2 +- .../verify/VerifyBackupPartitionsTask.java | 2 +- .../verify/VerifyBackupPartitionsTaskV2.java | 2 +- ...dAndDeleteGarbageInPersistenceClosure.java | 2 +- .../cache/IgniteCacheGroupsTest.java | 10 ++-- .../CacheRemoveWithTombstonesLoadTest.java | 26 +++++----- .../CacheRemoveWithTombstonesTest.java | 27 +++++----- .../visor/verify/ValidateIndexesClosure.java | 2 +- 15 files changed, 106 insertions(+), 83 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index d6b4c0e0246b4..990eacfbb2507 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -1306,7 +1306,7 @@ public boolean supportsTombstone() { * @param part Partition. * @return {@code True} if need create tombstone for remove in given partition. */ - public boolean createTombstone(@Nullable GridDhtLocalPartition part) { + public boolean shouldCreateTombstone(@Nullable GridDhtLocalPartition part) { return part != null && supportsTombstone() && part.state() == GridDhtPartitionState.MOVING; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index c9735bb4eac9e..1fed5ddf4b041 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -1713,16 +1713,14 @@ protected Object keyValue(boolean cpy) { interceptRes = cctx.config().getInterceptor().onBeforeRemove(entry0); - if (cctx.cancelRemove(interceptRes)) { + if (cctx.cancelRemove(interceptRes)) return new GridCacheUpdateTxResult(false, logPtr); - } } - if (cctx.group().createTombstone(localPartition())) { + if (cctx.group().shouldCreateTombstone(localPartition())) { cctx.offheap().removeWithTombstone(cctx, key, newVer, localPartition()); - if (!cctx.group().createTombstone(localPartition())) - removeTombstone0(newVer); + assert cctx.group().shouldCreateTombstone(localPartition()) : "Partition state is changed: " + localPartition(); } else removeValue(); @@ -2845,9 +2843,9 @@ public void removeTombstone(GridCacheVersion tombstoneVer) throws GridCacheEntry * @throws IgniteCheckedException If failed. */ private void removeTombstone0(GridCacheVersion tombstoneVer) throws IgniteCheckedException { - RemoveClosure closure = new RemoveClosure(this, tombstoneVer); + RemoveClosure c = new RemoveClosure(this, tombstoneVer); - cctx.offheap().invoke(cctx, key, localPartition(), closure); + cctx.offheap().invoke(cctx, key, localPartition(), c); } /** @@ -3371,17 +3369,17 @@ private boolean skipInterceptor(@Nullable GridCacheVersion explicitVer) { boolean update; IgniteBiPredicate p = new IgniteBiPredicate() { - @Override public boolean apply(@Nullable CacheObject val, GridCacheVersion currentVer) { + @Override public boolean apply(@Nullable CacheObject val, GridCacheVersion currVer) { boolean update0; - boolean isStartVer = cctx.shared().versions().isStartVersion(currentVer); + boolean isStartVer = cctx.shared().versions().isStartVersion(currVer); if (cctx.group().persistenceEnabled()) { if (!isStartVer) { if (cctx.atomic()) - update0 = ATOMIC_VER_COMPARATOR.compare(currentVer, ver) < 0; + update0 = ATOMIC_VER_COMPARATOR.compare(currVer, ver) < 0; else - update0 = currentVer.compareTo(ver) < 0; + update0 = currVer.compareTo(ver) < 0; } else update0 = true; @@ -4287,9 +4285,11 @@ private IgniteTxLocalAdapter currentTx() { * @param ver New entry version. * @throws IgniteCheckedException If update failed. */ - protected boolean storeValue(@Nullable CacheObject val, + protected boolean storeValue( + @Nullable CacheObject val, long expireTime, - GridCacheVersion ver) throws IgniteCheckedException { + GridCacheVersion ver + ) throws IgniteCheckedException { return storeValue(val, expireTime, ver, null, null); } @@ -4304,21 +4304,21 @@ protected boolean storeValue(@Nullable CacheObject val, * @return {@code True} if storage was modified. * @throws IgniteCheckedException If update failed. */ - protected boolean storeValue( + private boolean storeValue( @Nullable CacheObject val, long expireTime, GridCacheVersion ver, - @Nullable IgniteBiPredicate p) throws IgniteCheckedException { + @Nullable IgniteBiPredicate p, @Nullable CacheDataRow row ) throws IgniteCheckedException { assert lock.isHeldByCurrentThread(); assert localPartition() == null || localPartition().state() != RENTING : localPartition(); - UpdateClosure closure = new UpdateClosure(this, val, ver, expireTime, p, row); + UpdateClosure c = new UpdateClosure(this, val, ver, expireTime, p, row); - cctx.offheap().invoke(cctx, key, localPartition(), closure); + cctx.offheap().invoke(cctx, key, localPartition(), c); - return closure.treeOp != IgniteTree.OperationType.NOOP; + return c.treeOp != IgniteTree.OperationType.NOOP; } /** @@ -5862,10 +5862,17 @@ private static class UpdateClosure implements IgniteCacheOffheapManager.OffheapI * @param val New value. * @param ver New version. * @param expireTime New expire time. - * @param predicate Optional predicate. + * @param p Optional predicate. + * @param newRow New row value. */ - UpdateClosure(GridCacheMapEntry entry, @Nullable CacheObject val, GridCacheVersion ver, long expireTime, - @Nullable IgniteBiPredicate p, @Nullable CacheDataRow newRow) { + private UpdateClosure( + GridCacheMapEntry entry, + @Nullable CacheObject val, + GridCacheVersion ver, + long expireTime, + @Nullable IgniteBiPredicate p, + @Nullable CacheDataRow newRow + ) { this.entry = entry; this.val = val; this.ver = ver; @@ -5907,7 +5914,8 @@ private static class UpdateClosure implements IgniteCacheOffheapManager.OffheapI val, ver, expireTime, - oldRow); + oldRow + ); } treeOp = oldRow != null && oldRow.link() == newRow.link() ? diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index b1efd964aa650..4190a853fcf9d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -464,6 +464,15 @@ public GridIterator cachePartitionIterator(int cacheId, final int */ public GridIterator partitionIterator(final int part, boolean withTombstones) throws IgniteCheckedException; + /** + * @param part Partition number. + * @return Iterator for given partition that skips tombstones. + * @throws IgniteCheckedException If failed. + */ + public default GridIterator partitionIterator(final int part) throws IgniteCheckedException { + return partitionIterator(part, false); + } + /** * @param part Partition number. * @param topVer Topology version. @@ -952,7 +961,8 @@ public void removeWithTombstone( GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, - GridDhtLocalPartition part) throws IgniteCheckedException; + GridDhtLocalPartition part + ) throws IgniteCheckedException; /** * @param cctx Cache context. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index c08e8c981af44..febd4c6224757 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -437,8 +437,8 @@ private Iterator cacheData(boolean primary, boolean backup, Affi GridCacheContext cctx, KeyCacheObject key, GridDhtLocalPartition part, - OffheapInvokeClosure c) - throws IgniteCheckedException { + OffheapInvokeClosure c + ) throws IgniteCheckedException { dataStore(part).invoke(cctx, key, c); } @@ -621,7 +621,8 @@ private Iterator cacheData(boolean primary, boolean backup, Affi GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, - GridDhtLocalPartition part) throws IgniteCheckedException { + GridDhtLocalPartition part + ) throws IgniteCheckedException { assert part != null; assert !cctx.isNear(); assert !cctx.isLocal(); @@ -2777,10 +2778,11 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn /** {@inheritDoc} */ @Override public void removeWithTombstone( - GridCacheContext cctx, - KeyCacheObject key, - GridCacheVersion ver, - GridDhtLocalPartition part) throws IgniteCheckedException { + GridCacheContext cctx, + KeyCacheObject key, + GridCacheVersion ver, + GridDhtLocalPartition part + ) throws IgniteCheckedException { if (!busyLock.enterBusy()) throw new NodeStoppingException("Operation has been cancelled (node is stopping)."); @@ -2811,13 +2813,15 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn * @param cctx Cache context. * @param key Key. * @param oldRow Removed row. - * @tombstoneRow Tombstone row (if tombstone was created for remove). + * @param tombstoneRow Tombstone row (if tombstone was created for remove). * @throws IgniteCheckedException If failed. */ - private void finishRemove(GridCacheContext cctx, + private void finishRemove( + GridCacheContext cctx, KeyCacheObject key, @Nullable CacheDataRow oldRow, - @Nullable CacheDataRow tombstoneRow) throws IgniteCheckedException { + @Nullable CacheDataRow tombstoneRow + ) throws IgniteCheckedException { boolean oldTombstone = isTombstone(oldRow); boolean oldNull = oldRow == null || oldTombstone; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java index 50a560245d4aa..e273da0ca3497 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java @@ -241,7 +241,7 @@ private void scheduleNextTask(int bucket) { }); // Submit task to executor. - cctx.kernalContext() + cctx.kernalContext() .closure() .runLocalSafe(evictionTask, EVICT_POOL_PLC); } @@ -306,7 +306,7 @@ private void showProgress() { /** * */ - private class TasksStatistics { + private static class TasksStatistics { /** */ private int total; @@ -356,7 +356,7 @@ private class GroupEvictionContext implements EvictionContext { private AtomicInteger totalTasks = new AtomicInteger(); /** */ - private Map stats = U.newHashMap(2); + private Map stats = U.newHashMap(TaskType.VALS.length); /** * @param grp Group context. @@ -392,7 +392,7 @@ private synchronized void taskScheduled(AbstractEvictionTask task) { if (shouldStop()) return; - stats.get(task.id.type).taskStarted(); + stats.get(task.id.type).taskStarted(); GridFutureAdapter fut = task.finishFut; @@ -652,8 +652,8 @@ private class ClearTombstonesTask extends AbstractEvictionTask { * @param grpEvictionCtx Eviction context. */ private ClearTombstonesTask( - GridDhtLocalPartition part, - GroupEvictionContext grpEvictionCtx + GridDhtLocalPartition part, + GroupEvictionContext grpEvictionCtx ) { super(part, grpEvictionCtx, TaskType.CLEAR_TOMBSTONES); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index 83d5796012031..ff76a3ba71b3b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -2440,10 +2440,11 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException { /** {@inheritDoc} */ @Override public void removeWithTombstone( - GridCacheContext cctx, - KeyCacheObject key, - GridCacheVersion ver, - GridDhtLocalPartition part) throws IgniteCheckedException { + GridCacheContext cctx, + KeyCacheObject key, + GridCacheVersion ver, + GridDhtLocalPartition part + ) throws IgniteCheckedException { assert ctx.database().checkpointLockIsHeldByThread(); CacheDataStore delegate = init0(false); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index 8773ae2ae5022..c4efadcc6f491 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache.persistence; -import javax.management.InstanceNotFoundException; import java.io.File; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -29,6 +28,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import javax.management.InstanceNotFoundException; import org.apache.ignite.DataRegionMetrics; import org.apache.ignite.DataRegionMetricsProvider; import org.apache.ignite.DataStorageMetrics; @@ -192,8 +192,7 @@ public boolean isTombstone(@Nullable CacheDataRow row) throws IgniteCheckedExcep if (val.cacheObjectType() == CacheObject.TYPE_REGULAR) { byte[] bytes = val.valueBytes(null); - if (Arrays.equals(tombstoneBytes, bytes)) - return true; + return Arrays.equals(tombstoneBytes, bytes); } return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java index bd9db7bde08d2..8e8128cc9aaaa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java @@ -178,7 +178,7 @@ private CollectPartitionEntryHashesJob(PartitionKey partKey) { partSize = part.dataStore().fullSize(); - GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); + GridIterator it = grpCtx.offheap().partitionIterator(part.id()); partEntryHashRecords = new ArrayList<>(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java index 8b18834fb9ed2..f70056792f266 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java @@ -317,7 +317,7 @@ private Map calculatePartitionHash( partSize = part.dataStore().fullSize(); - GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); + GridIterator it = grpCtx.offheap().partitionIterator(part.id()); while (it.hasNextX()) { CacheDataRow row = it.nextX(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java index fb23c12f5b1ea..187dc8ce2549c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java @@ -595,7 +595,7 @@ private Map calculatePartitionHash( if (arg.checkCrc()) checkPartitionCrc(grpCtx, part, cpFlag); - GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); + GridIterator it = grpCtx.offheap().partitionIterator(part.id()); while (it.hasNextX()) { CacheDataRow row = it.nextX(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorFindAndDeleteGarbageInPersistenceClosure.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorFindAndDeleteGarbageInPersistenceClosure.java index 552a1a7c679ea..722686e578011 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorFindAndDeleteGarbageInPersistenceClosure.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorFindAndDeleteGarbageInPersistenceClosure.java @@ -266,7 +266,7 @@ private Map> processPartition( if (part.state() != GridDhtPartitionState.OWNING) return Collections.emptyMap(); - GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); + GridIterator it = grpCtx.offheap().partitionIterator(part.id()); while (it.hasNextX()) { CacheDataRow row = it.nextX(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java index 35be0893e830d..1e7f580dd49e8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java @@ -1667,17 +1667,17 @@ public void testRebalance2() throws Exception { for (int p = 0; p < aff.partitions(); p++) { if (srv1Parts.contains(p)) { - GridIterator it = grpSrv0.offheap().partitionIterator(p, false); + GridIterator it = grpSrv0.offheap().partitionIterator(p); assertFalse(it.hasNext()); - it = grpSrv1.offheap().partitionIterator(p, false); + it = grpSrv1.offheap().partitionIterator(p); assertTrue(it.hasNext()); } else { - GridIterator it = grpSrv0.offheap().partitionIterator(p, false); + GridIterator it = grpSrv0.offheap().partitionIterator(p); assertTrue(it.hasNext()); - it = grpSrv1.offheap().partitionIterator(p, false); + it = grpSrv1.offheap().partitionIterator(p); assertFalse(it.hasNext()); } } @@ -3920,7 +3920,7 @@ public void testCacheIdSort() throws Exception { Integer cacheId = null; - GridIterator it = grp.offheap().partitionIterator(0, false); + GridIterator it = grp.offheap().partitionIterator(0); int c = 0; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java index 6b0aefdfe1616..f2af27b544d50 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java @@ -17,6 +17,15 @@ package org.apache.ignite.internal.processors.cache.distributed; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.Ignition; @@ -36,16 +45,6 @@ import org.junit.Assume; import org.junit.Test; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.Callable; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicInteger; - import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; @@ -74,6 +73,9 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { cfg.setDataStorageConfiguration(dsCfg); + // Long rebalance. + cfg.setRebalanceThrottle(100); + return cfg; } @@ -280,11 +282,11 @@ private void waitTombstoneCleanup() throws Exception { GridTestUtils.waitForCondition(new GridAbsPredicate() { @Override public boolean apply() { - return tombstones.get() == 0; + return tombstones.value() == 0; } }, 30_000); - assertEquals("Failed to wait for tombstone cleanup: " + node.name(), 0, tombstones.get()); + assertEquals("Failed to wait for tombstone cleanup: " + node.name(), 0, tombstones.value()); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index daab924c925f0..05b4158f7825e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -17,6 +17,11 @@ package org.apache.ignite.internal.processors.cache.distributed; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; @@ -41,12 +46,6 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Callable; - import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -190,12 +189,12 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); // On first node there should not be tombstones. - assertEquals(0, tombstoneMetric0.get()); + assertEquals(0, tombstoneMetric0.value()); if (expTombstone) - assertEquals(removed.size(), tombstoneMetric1.get()); + assertEquals(removed.size(), tombstoneMetric1.value()); else - assertEquals(0, tombstoneMetric1.get()); + assertEquals(0, tombstoneMetric1.value()); // Update some of removed keys, this should remove tombstones. for (int i = 0; i < KEYS; i++) { @@ -208,12 +207,12 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea assert !removed.isEmpty(); - assertEquals(0, tombstoneMetric0.get()); + assertEquals(0, tombstoneMetric0.value()); if (expTombstone) - assertEquals(removed.size(), tombstoneMetric1.get()); + assertEquals(removed.size(), tombstoneMetric1.value()); else - assertEquals(0, tombstoneMetric1.get()); + assertEquals(0, tombstoneMetric1.value()); TestRecordingCommunicationSpi.spi(ignite0).stopBlock(); @@ -231,11 +230,11 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea // Tombstones should be removed after once rebalance is completed. GridTestUtils.waitForCondition(new GridAbsPredicate() { @Override public boolean apply() { - return tombstoneMetric1.get() == 0; + return tombstoneMetric1.value() == 0; } }, 30_000); - assertEquals(0, tombstoneMetric1.get()); + assertEquals(0, tombstoneMetric1.value()); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java index fd4fbc6176de9..c7fa349407f28 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java @@ -446,7 +446,7 @@ private Map processPartition( long partSize = part.dataStore().fullSize(); - GridIterator it = grpCtx.offheap().partitionIterator(part.id(), false); + GridIterator it = grpCtx.offheap().partitionIterator(part.id()); Object consId = ignite.context().discovery().localNode().consistentId(); From cf4486edd61f94f51686f49e3285a48659a2502f Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Fri, 27 Sep 2019 13:49:10 +0300 Subject: [PATCH 22/30] IGNITE-11704 Cleanup --- .../distributed/dht/topology/GridDhtLocalPartition.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 72e7a11263838..936e18565dfbd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -180,10 +180,10 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements * @param recovery Flag indicates that partition is created during recovery phase. */ public GridDhtLocalPartition( - GridCacheSharedContext ctx, - CacheGroupContext grp, - int id, - boolean recovery + GridCacheSharedContext ctx, + CacheGroupContext grp, + int id, + boolean recovery ) { super(ENTRY_FACTORY); From 0f4585041b2faf4246123c8abcbb0b92291e4510 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 2 Oct 2019 19:22:16 +0300 Subject: [PATCH 23/30] IGNITE-11704 Failover for tombstones. --- .../pagemem/wal/record/WALRecord.java | 7 +- .../MetaPageUpdatePartitionDataRecord.java | 19 +- .../MetaPageUpdatePartitionDataRecordV2.java | 22 +- .../MetaPageUpdatePartitionDataRecordV3.java | 108 ++++++ .../cache/CacheGroupMetricsImpl.java | 23 +- .../processors/cache/CacheMetricsImpl.java | 21 -- .../processors/cache/GridCacheContext.java | 20 -- .../cache/IgniteCacheOffheapManager.java | 5 + .../cache/IgniteCacheOffheapManagerImpl.java | 53 ++- .../cache/PartitionUpdateCounter.java | 4 +- .../dht/topology/GridDhtLocalPartition.java | 34 +- .../GridDhtPartitionTopologyImpl.java | 5 +- .../dht/topology/PartitionsEvictManager.java | 11 +- .../persistence/GridCacheOffheapManager.java | 328 +++++++++++------- .../tree/io/PagePartitionMetaIO.java | 22 +- .../tree/io/PagePartitionMetaIOV2.java | 25 +- .../serializer/RecordDataV1Serializer.java | 11 + .../processors/metric/impl/MetricUtils.java | 9 + .../CacheRemoveWithTombstonesLoadTest.java | 200 +++++++---- .../CacheRemoveWithTombstonesTest.java | 191 +++++----- ...CacheRemoveWithTombstonesFailoverTest.java | 187 ++++++++++ .../DropCacheContextDuringEvictionTest.java | 24 +- .../PartitionsEvictManagerAbstractTest.java | 113 +++--- ...titionsEvictionTaskFailureHandlerTest.java | 72 +++- .../testsuites/IgniteCacheTestSuite9.java | 2 + 25 files changed, 1030 insertions(+), 486 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV3.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/CacheRemoveWithTombstonesFailoverTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java index 269b28434816a..405e6305ba8ab 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java @@ -220,8 +220,11 @@ public enum RecordType { /** Rollback tx record. */ ROLLBACK_TX_RECORD (57, LOGICAL), - /** */ - PARTITION_META_PAGE_UPDATE_COUNTERS_V2 (58, PHYSICAL); + /** Partition meta page containing update counter gaps. */ + PARTITION_META_PAGE_UPDATE_COUNTERS_V2 (58, PHYSICAL), + + /** Partition meta page containing tombstone presence flag. */ + PARTITION_META_PAGE_UPDATE_COUNTERS_V3 (59, PHYSICAL); /** Index for serialization. Should be consistent throughout all versions. */ private final int idx; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java index 3e2b67bd3c538..0c9f7fcd76e0c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java @@ -46,7 +46,7 @@ public class MetaPageUpdatePartitionDataRecord extends PageDeltaRecord { private int allocatedIdxCandidate; /** */ - private long cntrsPageId; + private long cacheSizesPageId; /** * @param grpId Cache group ID. @@ -59,9 +59,10 @@ public MetaPageUpdatePartitionDataRecord( long updateCntr, long globalRmvId, int partSize, - long cntrsPageId, + long cacheSizesPageId, byte state, - int allocatedIdxCandidate) { + int allocatedIdxCandidate + ) { super(grpId, pageId); this.updateCntr = updateCntr; @@ -69,7 +70,7 @@ public MetaPageUpdatePartitionDataRecord( this.partSize = partSize; this.state = state; this.allocatedIdxCandidate = allocatedIdxCandidate; - this.cntrsPageId = cntrsPageId; + this.cacheSizesPageId = cacheSizesPageId; } /** @@ -81,7 +82,7 @@ public MetaPageUpdatePartitionDataRecord(DataInput in) throws IOException{ this.updateCntr = in.readLong(); this.globalRmvId = in.readLong(); this.partSize = in.readInt(); - this.cntrsPageId = in.readLong(); + this.cacheSizesPageId = in.readLong(); this.state = in.readByte(); this.allocatedIdxCandidate = in.readInt(); } @@ -110,8 +111,8 @@ public int partitionSize() { /** * @return Partition size. */ - public long countersPageId() { - return cntrsPageId; + public long cacheSizesPageId() { + return cacheSizesPageId; } /** @@ -128,7 +129,7 @@ public byte state() { io.setUpdateCounter(pageAddr, updateCntr); io.setGlobalRemoveId(pageAddr, globalRmvId); io.setSize(pageAddr, partSize); - io.setCountersPageId(pageAddr, cntrsPageId); + io.setSizesPageId(pageAddr, cacheSizesPageId); io.setPartitionState(pageAddr, state); io.setCandidatePageCount(pageAddr, allocatedIdxCandidate); } @@ -150,7 +151,7 @@ public void toBytes(ByteBuffer buf) { buf.putLong(updateCounter()); buf.putLong(globalRemoveId()); buf.putInt(partitionSize()); - buf.putLong(countersPageId()); + buf.putLong(cacheSizesPageId()); buf.put(state()); buf.putInt(allocatedIndexCandidate()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV2.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV2.java index ab3ccf8f35a37..a8a859764531e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV2.java @@ -28,11 +28,12 @@ import org.apache.ignite.internal.util.typedef.internal.S; /** - * + * Partition meta page delta record. + * Contains reference to update counters gaps. */ public class MetaPageUpdatePartitionDataRecordV2 extends MetaPageUpdatePartitionDataRecord { /** */ - private long link; + private long gapsLink; /** * @param grpId Group id. @@ -43,7 +44,7 @@ public class MetaPageUpdatePartitionDataRecordV2 extends MetaPageUpdatePartition * @param cntrsPageId Cntrs page id. * @param state State. * @param allocatedIdxCandidate Allocated index candidate. - * @param link Link. + * @param gapsLink Link. */ public MetaPageUpdatePartitionDataRecordV2( int grpId, @@ -54,9 +55,10 @@ public MetaPageUpdatePartitionDataRecordV2( long cntrsPageId, byte state, int allocatedIdxCandidate, - long link) { + long gapsLink + ) { super(grpId, pageId, updateCntr, globalRmvId, partSize, cntrsPageId, state, allocatedIdxCandidate); - this.link = link; + this.gapsLink = gapsLink; } /** @@ -65,7 +67,7 @@ public MetaPageUpdatePartitionDataRecordV2( public MetaPageUpdatePartitionDataRecordV2(DataInput in) throws IOException { super(in); - this.link = in.readLong(); + this.gapsLink = in.readLong(); } /** {@inheritDoc} */ @@ -74,21 +76,21 @@ public MetaPageUpdatePartitionDataRecordV2(DataInput in) throws IOException { PagePartitionMetaIOV2 io = (PagePartitionMetaIOV2)PagePartitionMetaIO.VERSIONS.forPage(pageAddr); - io.setGapsLink(pageAddr, link); + io.setGapsLink(pageAddr, gapsLink); } /** * */ - public long link() { - return link; + public long gapsLink() { + return gapsLink; } /** {@inheritDoc} */ @Override public void toBytes(ByteBuffer buf) { super.toBytes(buf); - buf.putLong(link()); + buf.putLong(gapsLink()); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV3.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV3.java new file mode 100644 index 0000000000000..1263c43db32fd --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecordV3.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.pagemem.wal.record.delta; + +import java.io.DataInput; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.PageIdUtils; +import org.apache.ignite.internal.pagemem.PageMemory; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIOV2; +import org.apache.ignite.internal.util.typedef.internal.S; + +/** + * Partition meta page delta record. + * Contains information about tombstones count. + */ +public class MetaPageUpdatePartitionDataRecordV3 extends MetaPageUpdatePartitionDataRecordV2 { + /** Tombstones count. */ + private long tombstonesCnt; + + /** + * @param grpId Group id. + * @param pageId Page id. + * @param updateCntr Update counter. + * @param globalRmvId Global remove id. + * @param partSize Partition size. + * @param cacheSizesPageId Cache sizes page id. + * @param state State. + * @param allocatedIdxCandidate Allocated index candidate. + * @param gapsLink Gaps link. + * @param tombstonesCnt Tombstones count. + */ + public MetaPageUpdatePartitionDataRecordV3( + int grpId, + long pageId, + long updateCntr, + long globalRmvId, + int partSize, + long cacheSizesPageId, + byte state, + int allocatedIdxCandidate, + long gapsLink, + long tombstonesCnt + ) { + super(grpId, pageId, updateCntr, globalRmvId, partSize, cacheSizesPageId, state, allocatedIdxCandidate, gapsLink); + this.tombstonesCnt = tombstonesCnt; + } + + /** + * @param in In. + */ + public MetaPageUpdatePartitionDataRecordV3(DataInput in) throws IOException { + super(in); + + this.tombstonesCnt = in.readLong(); + } + + /** + * @return Tombstones count. + */ + public long tombstonesCount() { + return tombstonesCnt; + } + + /** {@inheritDoc} */ + @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException { + super.applyDelta(pageMem, pageAddr); + + PagePartitionMetaIOV2 io = (PagePartitionMetaIOV2) PagePartitionMetaIO.VERSIONS.forPage(pageAddr); + + io.setTombstonesCount(pageAddr, tombstonesCnt); + } + + /** {@inheritDoc} */ + @Override public void toBytes(ByteBuffer buf) { + super.toBytes(buf); + + buf.putLong(tombstonesCnt); + } + + /** {@inheritDoc} */ + @Override public RecordType type() { + return RecordType.PARTITION_META_PAGE_UPDATE_COUNTERS_V3; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(MetaPageUpdatePartitionDataRecordV2.class, this, "partId", PageIdUtils.partId(pageId()), + "super", super.toString()); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsImpl.java index e82e451ab4aa0..fab2e1fd96181 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsImpl.java @@ -70,6 +70,7 @@ public class CacheGroupMetricsImpl { private final LongMetric sparseStorageSize; /** Interface describing a predicate of two integers. */ + @FunctionalInterface private interface IntBiPredicate { /** * Predicate body. @@ -169,6 +170,10 @@ public void onTopologyInitialized() { mreg.register("TotalAllocatedSize", this::getTotalAllocatedSize, "Total size of memory allocated for group, in bytes."); + + mreg.register("Tombstones", + this::getTombstones, + "Number of tombstone entries."); } /** */ @@ -253,20 +258,12 @@ private int numberOfPartitionCopies(IntBiPredicate pred) { /** */ public int getMinimumNumberOfPartitionCopies() { - return numberOfPartitionCopies(new IntBiPredicate() { - @Override public boolean apply(int targetVal, int nextVal) { - return nextVal < targetVal; - } - }); + return numberOfPartitionCopies((targetVal, nextVal) -> nextVal < targetVal); } /** */ public int getMaximumNumberOfPartitionCopies() { - return numberOfPartitionCopies(new IntBiPredicate() { - @Override public boolean apply(int targetVal, int nextVal) { - return nextVal > targetVal; - } - }); + return numberOfPartitionCopies((targetVal, nextVal) -> nextVal > targetVal); } /** @@ -462,6 +459,12 @@ public long getSparseStorageSize() { return sparseStorageSize == null ? 0 : sparseStorageSize.value(); } + /** */ + public long getTombstones() { + return ctx.topology().localPartitions().stream() + .map(part -> part.dataStore().tombstonesCount()).reduce(Long::sum).orElse(0L); + } + /** Removes all metric for cache group. */ public void remove() { ctx.shared().kernalContext().metric().remove(metricGroupName()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java index 81db75717b5ef..eefcb19b69c75 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java @@ -27,8 +27,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore; import org.apache.ignite.internal.processors.metric.MetricRegistry; -import org.apache.ignite.internal.processors.metric.impl.HitRateMetric; -import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric; import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric; import org.apache.ignite.internal.processors.metric.impl.HitRateMetric; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; @@ -164,9 +162,6 @@ public class CacheMetricsImpl implements CacheMetrics { /** Number of currently clearing partitions for rebalancing. */ private final AtomicLongMetric rebalanceClearingPartitions; - /** */ - private LongAdderMetric tombstones; - /** Cache metrics. */ @GridToStringExclude private transient CacheMetricsImpl delegate; @@ -316,8 +311,6 @@ public CacheMetricsImpl(GridCacheContext cctx, boolean isNear) { rebalanceClearingPartitions = mreg.longMetric("RebalanceClearingPartitionsLeft", "Number of partitions need to be cleared before actual rebalance start."); - - tombstones = mreg.longAdderMetric("Tombstones", "Number of tombstone entries"); } /** @@ -1027,20 +1020,6 @@ public void addPutAndGetTimeNanos(long duration) { delegate.addPutAndGetTimeNanos(duration); } - /** - * Increments tombstones counter. - */ - public void tombstoneCreated() { - tombstones.increment(); - } - - /** - * Decrements tombstones counter. - */ - public void tombstoneRemoved() { - tombstones.decrement(); - } - /** {@inheritDoc} */ @Override public String getKeyType() { CacheConfiguration ccfg = cctx.config(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 564f7a8a78b5c..431118cf36e8a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -2384,26 +2384,6 @@ protected Object readResolve() throws ObjectStreamException { } } - /** - * Increments tombstones counter. - */ - public void tombstoneCreated() { - GridCacheAdapter cache = this.cache; - - if (cache != null) - cache.metrics0().tombstoneCreated(); - } - - /** - * Decrements tombstones counter. - */ - public void tombstoneRemoved() { - GridCacheAdapter cache = this.cache; - - if (cache != null) - cache.metrics0().tombstoneRemoved(); - } - /** {@inheritDoc} */ @Override public String toString() { return "GridCacheContext: " + name(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index 4190a853fcf9d..c5da2b4271d87 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -1142,5 +1142,10 @@ public GridCursor cursor(int cacheId, * Partition storage. */ public PartitionMetaStorage partStorage(); + + /** + * @return Number of tombstone entries. + */ + public long tombstonesCount(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index febd4c6224757..9c9ad577f3058 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -1472,6 +1472,9 @@ protected class CacheDataStoreImpl implements CacheDataStore { /** */ private final PageHandler mvccApplyChanges = new MvccApplyChangesHandler(); + /** Tombstones counter. */ + private final AtomicLong tombstonesCnt = new AtomicLong(); + /** * @param partId Partition number. * @param rowStore Row store. @@ -1732,7 +1735,7 @@ private void invoke0(GridCacheContext cctx, CacheSearchRow row, OffheapInvokeClo assert !isTombstone(c.newRow()); if (isTombstone(c.oldRow())) { - cctx.tombstoneRemoved(); + tombstoneRemoved(); incrementSize(cctx.cacheId()); } @@ -2667,7 +2670,7 @@ private void finishUpdate(GridCacheContext cctx, CacheDataRow newRow, @Nullable updateIgfsMetrics(cctx, key, (oldNull ? null : oldRow.value()), newRow.value()); if (oldTombstone) - cctx.tombstoneRemoved(); + tombstoneRemoved(); } /** @@ -2797,10 +2800,8 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn assert c.operationType() == PUT || c.operationType() == IN_PLACE : c.operationType(); - part.tombstoneCreated(); - if (!isTombstone(c.oldRow)) - cctx.tombstoneCreated(); + tombstoneCreated(); finishRemove(cctx, key, c.oldRow, c.newRow); } @@ -2842,7 +2843,7 @@ private void finishRemove( updateIgfsMetrics(cctx, key, (oldNull ? null : oldRow.value()), null); if (oldTombstone && tombstoneRow == null) - cctx.tombstoneRemoved(); + tombstoneRemoved(); } /** @@ -3233,17 +3234,24 @@ private GridCursor cursorSkipTombstone(final GridCursor< * @param size Size to init. * @param updCntr Update counter. * @param cacheSizes Cache sizes if store belongs to group containing multiple caches. - * @param cntrUpdData Counter updates. + * @param updCntrGapsData Update counters gaps raw data. + * @param tombstonesCnt Tombstones count. */ - public void restoreState(long size, long updCntr, @Nullable Map cacheSizes, byte[] cntrUpdData) { - pCntr.init(updCntr, cntrUpdData); + public void restoreState( + long size, + long updCntr, + Map cacheSizes, + byte[] updCntrGapsData, + long tombstonesCnt + ) { + pCntr.init(updCntr, updCntrGapsData); storageSize.set(size); - if (cacheSizes != null) { - for (Map.Entry e : cacheSizes.entrySet()) - this.cacheSizes.put(e.getKey(), new AtomicLong(e.getValue())); - } + for (Map.Entry e : cacheSizes.entrySet()) + this.cacheSizes.put(e.getKey(), new AtomicLong(e.getValue())); + + this.tombstonesCnt.set(tombstonesCnt); } /** {@inheritDoc} */ @@ -3266,6 +3274,25 @@ public void restoreState(long size, long updCntr, @Nullable Map c return null; } + /** {@inheritDoc} */ + @Override public long tombstonesCount() { + return tombstonesCnt.get(); + } + + /** + * Called when tombstone has removed from partition. + */ + private void tombstoneRemoved() { + tombstonesCnt.decrementAndGet(); + } + + /** + * Called when tombstone has created in partition. + */ + private void tombstoneCreated() { + tombstonesCnt.incrementAndGet(); + } + /** * @param cctx Cache context. * @param key Key. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java index 112a11051fd86..1d41d7f05e297 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/PartitionUpdateCounter.java @@ -35,9 +35,9 @@ public interface PartitionUpdateCounter extends Iterable { * Restores update counter state. * * @param initUpdCntr LWM. - * @param cntrUpdData Counter updates raw data. + * @param updCntrGapsData Updates counters gaps raw data. */ - public void init(long initUpdCntr, @Nullable byte[] cntrUpdData); + public void init(long initUpdCntr, @Nullable byte[] updCntrGapsData); /** * @deprecated TODO LWM should be used as initial counter https://ggsystems.atlassian.net/browse/GG-17396 diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 936e18565dfbd..37aa0f37a6458 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -170,9 +170,6 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements /** Set if topology update sequence should be updated on partition destroy. */ private boolean updateSeqOnDestroy; - /** Set if tombstone was created in partition. */ - private volatile boolean tombstoneCreated; - /** * @param ctx Context. * @param grp Cache group. @@ -609,8 +606,8 @@ public boolean own() { assert partState == MOVING || partState == LOST; if (casState(state, OWNING)) { - if (grp.supportsTombstone() && tombstoneCreated) - grp.shared().evict().clearTombstonesAsync(grp, this); + if (hasTombstones()) + clearTombstonesAsync(); return true; } @@ -766,6 +763,21 @@ public void clearAsync() { clearAsync0(false); } + /** + * @return {@code True} if partition has tombstone entries. + */ + boolean hasTombstones() { + return grp.supportsTombstone() && dataStore().tombstonesCount() > 0; + } + + /** + * Adds async task that will clear tombstone entries from partition. + * @see #clearTombstones(EvictionContext). + */ + void clearTombstonesAsync() { + grp.shared().evict().clearTombstonesAsync(grp, this); + } + /** * Continues delayed clearing of partition if possible. * Clearing may be delayed because of existing reservations. @@ -1130,16 +1142,14 @@ public long fullSize() { } /** + * Iterates over partition entries and removes tombstone entries. * + * @param evictionCtx Eviction context. */ - public void tombstoneCreated() { - tombstoneCreated = true; - } + void clearTombstones(EvictionContext evictionCtx) { + if (evictionCtx.shouldStop()) + return; - /** - * - */ - public void clearTombstones(EvictionContext evictionCtx) { final int stopCheckingFreq = 1000; CacheMapHolder hld = grp.sharedGroup() ? null : singleCacheEntryMap; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java index e5d5651064e8d..f5681bd75b455 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java @@ -715,9 +715,10 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) { updateLocal(p, state, updateSeq, topVer); - // Restart cleaning. - if (state == RENTING) + if (state == RENTING) // Restart cleaning. locPart.clearAsync(); + else if (state == OWNING && locPart.hasTombstones()) + locPart.clearTombstonesAsync(); // Restart tombstones cleaning. } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java index 921281bc778cc..2d3b813aec9b3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java @@ -603,7 +603,7 @@ private AbstractEvictionTask( true); } else { - LT.error(log, ex, "Partition eviction failed, this can cause grid hang."); + LT.error(log, ex, "Partition eviction failed."); cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, ex)); } @@ -614,7 +614,7 @@ private AbstractEvictionTask( /** * Task for self-scheduled partition eviction / clearing. */ - private class PartitionEvictionTask extends AbstractEvictionTask { + class PartitionEvictionTask extends AbstractEvictionTask { /** * @param part Partition. * @param grpEvictionCtx Eviction context. @@ -651,7 +651,7 @@ private PartitionEvictionTask( /** * */ - private class ClearTombstonesTask extends AbstractEvictionTask { + class ClearTombstonesTask extends AbstractEvictionTask { /** * @param part Partition. * @param grpEvictionCtx Eviction context. @@ -681,14 +681,11 @@ private ClearTombstonesTask( */ class BucketQueue { /** Queues contains partitions scheduled for eviction. */ - private final Queue[] buckets; + final Queue[] buckets; /** */ private final long[] bucketSizes; - /** Queues contains partitions scheduled for eviction. */ - final Queue[] buckets; - /** * @param buckets Number of buckets. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java index ff76a3ba71b3b..4f3e0cce35577 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java @@ -52,7 +52,7 @@ import org.apache.ignite.internal.pagemem.wal.record.RollbackRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord; -import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV2; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV3; import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheDiagnosticManager; @@ -296,14 +296,10 @@ private void saveStoreMetadata( if (rowStore0 != null) { ((CacheFreeList)rowStore0.freeList()).saveMetadata(); - PartitionMetaStorage partStore = store.partStorage(); - long updCntr = store.updateCounter(); long size = store.fullSize(); long rmvId = globalRemoveId().get(); - byte[] updCntrsBytes = store.partUpdateCounter().getBytes(); - PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory(); IgniteWriteAheadLogManager wal = this.ctx.wal(); @@ -328,6 +324,9 @@ private void saveStoreMetadata( return; } + assert state != null || grp.isLocal() : "Partition state is undefined " + + "[grp=" + grp.cacheOrGroupName() + ", part=" + part + "]"; + int grpId = grp.groupId(); long partMetaId = pageMem.partitionMetaPageId(grpId, store.partId()); @@ -348,126 +347,29 @@ private void saveStoreMetadata( try { PagePartitionMetaIOV2 io = PageIO.getPageIO(partMetaPageAddr); - long link = io.getGapsLink(partMetaPageAddr); - - if (updCntrsBytes == null && link != 0) { - partStore.removeDataRowByLink(link, grp.statisticsHolderData()); - - io.setGapsLink(partMetaPageAddr, (link = 0)); - - changed = true; - } - else if (updCntrsBytes != null && link == 0) { - SimpleDataRow row = new SimpleDataRow(store.partId(), updCntrsBytes); - - partStore.insertDataRow(row, grp.statisticsHolderData()); - - io.setGapsLink(partMetaPageAddr, (link = row.link())); - - changed = true; - } - else if (updCntrsBytes != null && link != 0) { - byte[] prev = partStore.readRow(link); - - assert prev != null : "Read null gaps using link=" + link; - - if (!Arrays.equals(prev, updCntrsBytes)) { - partStore.removeDataRowByLink(link, grp.statisticsHolderData()); - - SimpleDataRow row = new SimpleDataRow(store.partId(), updCntrsBytes); - - partStore.insertDataRow(row, grp.statisticsHolderData()); - - io.setGapsLink(partMetaPageAddr, (link = row.link())); - - changed = true; - } - } - - if (changed) - partStore.saveMetadata(); - + changed |= io.setPartitionState(partMetaPageAddr, state != null ? (byte)state.ordinal() : -1); changed |= io.setUpdateCounter(partMetaPageAddr, updCntr); changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId); changed |= io.setSize(partMetaPageAddr, size); + changed |= io.setTombstonesCount(partMetaPageAddr, store.tombstonesCount()); + changed |= savePartitionUpdateCounterGaps(store, io, partMetaPageAddr); + changed |= saveCacheSizes(store, io, partMetaPageAddr); - if (state != null) - changed |= io.setPartitionState(partMetaPageAddr, (byte)state.ordinal()); - else - assert grp.isLocal() : grp.cacheOrGroupName(); - - long cntrsPageId; - - if (grp.sharedGroup()) { - long initCntrPageId = io.getCountersPageId(partMetaPageAddr); - - Map newSizes = store.cacheSizes(); - Map prevSizes = readSharedGroupCacheSizes(pageMem, grpId, initCntrPageId); - - if (prevSizes != null && prevSizes.equals(newSizes)) - cntrsPageId = initCntrPageId; // Preventing modification of sizes pages for store - else { - cntrsPageId = writeSharedGroupCacheSizes(pageMem, grpId, initCntrPageId, - store.partId(), newSizes); - - if (initCntrPageId == 0 && cntrsPageId != 0) { - io.setCountersPageId(partMetaPageAddr, cntrsPageId); - - changed = true; - } - } - } - else - cntrsPageId = 0L; - - int pageCnt; - - if (needSnapshot) { - pageCnt = this.ctx.pageStore().pages(grpId, store.partId()); - - io.setCandidatePageCount(partMetaPageAddr, size == 0 ? 0 : pageCnt); - - if (state == OWNING) { - assert part != null; - - if (!addPartition( - part, - ctx.partitionStatMap(), - partMetaPageAddr, - io, - grpId, - store.partId(), - this.ctx.pageStore().pages(grpId, store.partId()), - store.fullSize() - )) - U.warn(log, "Partition was concurrently evicted grpId=" + grpId + - ", partitionId=" + part.id()); - } - else if (state == MOVING || state == RENTING) { - if (ctx.partitionStatMap().forceSkipIndexPartition(grpId)) { - if (log.isInfoEnabled()) - log.info("Will not include SQL indexes to snapshot because there is " + - "a partition not in " + OWNING + " state [grp=" + grp.cacheOrGroupName() + - ", partId=" + store.partId() + ", state=" + state + ']'); - } - } - - changed = true; - } - else - pageCnt = io.getCandidatePageCount(partMetaPageAddr); + if (needSnapshot) + changed |= savePagesCount(ctx, part, store, io, partMetaPageAddr); if (changed && PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null)) - wal.log(new MetaPageUpdatePartitionDataRecordV2( + wal.log(new MetaPageUpdatePartitionDataRecordV3( grpId, partMetaId, updCntr, rmvId, (int)size, // TODO: Partition size may be long - cntrsPageId, - state == null ? -1 : (byte)state.ordinal(), - pageCnt, - link + io.getCacheSizesPageId(partMetaPageAddr), + io.getPartitionState(partMetaPageAddr), + io.getCandidatePageCount(partMetaPageAddr), + io.getGapsLink(partMetaPageAddr), + io.getTombstonesCount(partMetaPageAddr) )); } finally { @@ -485,6 +387,158 @@ else if (needSnapshot) tryAddEmptyPartitionToSnapshot(store, ctx); } + /** + * Saves to partition meta page information about partition update counter gaps. + * + * @param store Partition data store. + * @param io I/O for partition meta page. + * @param partMetaPageAddr Partition meta page address. + * @return {@code True} if partition meta data is changed. + * @throws IgniteCheckedException If failed. + */ + private boolean savePartitionUpdateCounterGaps( + CacheDataStore store, + PagePartitionMetaIOV2 io, + long partMetaPageAddr + ) throws IgniteCheckedException { + PartitionMetaStorage partStore = store.partStorage(); + + byte[] updCntrsBytes = store.partUpdateCounter().getBytes(); + + long gapsLink = io.getGapsLink(partMetaPageAddr); + + boolean changed = false; + + if (updCntrsBytes == null && gapsLink != 0) { + partStore.removeDataRowByLink(gapsLink, grp.statisticsHolderData()); + + io.setGapsLink(partMetaPageAddr, 0); + + changed = true; + } + else if (updCntrsBytes != null && gapsLink == 0) { + SimpleDataRow row = new SimpleDataRow(store.partId(), updCntrsBytes); + + partStore.insertDataRow(row, grp.statisticsHolderData()); + + io.setGapsLink(partMetaPageAddr, row.link()); + + changed = true; + } + else if (updCntrsBytes != null && gapsLink != 0) { + byte[] prev = partStore.readRow(gapsLink); + + assert prev != null : "Read null gaps using link=" + gapsLink; + + if (!Arrays.equals(prev, updCntrsBytes)) { + partStore.removeDataRowByLink(gapsLink, grp.statisticsHolderData()); + + SimpleDataRow row = new SimpleDataRow(store.partId(), updCntrsBytes); + + partStore.insertDataRow(row, grp.statisticsHolderData()); + + io.setGapsLink(partMetaPageAddr, row.link()); + + changed = true; + } + } + + if (changed) + partStore.saveMetadata(); + + return changed; + } + + /** + * Saves to partition meta page information about logical cache sizes inside cache group. + * + * @param store Partition data store. + * @param io I/O for partition meta page. + * @param partMetaPageAddr Partition meta page address. + * @return {@code True} if partition meta data is changed. + * @throws IgniteCheckedException If failed. + */ + private boolean saveCacheSizes( + CacheDataStore store, + PagePartitionMetaIOV2 io, + long partMetaPageAddr + ) throws IgniteCheckedException { + if (grp.sharedGroup()) { + PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory(); + + long oldCacheSizesPageId = io.getCacheSizesPageId(partMetaPageAddr); + + Map newSizes = store.cacheSizes(); + Map prevSizes = readSharedGroupCacheSizes(pageMem, grp.groupId(), oldCacheSizesPageId); + + if (prevSizes == null || !prevSizes.equals(newSizes)) { + long cacheSizesPageId = writeSharedGroupCacheSizes(pageMem, grp.groupId(), oldCacheSizesPageId, + store.partId(), newSizes); + + if (oldCacheSizesPageId == 0 && cacheSizesPageId != 0) { + io.setSizesPageId(partMetaPageAddr, cacheSizesPageId); + + return true; + } + } + } + else + io.setSizesPageId(partMetaPageAddr, 0); + + return false; + } + + /** + * Saves to partition meta page information about pages count. + * + * @param ctx Checkpoint context. + * @param part Partition. + * @param store Partition data store. + * @param io I/O for partition meta page. + * @param partMetaPageAddr Partition meta page address. + * @return {@code True} if partition meta data is changed. + * @throws IgniteCheckedException If failed. + */ + private boolean savePagesCount( + Context ctx, + GridDhtLocalPartition part, + CacheDataStore store, + PagePartitionMetaIOV2 io, + long partMetaPageAddr + ) throws IgniteCheckedException { + int grpId = grp.groupId(); + int pageCnt = this.ctx.pageStore().pages(grpId, store.partId()); + + io.setCandidatePageCount(partMetaPageAddr, io.getSize(partMetaPageAddr) == 0 ? 0 : pageCnt); + + if (part.state() == OWNING) { + assert part != null; + + if (!addPartition( + part, + ctx.partitionStatMap(), + partMetaPageAddr, + io, + grpId, + store.partId(), + this.ctx.pageStore().pages(grp.groupId(), store.partId()), + store.fullSize() + )) + U.warn(log, "Partition was concurrently evicted grpId=" + grpId + + ", partitionId=" + part.id()); + } + else if (part.state() == MOVING || part.state() == RENTING) { + if (ctx.partitionStatMap().forceSkipIndexPartition(grpId)) { + if (log.isInfoEnabled()) + log.info("Will not include SQL indexes to snapshot because there is " + + "a partition not in " + OWNING + " state [grp=" + grp.cacheOrGroupName() + + ", partId=" + store.partId() + ", state=" + part.state() + ']'); + } + } + + return true; + } + /** {@inheritDoc} */ @Override public long restorePartitionStates(Map partitionRecoveryStates) throws IgniteCheckedException { if (grp.isLocal() || !grp.affinityNode() || !grp.dataRegion().config().isPersistenceEnabled()) @@ -646,11 +700,13 @@ private GridDhtLocalPartition getPartition(CacheDataStore store) { * return null if counter page does not exist. * @throws IgniteCheckedException If page memory operation failed. */ - @Nullable private static Map readSharedGroupCacheSizes(PageSupport pageMem, int grpId, - long cntrsPageId) throws IgniteCheckedException { - + private static Map readSharedGroupCacheSizes( + PageSupport pageMem, + int grpId, + long cntrsPageId + ) throws IgniteCheckedException { if (cntrsPageId == 0L) - return null; + return Collections.emptyMap(); Map cacheSizes = new HashMap<>(); @@ -683,6 +739,7 @@ private GridDhtLocalPartition getPartition(CacheDataStore store) { pageMem.releasePage(grpId, curId, curPage); } } + return cacheSizes; } @@ -1824,16 +1881,19 @@ private CacheDataStore init0(boolean checkExists) throws IgniteCheckedException if (PageIO.getType(pageAddr) != 0) { PagePartitionMetaIOV2 io = (PagePartitionMetaIOV2)PagePartitionMetaIO.VERSIONS.latest(); - Map cacheSizes = null; - - if (grp.sharedGroup()) - cacheSizes = readSharedGroupCacheSizes(pageMem, grpId, io.getCountersPageId(pageAddr)); - - long link = io.getGapsLink(pageAddr); + long gapsLink = io.getGapsLink(pageAddr); - byte[] data = link == 0 ? null : partStorage.readRow(link); + byte[] updCntrGapsData = gapsLink == 0 ? null : partStorage.readRow(gapsLink); - delegate0.restoreState(io.getSize(pageAddr), io.getUpdateCounter(pageAddr), cacheSizes, data); + delegate0.restoreState( + io.getSize(pageAddr), + io.getUpdateCounter(pageAddr), + grp.sharedGroup() + ? readSharedGroupCacheSizes(pageMem, grpId, io.getCacheSizesPageId(pageAddr)) + : Collections.emptyMap(), + updCntrGapsData, + io.getTombstonesCount(pageAddr) + ); globalRemoveId().setIfGreater(io.getGlobalRemoveId(pageAddr)); } @@ -2791,9 +2851,25 @@ private int purgeExpiredInternal( } } + /** {@inheritDoc} */ @Override public PartitionMetaStorage partStorage() { return partStorage; } + + /** {@inheritDoc} */ + @Override public long tombstonesCount() { + try { + CacheDataStore delegate0 = init0(true); + + if (delegate0 == null) + return 0; + + return delegate0.tombstonesCount(); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java index c86a64a1e44f1..244845ab1e029 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java @@ -59,7 +59,7 @@ public class PagePartitionMetaIO extends PageMetaIO { setUpdateCounter(pageAddr, 0); setGlobalRemoveId(pageAddr, 0); setPartitionState(pageAddr, (byte)-1); - setCountersPageId(pageAddr, 0); + setSizesPageId(pageAddr, 0); } /** @@ -153,22 +153,22 @@ public boolean setPartitionState(long pageAddr, byte state) { } /** - * Returns partition counters page identifier, page with caches in cache group sizes. + * Returns page identifier related to page with logical cache sizes in cache group. * * @param pageAddr Partition metadata page address. * @return Next meta partial page ID or {@code 0} if it does not exist. */ - public long getCountersPageId(long pageAddr) { + public long getCacheSizesPageId(long pageAddr) { return PageUtils.getLong(pageAddr, NEXT_PART_META_PAGE_OFF); } /** - * Sets new reference to partition counters page (logical cache sizes). + * Sets new reference to page with logical cache sizes in cache group. * * @param pageAddr Partition metadata page address. * @param cntrsPageId New cache sizes page ID. */ - public void setCountersPageId(long pageAddr, long cntrsPageId) { + public void setSizesPageId(long pageAddr, long cntrsPageId) { PageUtils.putLong(pageAddr, NEXT_PART_META_PAGE_OFF, cntrsPageId); } @@ -228,6 +228,16 @@ public boolean setGapsLink(long pageAddr, long link) { "this PagePartitionMetaIO version: ver=" + getVersion()); } + public long getTombstonesCount(long pageAddr) { + throw new UnsupportedOperationException("Tombstones count is not supported by " + + "this PagePartitionMetaIO version: ver=" + getVersion()); + } + + public boolean setTombstonesCount(long pageAddr, long tombstonesCount) { + throw new UnsupportedOperationException("Tombstones count is not supported by " + + "this PagePartitionMetaIO version: ver=" + getVersion()); + } + /** {@inheritDoc} */ @Override protected void printPage(long pageAddr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException { super.printPage(pageAddr, pageSize, sb); @@ -238,7 +248,7 @@ public boolean setGapsLink(long pageAddr, long link) { .a(",\n\tupdateCounter=").a(getUpdateCounter(pageAddr)) .a(",\n\tglobalRemoveId=").a(getGlobalRemoveId(pageAddr)) .a(",\n\tpartitionState=").a(state).a("(").a(GridDhtPartitionState.fromOrdinal(state)).a(")") - .a(",\n\tcountersPageId=").a(getCountersPageId(pageAddr)) + .a(",\n\tcacheSizesPageId=").a(getCacheSizesPageId(pageAddr)) .a("\n]"); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java index 37b7243b35f4e..e915e0bfb62ac 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIOV2.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.processors.cache.persistence.tree.io; -import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.pagemem.PageUtils; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.util.GridStringBuilder; @@ -37,6 +36,9 @@ public class PagePartitionMetaIOV2 extends PagePartitionMetaIO { /** */ private static final int GAPS_LINK = PART_META_REUSE_LIST_ROOT_OFF + 8; + /** */ + private static final int TOMBSTONES_COUNT = GAPS_LINK + 8; + /** * @param ver Version. */ @@ -100,7 +102,22 @@ public boolean setGapsLink(long pageAddr, long link) { } /** {@inheritDoc} */ - @Override protected void printPage(long pageAddr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException { + @Override public long getTombstonesCount(long pageAddr) { + return PageUtils.getLong(pageAddr, TOMBSTONES_COUNT); + } + + /** {@inheritDoc} */ + @Override public boolean setTombstonesCount(long pageAddr, long tombstonesCnt) { + if (getTombstonesCount(pageAddr) == tombstonesCnt) + return false; + + PageUtils.putLong(pageAddr, TOMBSTONES_COUNT, tombstonesCnt); + + return true; + } + + /** {@inheritDoc} */ + @Override protected void printPage(long pageAddr, int pageSize, GridStringBuilder sb) { byte state = getPartitionState(pageAddr); sb.a("PagePartitionMeta[\n\ttreeRoot=").a(getReuseListRoot(pageAddr)); @@ -115,8 +132,9 @@ public boolean setGapsLink(long pageAddr, long link) { sb.a(",\n\tupdateCounter=").a(getUpdateCounter(pageAddr)); sb.a(",\n\tglobalRemoveId=").a(getGlobalRemoveId(pageAddr)); sb.a(",\n\tpartitionState=").a(state).a("(").a(GridDhtPartitionState.fromOrdinal(state)).a(")"); - sb.a(",\n\tcountersPageId=").a(getCountersPageId(pageAddr)); + sb.a(",\n\tcacheSizesPageId=").a(getCacheSizesPageId(pageAddr)); sb.a(",\n\tcntrUpdDataPageId=").a(getGapsLink(pageAddr)); + sb.a(",\n\ttombstonesCount=").a(getTombstonesCount(pageAddr)); sb.a("\n]"); } @@ -133,5 +151,6 @@ public void upgradePage(long pageAddr) { setPendingTreeRoot(pageAddr, 0); setPartitionMetaStoreReuseListRoot(pageAddr, 0); setGapsLink(pageAddr, 0); + setTombstonesCount(pageAddr, 0); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java index ec6897236eb0d..08e980f9c4252 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java @@ -71,6 +71,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSnapshotId; import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV2; +import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecordV3; import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.PageListMetaResetCountRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListAddPageRecord; @@ -379,6 +380,10 @@ assert record instanceof PageSnapshot; return /*cache ID*/4 + /*page ID*/8 + /*upd cntr*/8 + /*rmv id*/8 + /*part size*/4 + /*counters page id*/8 + /*state*/ 1 + /*allocatedIdxCandidate*/ 4 + /*link*/ 8; + case PARTITION_META_PAGE_UPDATE_COUNTERS_V3: + return /*cache ID*/4 + /*page ID*/8 + /*upd cntr*/8 + /*rmv id*/8 + /*part size*/4 + /*counters page id*/8 + /*state*/ 1 + + /*allocatedIdxCandidate*/ 4 + /*link*/ 8 + /*tombstones cnt*/ 8; + case MEMORY_RECOVERY: return 8; @@ -611,6 +616,11 @@ WALRecord readPlainRecord(RecordType type, ByteBufferBackedDataInput in, break; + case PARTITION_META_PAGE_UPDATE_COUNTERS_V3: + res = new MetaPageUpdatePartitionDataRecordV3(in); + + break; + case MEMORY_RECOVERY: long ts = in.readLong(); @@ -1202,6 +1212,7 @@ void writePlainRecord(WALRecord rec, ByteBuffer buf) throws IgniteCheckedExcepti case PARTITION_META_PAGE_UPDATE_COUNTERS: case PARTITION_META_PAGE_UPDATE_COUNTERS_V2: + case PARTITION_META_PAGE_UPDATE_COUNTERS_V3: ((MetaPageUpdatePartitionDataRecord)rec).toBytes(buf); break; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/MetricUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/MetricUtils.java index fad16b5e95168..d80e33b57ea3a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/MetricUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/impl/MetricUtils.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.metric.impl; import java.util.Map; +import org.apache.ignite.internal.processors.cache.CacheGroupMetricsImpl; import org.apache.ignite.internal.processors.metric.GridMetricManager; import org.apache.ignite.internal.processors.metric.MetricRegistry; import org.apache.ignite.internal.util.typedef.T2; @@ -85,6 +86,14 @@ public static String cacheMetricsRegistryName(String cacheName, boolean isNear) return metricName(CACHE_METRICS, cacheName); } + /** + * @param cacheGrpName Cache group name. + * @return Cache group metrics registry name. + */ + public static String cacheGroupMetricsRegistryName(String cacheGrpName) { + return metricName(CacheGroupMetricsImpl.CACHE_GROUP_METRICS_PREFIX, cacheGrpName); + } + /** * Atomically sets the value to the given updated value * if the current value {@code ==} the expected value. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java index f2af27b544d50..d0661534250b9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java @@ -19,17 +19,19 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.Callable; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.Ignition; import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; @@ -37,103 +39,156 @@ import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.MvccFeatureChecker; +import org.apache.ignite.testframework.junits.WithSystemProperty; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.After; import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; -import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; -import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.cacheMetricsRegistryName; +import static org.apache.ignite.cache.CacheRebalanceMode.SYNC; /** * */ +@RunWith(Parameterized.class) public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { + /** Dummy data. */ + private static final byte[] DUMMY_DATA = {}; + + /** Test parameters. */ + @Parameterized.Parameters(name = "persistenceEnabled={0}, historicalRebalance={1}") + public static Collection parameters() { + List res = new ArrayList<>(); + + for (boolean persistenceEnabled : new boolean[] {false, true}) { + for (boolean histRebalance : new boolean[] {false, true}) { + if (!persistenceEnabled && histRebalance) + continue; + + res.add(new Object[]{persistenceEnabled, histRebalance}); + } + } + + return res; + } + /** */ - private boolean persistence; + @Parameterized.Parameter(0) + public boolean persistence; + + /** */ + @Parameterized.Parameter(1) + public boolean histRebalance; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); + cfg.setConsistentId(gridName); + DataStorageConfiguration dsCfg = new DataStorageConfiguration(); if (persistence) { dsCfg.setDefaultDataRegionConfiguration( - new DataRegionConfiguration().setMaxSize(100L * 1024 * 1024).setPersistenceEnabled(true)) - .setWalMode(WALMode.LOG_ONLY); + new DataRegionConfiguration() + .setInitialSize(256L * 1024 * 1024) + .setMaxSize(256L * 1024 * 1024) + .setPersistenceEnabled(true)) + .setWalMode(WALMode.LOG_ONLY); } dsCfg.setPageSize(1024); cfg.setDataStorageConfiguration(dsCfg); - // Long rebalance. + // Throttle rebalance. cfg.setRebalanceThrottle(100); return cfg; } - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - super.beforeTestsStarted(); - + /** + * + */ + @BeforeClass + public static void beforeTests() { Assume.assumeFalse(MvccFeatureChecker.forcedMvcc()); } - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - + /** + * + */ + @Before + public void before() throws Exception { cleanPersistenceDir(); - } - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { stopAllGrids(); - cleanPersistenceDir(); - - super.afterTest(); + if (histRebalance) + System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0"); } /** - * @throws Exception If failed. + * */ - @Test - public void testRemoveAndRebalance() throws Exception { - removeAndRebalance(); - } + @After + public void after() throws Exception { + if (histRebalance) + System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD); - /** - * @throws Exception If failed. - */ - @Test - public void testRemoveAndRebalanceWithPersistence() throws Exception { - persistence = true; + stopAllGrids(); - testRemoveAndRebalance(); + cleanPersistenceDir(); } /** * @throws Exception If failed. */ - private void removeAndRebalance() throws Exception { + @Test + @WithSystemProperty(key = IgniteSystemProperties.IGNITE_BASELINE_AUTO_ADJUST_ENABLED, value = "false") + public void removeAndRebalance() throws Exception { IgniteEx ignite0 = startGrid(0); - if (persistence) + IgniteCache cache0; + + final int ADD_NODES = persistence ? 2 : 3; + final int KEYS = persistence ? 5_000 : 10_000; + + if (persistence) { + // Preload initial data to all nodes to have start point for WAL rebalance. + for (int i = 0, idx = 1; i < ADD_NODES; i++, idx++) + startGrid(idx); + ignite0.cluster().active(true); - final int pageSize = ignite0.configuration().getDataStorageConfiguration().getPageSize(); + awaitPartitionMapExchange(); + + cache0 = ignite0.getOrCreateCache(cacheConfiguration()); + + for (int k = 0; k < KEYS; k++) + cache0.put(new TestKey(k, DUMMY_DATA), new TestValue(DUMMY_DATA)); + + forceCheckpoint(); - assert pageSize > 0; + for (int i = 0, idx = 1; i < ADD_NODES; i++, idx++) { + stopGrid(idx); - IgniteCache cache0 = ignite0.createCache(cacheConfiguration()); + awaitPartitionMapExchange(); + } + } + + final int pageSize = ignite0.configuration().getDataStorageConfiguration().getPageSize(); ThreadLocalRandom rnd = ThreadLocalRandom.current(); @@ -141,9 +196,6 @@ private void removeAndRebalance() throws Exception { Map data = new HashMap<>(); - final int KEYS = persistence ? 5_000 : 10_000; - final int ADD_NODES = persistence ? 2 : 3; - for (int i = 0; i < KEYS; i++) { TestKey key = new TestKey(i, new byte[rnd.nextInt(pageSize * 3)]); @@ -152,24 +204,26 @@ private void removeAndRebalance() throws Exception { data.put(key, new TestValue(new byte[rnd.nextInt(pageSize * 3)])); } + cache0 = ignite0.getOrCreateCache(cacheConfiguration()); + cache0.putAll(data); AtomicInteger nodeIdx = new AtomicInteger(); for (int iter = 0; iter < ADD_NODES; iter++) { - IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { - @Override public Object call() throws Exception { - int idx = nodeIdx.incrementAndGet(); + IgniteInternalFuture nodeStartFut = GridTestUtils.runAsync(() -> { + int idx = nodeIdx.incrementAndGet(); - info("Start node: " + idx); + info("Start node: " + idx); - return startGrid(idx); - } + U.sleep(500); + + return startGrid(idx); }); - long endTime = System.currentTimeMillis() + 2500; + long endTime = U.currentTimeMillis() + 5_000; - while (System.currentTimeMillis() < endTime) { + while (U.currentTimeMillis() < endTime) { for (int i = 0; i < 100; i++) { TestKey key = keys.get(rnd.nextInt(keys.size())); @@ -177,18 +231,19 @@ private void removeAndRebalance() throws Exception { cache0.remove(key); data.remove(key); - } else { + } + else { TestValue val = new TestValue(new byte[rnd.nextInt(pageSize * 3)]); cache0.put(key, val); data.put(key, val); } - Thread.sleep(10); + U.sleep(10); } } - fut.get(30_000); + nodeStartFut.get(30_000); checkData(keys, data); @@ -200,23 +255,21 @@ private void removeAndRebalance() throws Exception { awaitPartitionMapExchange(); for (int iter = 0; iter < ADD_NODES; iter++) { - IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { - @Override public Object call() throws Exception { - int idx = nodeIdx.getAndDecrement(); + IgniteInternalFuture nodeStopFut = GridTestUtils.runAsync(() -> { + int idx = nodeIdx.getAndDecrement(); - info("Stop node: " + idx); + info("Stop node: " + idx); - stopGrid(idx); + stopGrid(idx); - awaitPartitionMapExchange(); + awaitPartitionMapExchange(); - return null; - } + return null; }); - long endTime = System.currentTimeMillis() + 2500; + long endTime = U.currentTimeMillis() + 2_500; - while (System.currentTimeMillis() < endTime) { + while (U.currentTimeMillis() < endTime) { for (int i = 0; i < 100; i++) { TestKey key = keys.get(rnd.nextInt(keys.size())); @@ -232,10 +285,10 @@ private void removeAndRebalance() throws Exception { } } - Thread.sleep(10); + U.sleep(10); } - fut.get(30_000); + nodeStopFut.get(30_000); checkData(keys, data); @@ -251,9 +304,6 @@ private void removeAndRebalance() throws Exception { */ private void checkData(List keys, Map data) { for (Ignite node : Ignition.allGrids()) { - if (!node.name().endsWith("CacheRemoveWithTombstonesLoadTest1")) - continue; - info("Check node: " + node.name()); IgniteCache cache = node.cache(DEFAULT_CACHE_NAME); @@ -278,13 +328,9 @@ private void checkData(List keys, Map data) { private void waitTombstoneCleanup() throws Exception { for (Ignite node : Ignition.allGrids()) { final LongMetric tombstones = ((IgniteEx)node).context().metric().registry( - cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); + MetricUtils.cacheGroupMetricsRegistryName(DEFAULT_CACHE_NAME)).findMetric("Tombstones"); - GridTestUtils.waitForCondition(new GridAbsPredicate() { - @Override public boolean apply() { - return tombstones.value() == 0; - } - }, 30_000); + GridTestUtils.waitForCondition(() -> tombstones.value() == 0, 30_000); assertEquals("Failed to wait for tombstone cleanup: " + node.name(), 0, tombstones.value()); } @@ -298,9 +344,11 @@ private CacheConfiguration cacheConfiguration() { ccfg.setAtomicityMode(TRANSACTIONAL); ccfg.setCacheMode(PARTITIONED); - ccfg.setBackups(1); - ccfg.setRebalanceMode(ASYNC); + ccfg.setBackups(2); + ccfg.setRebalanceMode(SYNC); + ccfg.setReadFromBackup(true); ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + ccfg.setAffinity(new RendezvousAffinityFunction(false, 64)); return ccfg; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index 05b4158f7825e..1b6ca3fecb5ff 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -17,47 +17,72 @@ package org.apache.ignite.internal.processors.cache.distributed; -import java.util.HashMap; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; -import java.util.Map; +import java.util.List; import java.util.Set; -import java.util.concurrent.Callable; -import org.apache.ignite.Ignite; +import java.util.concurrent.TimeUnit; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; -import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.DataRegionConfiguration; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage; -import org.apache.ignite.internal.util.lang.GridAbsPredicate; -import org.apache.ignite.lang.IgniteBiPredicate; -import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; -import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.cacheMetricsRegistryName; /** * */ +@RunWith(Parameterized.class) public class CacheRemoveWithTombstonesTest extends GridCommonAbstractTest { + /** Test parameters. */ + @Parameterized.Parameters(name = "persistenceEnabled={0}, historicalRebalance={1}") + public static Collection parameters() { + List res = new ArrayList<>(); + + for (boolean persistenceEnabled : new boolean[] {false, true}) { + for (boolean histRebalance : new boolean[] {false, true}) { + if (!persistenceEnabled && histRebalance) + continue; + + res.add(new Object[]{persistenceEnabled, histRebalance}); + } + } + + return res; + } + /** */ - private boolean persistence; + @Parameterized.Parameter(0) + public boolean persistence; + + /** */ + @Parameterized.Parameter(1) + public boolean histRebalance; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { @@ -65,12 +90,18 @@ public class CacheRemoveWithTombstonesTest extends GridCommonAbstractTest { TestRecordingCommunicationSpi commSpi = new TestRecordingCommunicationSpi(); + cfg.setConsistentId(gridName); + cfg.setCommunicationSpi(commSpi); if (persistence) { DataStorageConfiguration dsCfg = new DataStorageConfiguration() .setDefaultDataRegionConfiguration( - new DataRegionConfiguration().setMaxSize(100L * 1024 * 1024).setPersistenceEnabled(true)) + new DataRegionConfiguration() + .setInitialSize(256L * 1024 * 1024) + .setMaxSize(256L * 1024 * 1024) + .setPersistenceEnabled(true) + ) .setWalMode(WALMode.LOG_ONLY); cfg.setDataStorageConfiguration(dsCfg); @@ -79,20 +110,30 @@ public class CacheRemoveWithTombstonesTest extends GridCommonAbstractTest { return cfg; } - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); + /** + * + */ + @Before + public void before() throws Exception { + stopAllGrids(); cleanPersistenceDir(); + + if (histRebalance) + System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0"); } - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { + /** + * + */ + @After + public void after() throws Exception { + if (histRebalance) + System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD); + stopAllGrids(); cleanPersistenceDir(); - - super.afterTest(); } /** @@ -111,26 +152,6 @@ public void testRemoveAndRebalanceRaceAtomic() throws Exception { testRemoveAndRebalanceRace(ATOMIC, false); } - /** - * @throws Exception If failed. - */ - @Test - public void testRemoveAndRebalanceRaceTxWithPersistence() throws Exception { - persistence = true; - - testRemoveAndRebalanceRaceTx(); - } - - /** - * @throws Exception If failed. - */ - @Test - public void testRemoveAndRebalanceRaceAtomicWithPersistence() throws Exception { - persistence = true; - - testRemoveAndRebalanceRaceAtomic(); - } - /** * @throws Exception If failed. * @param expTombstone {@code True} if tombstones should be created. @@ -138,6 +159,9 @@ public void testRemoveAndRebalanceRaceAtomicWithPersistence() throws Exception { private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolean expTombstone) throws Exception { IgniteEx ignite0 = startGrid(0); + if (histRebalance) + startGrid(1); + if (persistence) ignite0.cluster().active(true); @@ -146,27 +170,32 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea if (MvccFeatureChecker.forcedMvcc()) expTombstone = false; - LongMetric tombstoneMetric0 = ignite0.context().metric().registry( - cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); + final int KEYS = 1024 * 64; + + if (histRebalance) { + // Preload initial data to have start point for WAL rebalance. + try (IgniteDataStreamer streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) { + streamer.allowOverwrite(true); + + for (int i = 0; i < KEYS; i++) + streamer.addData(i, 0); + } - Map map = new HashMap<>(); + forceCheckpoint(); - final int KEYS = 1024; + stopGrid(1); + } - for (int i = 0; i < KEYS; i++) - map.put(i, i); + try (IgniteDataStreamer streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) { + streamer.allowOverwrite(true); - cache0.putAll(map); + for (int i = 0; i < KEYS; i++) + streamer.addData(i, i); + } blockRebalance(ignite0); - IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { - @Override public Object call() throws Exception { - return startGrid(1); - } - }); - - IgniteEx ignite1 = (IgniteEx)fut.get(30_000); + IgniteEx ignite1 = GridTestUtils.runAsync(() -> startGrid(1)).get(10, TimeUnit.SECONDS); if (persistence) { ignite0.cluster().baselineAutoAdjustEnabled(false); @@ -174,43 +203,44 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea ignite0.cluster().setBaselineTopology(2); } - Set removed = new HashSet<>(); + TestRecordingCommunicationSpi.spi(ignite0).waitForBlocked(); + + Set keysWithTombstone = new HashSet<>(); // Do removes while rebalance is in progress. - for (int i = 0; i < KEYS; i++) { - if (i % 2 == 0) { - removed.add(i); + for (int i = 0; i < KEYS; i += 64) { + keysWithTombstone.add(i); - cache0.remove(i); - } + cache0.remove(i); } - final LongMetric tombstoneMetric1 = ignite1.context().metric().registry( - cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); + final LongMetric tombstoneMetric0 = ignite0.context().metric().registry( + MetricUtils.cacheGroupMetricsRegistryName(DEFAULT_CACHE_NAME)).findMetric("Tombstones"); + + final LongMetric tombstoneMetric1 = ignite1.context().metric().registry( + MetricUtils.cacheGroupMetricsRegistryName(DEFAULT_CACHE_NAME)).findMetric("Tombstones"); // On first node there should not be tombstones. assertEquals(0, tombstoneMetric0.value()); if (expTombstone) - assertEquals(removed.size(), tombstoneMetric1.value()); + assertEquals(keysWithTombstone.size(), tombstoneMetric1.value()); else assertEquals(0, tombstoneMetric1.value()); // Update some of removed keys, this should remove tombstones. - for (int i = 0; i < KEYS; i++) { - if (i % 4 == 0) { - removed.remove(i); + for (int i = 0; i < KEYS; i += 128) { + keysWithTombstone.remove(i); - cache0.put(i, i); - } + cache0.put(i, i); } - assert !removed.isEmpty(); + assertTrue("Keys with tombstones should exist", !keysWithTombstone.isEmpty()); assertEquals(0, tombstoneMetric0.value()); if (expTombstone) - assertEquals(removed.size(), tombstoneMetric1.value()); + assertEquals(keysWithTombstone.size(), tombstoneMetric1.value()); else assertEquals(0, tombstoneMetric1.value()); @@ -221,18 +251,14 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea IgniteCache cache1 = ignite(1).cache(DEFAULT_CACHE_NAME); for (int i = 0; i < KEYS; i++) { - if (removed.contains(i)) + if (keysWithTombstone.contains(i)) assertNull(cache1.get(i)); else assertEquals((Object)i, cache1.get(i)); } // Tombstones should be removed after once rebalance is completed. - GridTestUtils.waitForCondition(new GridAbsPredicate() { - @Override public boolean apply() { - return tombstoneMetric1.value() == 0; - } - }, 30_000); + GridTestUtils.waitForCondition(() -> tombstoneMetric1.value() == 0, 30_000); assertEquals(0, tombstoneMetric1.value()); } @@ -240,15 +266,13 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea /** * */ - private void blockRebalance(Ignite node) { - final int grpId = groupIdForCache(ignite(0), DEFAULT_CACHE_NAME); + private static void blockRebalance(IgniteEx node) { + final int grpId = groupIdForCache(node, DEFAULT_CACHE_NAME); - TestRecordingCommunicationSpi.spi(node).blockMessages(new IgniteBiPredicate() { - @Override public boolean apply(ClusterNode node, Message msg) { - return (msg instanceof GridDhtPartitionSupplyMessage) - && ((GridCacheGroupIdMessage)msg).groupId() == grpId; - } - }); + TestRecordingCommunicationSpi.spi(node).blockMessages((node0, msg) -> + (msg instanceof GridDhtPartitionSupplyMessage) + && ((GridCacheGroupIdMessage)msg).groupId() == grpId + ); } /** @@ -263,6 +287,7 @@ private CacheConfiguration cacheConfiguration(CacheAtomicityMo ccfg.setBackups(2); ccfg.setRebalanceMode(ASYNC); ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + ccfg.setAffinity(new RendezvousAffinityFunction(false, 64)); return ccfg; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/CacheRemoveWithTombstonesFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/CacheRemoveWithTombstonesFailoverTest.java new file mode 100644 index 0000000000000..db60ffc8a062d --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/CacheRemoveWithTombstonesFailoverTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.dht.topology; + +import java.util.HashSet; +import java.util.Set; +import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage; +import org.apache.ignite.internal.processors.metric.impl.MetricUtils; +import org.apache.ignite.spi.metric.LongMetric; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.WithSystemProperty; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; + +/** + * Tests to check failover scenarios over tombstones. + */ +public class CacheRemoveWithTombstonesFailoverTest extends PartitionsEvictManagerAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + TestRecordingCommunicationSpi commSpi = new TestRecordingCommunicationSpi(); + + cfg.setConsistentId(gridName); + + cfg.setCommunicationSpi(commSpi); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setInitialSize(256L * 1024 * 1024) + .setMaxSize(256L * 1024 * 1024) + .setPersistenceEnabled(true) + ) + .setCheckpointFrequency(1024 * 1024 * 1024) + .setWalMode(WALMode.LOG_ONLY); + + cfg.setDataStorageConfiguration(dsCfg); + + cfg.setCacheConfiguration(cacheConfiguration()); + + return cfg; + } + + /** + * Test check that tombstones reside in persistent partition will be cleared after node restart. + */ + @Test + @WithSystemProperty(key = IgniteSystemProperties.IGNITE_BASELINE_AUTO_ADJUST_ENABLED, value = "false") + public void testTombstonesClearedAfterRestart() throws Exception { + IgniteEx crd = startGrid(0); + + crd.cluster().active(true); + + final int KEYS = 1024; + + for (int k = 0; k < KEYS; k++) + crd.cache(DEFAULT_CACHE_NAME).put(k, k); + + blockRebalance(crd); + + IgniteEx node = startGrid(1); + + // Do not run clear tombsones task. + instrumentEvictionQueue(node, task -> { + if (task instanceof PartitionsEvictManager.ClearTombstonesTask) + return null; + + return task; + }); + + resetBaselineTopology(); + + TestRecordingCommunicationSpi.spi(crd).waitForBlocked(); + + Set keysWithTombstone = new HashSet<>(); + + // Do removes while rebalance is in progress. + for (int i = 0; i < KEYS; i += 2) { + keysWithTombstone.add(i); + + crd.cache(DEFAULT_CACHE_NAME).remove(i); + } + + final LongMetric tombstoneMetric = node.context().metric().registry( + MetricUtils.cacheGroupMetricsRegistryName(DEFAULT_CACHE_NAME)).findMetric("Tombstones"); + + Assert.assertEquals(keysWithTombstone.size(), tombstoneMetric.value()); + + // Resume rebalance. + TestRecordingCommunicationSpi.spi(crd).stopBlock(); + + // Partitions should be in OWNING state. + awaitPartitionMapExchange(); + + // But tombstones removal should be skipped. + Assert.assertEquals(keysWithTombstone.size(), tombstoneMetric.value()); + + // Stop node with tombstones. + stopGrid(1); + + // Stop coordinator. + stopGrid(0); + + // Startup node with tombstones in inactive state. + node = startGrid(1); + + final int grpId = groupIdForCache(node, DEFAULT_CACHE_NAME); + + // Tombstone metrics are unavailable before join to topology, using internal api. + long tombstonesBeforeActivation = node.context().cache().cacheGroup(grpId).topology().localPartitions() + .stream().map(part -> part.dataStore().tombstonesCount()).reduce(Long::sum).orElse(0L); + + Assert.assertEquals(keysWithTombstone.size(), tombstonesBeforeActivation); + + crd = startGrid(0); + + crd.cluster().active(true); + + awaitPartitionMapExchange(); + + final LongMetric tombstoneMetric1 = node.context().metric().registry( + MetricUtils.cacheGroupMetricsRegistryName(DEFAULT_CACHE_NAME)).findMetric("Tombstones"); + + // Tombstones should be removed after join to topology. + GridTestUtils.waitForCondition(() -> tombstoneMetric1.value() == 0, 30_000); + + assertEquals(0, tombstoneMetric1.value()); + } + + /** + * + */ + private static void blockRebalance(IgniteEx node) { + final int grpId = groupIdForCache(node, DEFAULT_CACHE_NAME); + + TestRecordingCommunicationSpi.spi(node).blockMessages((node0, msg) -> + (msg instanceof GridDhtPartitionSupplyMessage) + && ((GridCacheGroupIdMessage)msg).groupId() == grpId + ); + } + + /** + * @return Cache configuration. + */ + private CacheConfiguration cacheConfiguration() { + return new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setAtomicityMode(TRANSACTIONAL) + .setCacheMode(PARTITIONED) + .setBackups(1) + .setRebalanceMode(ASYNC) + .setReadFromBackup(true) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setAffinity(new RendezvousAffinityFunction(false, 64)); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/DropCacheContextDuringEvictionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/DropCacheContextDuringEvictionTest.java index 4a7de041fd0ea..72a28532b4927 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/DropCacheContextDuringEvictionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/DropCacheContextDuringEvictionTest.java @@ -39,11 +39,15 @@ public class DropCacheContextDuringEvictionTest extends PartitionsEvictManagerAb public void testDeactivation() throws Exception { T2 nodeAndEvictLatch = makeNodeWithEvictLatch(); - IgniteCache cache = nodeAndEvictLatch.get1().createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) + nodeAndEvictLatch.get1().createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) .setGroupName("test-grp")); - for (int i = 0; i < 100_000; i++) - cache.put(i, i); + try (IgniteDataStreamer streamer = nodeAndEvictLatch.get1().dataStreamer(DEFAULT_CACHE_NAME)) { + streamer.allowOverwrite(true); + + for (int k = 0; k < 100_000; k++) + streamer.addData(k, k); + } doActionDuringEviction(nodeAndEvictLatch, () -> nodeAndEvictLatch.get1().cluster().active(false)); @@ -60,17 +64,19 @@ public void testDestroyCacheGroup() throws Exception { List caches = new ArrayList<>(); for (int idx = 0; idx < 10; idx++) { - IgniteCache cache = nodeAndEvictLatch.get1().createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME + idx) - .setGroupName("test-grp")); + String cacheName = DEFAULT_CACHE_NAME + idx; - caches.add(cache.getName()); + nodeAndEvictLatch.get1().createCache(new CacheConfiguration<>(cacheName) + .setGroupName("test-grp")); - try (IgniteDataStreamer streamer = nodeAndEvictLatch.get1().dataStreamer(cache.getName())) { + try (IgniteDataStreamer streamer = nodeAndEvictLatch.get1().dataStreamer(cacheName)) { streamer.allowOverwrite(true); - for (int i = 0; i < 200_000; i++) - streamer.addData(i, i); + for (int k = 0; k < 100_000; k++) + streamer.addData(k, k); } + + caches.add(cacheName); } doActionDuringEviction(nodeAndEvictLatch, () -> nodeAndEvictLatch.get1().destroyCaches(caches)); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManagerAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManagerAbstractTest.java index e49e07fe6dc15..233b25ec5db80 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManagerAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManagerAbstractTest.java @@ -17,26 +17,22 @@ package org.apache.ignite.internal.processors.cache.distributed.dht.topology; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.util.Queue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; -import javax.annotation.Nullable; import org.apache.ignite.Ignite; -import org.apache.ignite.configuration.DataRegionConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.failure.AbstractFailureHandler; import org.apache.ignite.failure.FailureContext; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteClosure; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.NotNull; /** * @@ -49,8 +45,7 @@ public abstract class PartitionsEvictManagerAbstractTest extends GridCommonAbstr @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - cfg.setDataStorageConfiguration(new DataStorageConfiguration() - .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true))); + cfg.setActiveOnStart(false); cfg.setFailureHandler(new AbstractFailureHandler() { /** {@inheritDoc} */ @@ -67,12 +62,16 @@ public abstract class PartitionsEvictManagerAbstractTest extends GridCommonAbstr /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { + stopAllGrids(); + cleanPersistenceDir(); } /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); + + cleanPersistenceDir(); } /** @@ -92,40 +91,43 @@ protected void awaitEvictionQueueIsEmpty(IgniteEx node, int ms) throws IgniteInt protected void awaitEvictionQueueForFilling(IgniteEx node, int ms) throws IgniteInterruptedCheckedException { PartitionsEvictManager.BucketQueue evictionQueue = node.context().cache().context().evict().evictionQueue; - assertTrue(GridTestUtils.waitForCondition(() -> !evictionQueue.isEmpty(), ms)); + assertTrue(GridTestUtils.waitForCondition(() -> { + for (Queue queue : evictionQueue.buckets) + return ((InstrumentedEvictionQueue) queue).itemOffered; + + return false; + }, ms)); } /** * @param node Node. - * @param latch Latch. - * @param completeWithError Inner future throws exception. + * @param interceptor Interceptor that will be invoked after task from eviction has polled. */ - protected void subscribeEvictionQueueAtLatch(IgniteEx node, CountDownLatch latch, boolean completeWithError) { + protected void instrumentEvictionQueue( + IgniteEx node, + IgniteClosure interceptor + ) { PartitionsEvictManager.BucketQueue evictionQueue = node.context().cache().context().evict().evictionQueue; Queue[] buckets = evictionQueue.buckets; for (int i = 0; i < buckets.length; i++) - buckets[i] = new WaitingQueue(latch, completeWithError); + buckets[i] = new InstrumentedEvictionQueue(interceptor); } /** * */ protected T2 makeNodeWithEvictLatch() throws Exception { - return makeNodeWithEvictLatch(false); - } - - /** - * - */ - protected T2 makeNodeWithEvictLatch(boolean completeWithError) throws Exception { IgniteEx node1 = startGrid(0); - node1.cluster().baselineAutoAdjustEnabled(false); - CountDownLatch latch = new CountDownLatch(1); - subscribeEvictionQueueAtLatch(node1, latch, completeWithError); + instrumentEvictionQueue(node1, task -> { + U.awaitQuiet(latch); + + return task; + }); node1.cluster().active(true); @@ -137,11 +139,7 @@ protected T2 makeNodeWithEvictLatch(boolean completeWi * @param r R. */ protected void doActionDuringEviction(T2 nodeAndEvictLatch, Runnable r) throws Exception { - IgniteEx node2 = startGrid(1); - - awaitPartitionMapExchange(); - - nodeAndEvictLatch.get1().cluster().setBaselineTopology(node2.cluster().topologyVersion()); + startGrid(1); awaitEvictionQueueForFilling(nodeAndEvictLatch.get1(), 100_000); @@ -153,55 +151,38 @@ protected void doActionDuringEviction(T2 nodeAndEvictL } /** - * Queue witch waits on the poll or breaks a PartitionEvictionTask. + * Queue that executes an interceptor during eviction task poll. */ - private class WaitingQueue extends LinkedBlockingQueue { - /** Latch. */ - private final CountDownLatch latch; + private static class InstrumentedEvictionQueue extends LinkedBlockingQueue { + /** Interceptor. */ + private final IgniteClosure interceptor; - /** Complete with error. */ - private final boolean completeWithError; + /** Empty indicator. */ + private volatile boolean itemOffered; /** - * @param latch Latch. - * @param completeWithError flag. + * @param interceptor Interceptor. */ - public WaitingQueue(CountDownLatch latch, boolean completeWithError) { - this.latch = latch; - this.completeWithError = completeWithError; + private InstrumentedEvictionQueue(IgniteClosure interceptor + ) { + this.interceptor = interceptor; } /** {@inheritDoc} */ - @Override public Object poll() { - U.awaitQuiet(latch); + @Override public boolean offer(@NotNull Object o) { + itemOffered = true; + return super.offer(o); + } + + /** {@inheritDoc} */ + @Override public Object poll() { Object obj = super.poll(); - // This code uses for failure handler testing into PartitionEvictionTask. - if(obj != null && completeWithError) { - try { - Field field = U.findField(PartitionsEvictManager.PartitionEvictionTask.class, "finishFut"); - - field.setAccessible(true); - - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - - field.set(obj, new GridFutureAdapter() { - @Override - protected boolean onDone(@Nullable Object res, @Nullable Throwable err, boolean cancel) { - if (err == null) - throw new RuntimeException("TEST"); - - return super.onDone(res, err, cancel); - } - }); - } - catch (Exception e) { - fail(); - } - } + if (obj instanceof PartitionsEvictManager.AbstractEvictionTask) + return interceptor.apply((PartitionsEvictManager.AbstractEvictionTask) obj); return obj; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictionTaskFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictionTaskFailureHandlerTest.java index 58c24608c29c5..e78c61b3d5075 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictionTaskFailureHandlerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictionTaskFailureHandlerTest.java @@ -17,11 +17,16 @@ package org.apache.ignite.internal.processors.cache.distributed.dht.topology; -import java.util.concurrent.CountDownLatch; -import org.apache.ignite.IgniteCache; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; +import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; import org.junit.Test; @@ -29,20 +34,69 @@ * */ public class PartitionsEvictionTaskFailureHandlerTest extends PartitionsEvictManagerAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setCacheConfiguration(new CacheConfiguration<>(DEFAULT_CACHE_NAME)); + + return cfg; + } + /** * */ @Test public void testEvictionTaskShouldCallFailureHandler() throws Exception { - T2 nodeAndEvictLatch = makeNodeWithEvictLatch(true); + IgniteEx node = startGrid(0); + + AtomicBoolean once = new AtomicBoolean(); + + // Partition eviction task should throw exception after completion. + instrumentEvictionQueue(node, task -> { + if (!(task instanceof PartitionsEvictManager.PartitionEvictionTask)) + return task; + + // Fail once. + if (!once.compareAndSet(false, true)) + return task; + + try { + Field field = U.findField(PartitionsEvictManager.PartitionEvictionTask.class, "finishFut"); + + field.setAccessible(true); + + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + + field.set(task, new GridFutureAdapter() { + @Override protected boolean onDone(@Nullable Object res, @Nullable Throwable err, boolean cancel) { + if (err == null) + throw new RuntimeException("TEST"); + + return super.onDone(res, err, cancel); + } + }); + } + catch (Exception e) { + fail(); + } + + return task; + }); + + node.cluster().active(true); - IgniteCache cache = nodeAndEvictLatch.get1().createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME) - .setGroupName("test-grp")); + try (IgniteDataStreamer streamer = node.dataStreamer(DEFAULT_CACHE_NAME)) { + streamer.allowOverwrite(true); - for (int i = 0; i < 100_000; i++) - cache.put(i, i); + for (int k = 0; k < 1024; k++) + node.cache(DEFAULT_CACHE_NAME).put(k, k); + } - doActionDuringEviction(nodeAndEvictLatch, () -> {}); + // Some partitions from node 0 should be evicted. + startGrid(1); assertTrue(GridTestUtils.waitForCondition(() -> failure.get(), 10_000)); } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java index 3518cc3708f1a..6d18a7239428d 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java @@ -43,6 +43,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCacheWriteSynchronizationModesMultithreadedTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxConcurrentRemoveObjectsTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.CacheRemoveWithTombstonesFailoverTest; import org.apache.ignite.internal.processors.cache.transactions.PartitionUpdateCounterTest; import org.apache.ignite.internal.processors.cache.transactions.TxDataConsistencyOnCommitFailureTest; import org.apache.ignite.internal.processors.cache.transactions.TxPartitionCounterStateConsistencyHistoryRebalanceTest; @@ -126,6 +127,7 @@ public static List> suite(Collection ignoredTests) { GridTestUtils.addTestIfNeeded(suite, CacheRemoveWithTombstonesTest.class, ignoredTests); GridTestUtils.addTestIfNeeded(suite, CacheRemoveWithTombstonesLoadTest.class, ignoredTests); + GridTestUtils.addTestIfNeeded(suite, CacheRemoveWithTombstonesFailoverTest.class, ignoredTests); return suite; } From da4c092d11960174f30e65f8a0e55f47ded11117 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 2 Oct 2019 19:31:52 +0300 Subject: [PATCH 24/30] IGNITE-11704 Missed docs. --- .../cache/persistence/tree/io/PagePartitionMetaIO.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java index 244845ab1e029..931fd92fbc4dd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java @@ -228,11 +228,18 @@ public boolean setGapsLink(long pageAddr, long link) { "this PagePartitionMetaIO version: ver=" + getVersion()); } + /** + * @param pageAddr Page address. + */ public long getTombstonesCount(long pageAddr) { throw new UnsupportedOperationException("Tombstones count is not supported by " + "this PagePartitionMetaIO version: ver=" + getVersion()); } + /** + * @param pageAddr Page address. + * @param tombstonesCount Tombstones count. + */ public boolean setTombstonesCount(long pageAddr, long tombstonesCount) { throw new UnsupportedOperationException("Tombstones count is not supported by " + "this PagePartitionMetaIO version: ver=" + getVersion()); From 81eab425914060e4d665b4dbedf2e4310fe0ae42 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 3 Oct 2019 13:25:31 +0300 Subject: [PATCH 25/30] IGNITE-11704 Fixed imports. --- .../util/GridCommandHandlerIndexingTest.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java index cb1451cc9954d..2050a11e15f74 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java @@ -20,12 +20,32 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; +import javax.cache.Cache; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.QueryIndex; +import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.metric.IoStatisticsHolderNoOp; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.tree.SearchRow; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; +import org.apache.ignite.internal.util.lang.GridIterator; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; import org.junit.Test; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; @@ -234,7 +254,7 @@ private void breakSqlIndex(Ignite ig, String cacheName) throws Exception { GridDhtLocalPartition locPart = ctx.topology().localPartitions().get(0); - GridIterator it = ctx.group().offheap().partitionIterator(locPart.id(), false); + GridIterator it = ctx.group().offheap().partitionIterator(locPart.id()); for (int i = 0; i < 500; i++) { if (!it.hasNextX()) { From 12baf1b1a038ed1dd9bbd5f511825a8bb4762220 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Thu, 3 Oct 2019 18:44:29 +0300 Subject: [PATCH 26/30] IGNITE-11704 Remove assertion during remove with tombstone and tests from MVCC. --- .../processors/cache/GridCacheMapEntry.java | 4 ++- .../CacheRemoveWithTombstonesTest.java | 31 +++++++------------ .../testsuites/IgniteCacheMvccTestSuite9.java | 4 +++ 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 1fed5ddf4b041..12198958e7a6e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -1720,7 +1720,9 @@ protected Object keyValue(boolean cpy) { if (cctx.group().shouldCreateTombstone(localPartition())) { cctx.offheap().removeWithTombstone(cctx, key, newVer, localPartition()); - assert cctx.group().shouldCreateTombstone(localPartition()) : "Partition state is changed: " + localPartition(); + // Partition may change his state during remove. + if (!cctx.group().shouldCreateTombstone(localPartition())) + removeTombstone0(newVer); } else removeValue(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index 1b6ca3fecb5ff..c6a550cf14808 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -41,7 +41,6 @@ import org.apache.ignite.internal.processors.metric.impl.MetricUtils; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.After; import org.junit.Before; @@ -165,16 +164,13 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea if (persistence) ignite0.cluster().active(true); - IgniteCache cache0 = ignite0.createCache(cacheConfiguration(atomicityMode)); + IgniteCache cache0 = ignite0.createCache(cacheConfiguration(atomicityMode)); - if (MvccFeatureChecker.forcedMvcc()) - expTombstone = false; - - final int KEYS = 1024 * 64; + final int KEYS = 1024 * 256; if (histRebalance) { // Preload initial data to have start point for WAL rebalance. - try (IgniteDataStreamer streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) { + try (IgniteDataStreamer streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) { streamer.allowOverwrite(true); for (int i = 0; i < KEYS; i++) @@ -186,7 +182,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea stopGrid(1); } - try (IgniteDataStreamer streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) { + try (IgniteDataStreamer streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) { streamer.allowOverwrite(true); for (int i = 0; i < KEYS; i++) @@ -279,16 +275,13 @@ private static void blockRebalance(IgniteEx node) { * @param atomicityMode Cache atomicity mode. * @return Cache configuration. */ - private CacheConfiguration cacheConfiguration(CacheAtomicityMode atomicityMode) { - CacheConfiguration ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME); - - ccfg.setAtomicityMode(atomicityMode); - ccfg.setCacheMode(PARTITIONED); - ccfg.setBackups(2); - ccfg.setRebalanceMode(ASYNC); - ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - ccfg.setAffinity(new RendezvousAffinityFunction(false, 64)); - - return ccfg; + private CacheConfiguration cacheConfiguration(CacheAtomicityMode atomicityMode) { + return new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setAtomicityMode(atomicityMode) + .setCacheMode(PARTITIONED) + .setBackups(2) + .setRebalanceMode(ASYNC) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setAffinity(new RendezvousAffinityFunction(false, 64)); } } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java index e5dff20575252..ff1e1ec6281eb 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java @@ -26,9 +26,11 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheLoadRebalanceEvictionSelfTest; import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest; import org.apache.ignite.internal.processors.cache.distributed.CacheRemoveWithTombstonesLoadTest; +import org.apache.ignite.internal.processors.cache.distributed.CacheRemoveWithTombstonesTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCachePrimarySyncTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteTxConcurrentRemoveObjectsTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.CacheRemoveWithTombstonesFailoverTest; import org.apache.ignite.internal.processors.cache.transactions.TxPartitionCounterStateConsistencyHistoryRebalanceTest; import org.apache.ignite.internal.metric.IoStatisticsCachePersistenceSelfTest; import org.apache.ignite.internal.metric.IoStatisticsCacheSelfTest; @@ -90,7 +92,9 @@ public static List> suite() { ignoredTests.add(IoStatisticsCachePersistenceSelfTest.class); // Tombstones are not created with mvcc. + ignoredTests.add(CacheRemoveWithTombstonesTest.class); ignoredTests.add(CacheRemoveWithTombstonesLoadTest.class); + ignoredTests.add(CacheRemoveWithTombstonesFailoverTest.class); return new ArrayList<>(IgniteCacheTestSuite9.suite(ignoredTests)); } From 027370dead6860fdde2f779e7e97fda08393fdec Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Mon, 14 Oct 2019 16:59:08 +0300 Subject: [PATCH 27/30] IGNITE-11704 Track suspicious update version in initialValue. --- .../ignite/internal/processors/cache/GridCacheMapEntry.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 12198958e7a6e..bee835320d273 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3391,6 +3391,12 @@ private boolean skipInterceptor(@Nullable GridCacheVersion explicitVer) { update0 |= (!preload && val == null); + if (!preload && val == null) { + U.dumpStack(log, "Suspsicious operation."); + + assert false; + } + return update0; } }; From d083b954d2918570a198f1e1c094247188256775 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Tue, 15 Oct 2019 16:27:43 +0300 Subject: [PATCH 28/30] IGNITE-11704 Tombstone storing is optimized. --- .../communication/GridIoMessageFactory.java | 6 ++ .../ignite/internal/pagemem/PageUtils.java | 4 +- .../processors/cache/CacheObject.java | 3 + .../processors/cache/GridCacheMapEntry.java | 7 +- .../cache/IgniteCacheOffheapManagerImpl.java | 2 +- .../cache/IncompleteCacheObject.java | 12 +++ .../cache/TombstoneCacheObject.java | 94 +++++++++++++++++++ .../CacheObjectBinaryProcessorImpl.java | 6 +- .../IgniteCacheDatabaseSharedManager.java | 85 ++--------------- 9 files changed, 134 insertions(+), 85 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TombstoneCacheObject.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java index d8d62d4595a96..b5ae4f63135e4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java @@ -53,6 +53,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheReturn; import org.apache.ignite.internal.processors.cache.GridChangeGlobalStateMessageResponse; import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl; +import org.apache.ignite.internal.processors.cache.TombstoneCacheObject; import org.apache.ignite.internal.processors.cache.WalStateAckMessage; import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage; import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage; @@ -1166,6 +1167,11 @@ public GridIoMessageFactory(MessageFactory[] ext) { break; + case 176: + msg = TombstoneCacheObject.INSTANCE; + + break; + // [-3..119] [124..129] [-23..-28] [-36..-55] - this // [120..123] - DR // [-4..-22, -30..-35] - SQL diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageUtils.java index 217164cf39790..0b9b1b47029d4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageUtils.java @@ -56,8 +56,8 @@ public static int getUnsignedByte(long addr, int off) { */ public static byte[] getBytes(long addr, int off, int len) { assert addr > 0 : addr; - assert off >= 0; - assert len >= 0; + assert off >= 0 : off; + assert len >= 0 : len; byte[] bytes = new byte[len]; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java index f9f384a7f9702..5e89926f621fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java @@ -38,6 +38,9 @@ public interface CacheObject extends Message { /** */ public static final byte TYPE_BINARY_ENUM = 101; + /** */ + public static final byte TOMBSTONE = -1; + /** * @param ctx Context. * @param cpy If {@code true} need to copy value. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index bee835320d273..3be252a768ffb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3389,14 +3389,9 @@ private boolean skipInterceptor(@Nullable GridCacheVersion explicitVer) { else update0 = isStartVer; + // Such combination may exist during datastreamer udpates. update0 |= (!preload && val == null); - if (!preload && val == null) { - U.dumpStack(log, "Suspsicious operation."); - - assert false; - } - return update0; } }; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 2dec9ee0fd326..85b3654b4f08b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -2765,7 +2765,7 @@ private class RemoveWithTombstone implements IgniteCacheOffheapManager.OffheapIn this.oldRow = oldRow; - newRow = createRow(cctx, key, cctx.shared().database().tombstoneValue(), ver, 0, oldRow); + newRow = createRow(cctx, key, TombstoneCacheObject.INSTANCE, ver, 0, oldRow); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java index 6b21145ba7a12..3802c2b6c72d1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IncompleteCacheObject.java @@ -43,6 +43,8 @@ public IncompleteCacheObject(final ByteBuffer buf) { if (buf.remaining() >= HEAD_LEN) { data = new byte[buf.getInt()]; type = buf.get(); + + headerReady(); } // We cannot fully read head to initialize data buffer. // Start partial read of header. @@ -68,6 +70,8 @@ public IncompleteCacheObject(final ByteBuffer buf) { data = new byte[headBuf.getInt()]; type = headBuf.get(); + + headerReady(); } } @@ -75,6 +79,14 @@ public IncompleteCacheObject(final ByteBuffer buf) { super.readData(buf); } + /** + * Invoke when object header is ready. + */ + private void headerReady() { + if (type == CacheObject.TOMBSTONE) + object(TombstoneCacheObject.INSTANCE); + } + /** * @return Size of already read data. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TombstoneCacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TombstoneCacheObject.java new file mode 100644 index 0000000000000..d87b0245a6412 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TombstoneCacheObject.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.io.IOException; +import java.io.ObjectInput; +import org.apache.ignite.IgniteCheckedException; +import org.jetbrains.annotations.Nullable; + +/** + * Special value object indicating that value is removed. + */ +public class TombstoneCacheObject extends CacheObjectAdapter { + /** */ + private static final long serialVersionUID = 2106775575127797257L; + + /** Empty. */ + private static final byte[] EMPTY = new byte[] { }; + + /** Instance. */ + public static final TombstoneCacheObject INSTANCE = new TombstoneCacheObject(); + + /** + * Default constructor. + */ + public TombstoneCacheObject() { + valBytes = EMPTY; + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + valBytes = EMPTY; + } + + /** {@inheritDoc} */ + @Override public @Nullable T value(CacheObjectValueContext ctx, boolean cpy) { + return null; + } + + /** {@inheritDoc} */ + @Override public byte[] valueBytes(CacheObjectValueContext ctx) throws IgniteCheckedException { + return valBytes; + } + + /** {@inheritDoc} */ + @Override public byte cacheObjectType() { + return CacheObject.TOMBSTONE; + } + + /** {@inheritDoc} */ + @Override public boolean isPlatformType() { + return true; + } + + /** {@inheritDoc} */ + @Override public CacheObject prepareForCache(CacheObjectContext ctx) { + return this; + } + + /** {@inheritDoc} */ + @Override public void finishUnmarshal(CacheObjectValueContext ctx, ClassLoader ldr) throws IgniteCheckedException { + + } + + /** {@inheritDoc} */ + @Override public void prepareMarshal(CacheObjectValueContext ctx) throws IgniteCheckedException { + + } + + /** {@inheritDoc} */ + @Override public short directType() { + return 176; + } + + /** {@inheritDoc} */ + @Override public void onAckReceived() { + + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java index ac2d237562535..8c8a3efe5ee2e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java @@ -17,7 +17,6 @@ package org.apache.ignite.internal.processors.cache.binary; -import javax.cache.CacheException; import java.io.File; import java.io.Serializable; import java.math.BigDecimal; @@ -32,6 +31,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import javax.cache.CacheException; import org.apache.ignite.IgniteBinary; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteClientDisconnectedException; @@ -83,6 +83,7 @@ import org.apache.ignite.internal.processors.cache.IncompleteCacheObject; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl; +import org.apache.ignite.internal.processors.cache.TombstoneCacheObject; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.processors.cacheobject.UserCacheObjectByteArrayImpl; @@ -1145,6 +1146,9 @@ private int partition(CacheObjectContext ctx, @Nullable GridCacheContext cctx, O case CacheObject.TYPE_REGULAR: return new CacheObjectImpl(null, bytes); + + case CacheObject.TOMBSTONE: + return TombstoneCacheObject.INSTANCE; } throw new IllegalArgumentException("Invalid object type: " + type); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java index c4efadcc6f491..ba4031f8d6b1f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java @@ -20,7 +20,6 @@ import java.io.File; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -56,7 +55,6 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheObject; -import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.GridCacheMapEntry; import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter; import org.apache.ignite.internal.processors.cache.IncompleteCacheObject; @@ -143,12 +141,6 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap /** First eviction was warned flag. */ private volatile boolean firstEvictWarn; - /** */ - private byte[] tombstoneBytes; - - /** */ - private CacheObject tombstoneVal; - /** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getDataStorageConfiguration() == null) @@ -163,17 +155,6 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap pageSize = memCfg.getPageSize(); initDataRegions(memCfg); - - tombstoneBytes = cctx.marshaller().marshal(null); - - tombstoneVal = new CacheObjectImpl(null, tombstoneBytes); - } - - /** - * @return Value to be stored for removed entry. - */ - public CacheObject tombstoneValue() { - return tombstoneVal; } /** @@ -189,13 +170,7 @@ public boolean isTombstone(@Nullable CacheDataRow row) throws IgniteCheckedExcep assert val != null : row; - if (val.cacheObjectType() == CacheObject.TYPE_REGULAR) { - byte[] bytes = val.valueBytes(null); - - return Arrays.equals(tombstoneBytes, bytes); - } - - return false; + return val.cacheObjectType() == CacheObject.TOMBSTONE; } /** @@ -204,14 +179,15 @@ public boolean isTombstone(@Nullable CacheDataRow row) throws IgniteCheckedExcep * @param incomplete Incomplete object. * @return Tombstone flag or {@code null} if there is no enough data. */ - public Boolean isTombstone(ByteBuffer buf, + public Boolean isTombstone( + ByteBuffer buf, @Nullable KeyCacheObject key, - @Nullable IncompleteCacheObject incomplete) { + @Nullable IncompleteCacheObject incomplete + ) { if (key == null) { if (incomplete == null) { // Did not start read key yet. - if (buf.remaining() < IncompleteCacheObject.HEAD_LEN) { + if (buf.remaining() < IncompleteCacheObject.HEAD_LEN) return null; - } int keySize = buf.getInt(buf.position()); @@ -253,15 +229,7 @@ public Boolean isTombstone(ByteBuffer buf, return isTombstone(buf, 0); } - byte[] data = incomplete.data(); - - if (data == null) // Header is not available yet. - return null; - - if (incomplete.type() != CacheObject.TYPE_REGULAR || data.length != tombstoneBytes.length) - return Boolean.FALSE; - - return null; + return incomplete.type() == CacheObject.TOMBSTONE; } /** @@ -270,23 +238,9 @@ public Boolean isTombstone(ByteBuffer buf, * @return Tombstone flag or {@code null} if there is no enough data. */ private Boolean isTombstone(ByteBuffer buf, int offset) { - int valLen = buf.getInt(buf.position() + offset); - if (valLen != tombstoneBytes.length) - return Boolean.FALSE; - byte valType = buf.get(buf.position() + offset + 4); - if (valType != CacheObject.TYPE_REGULAR) - return Boolean.FALSE; - - if (buf.remaining() < (offset + 5 + tombstoneBytes.length)) - return null; - for (int i = 0; i < tombstoneBytes.length; i++) { - if (tombstoneBytes[i] != buf.get(buf.position() + offset + 5 + i)) - return Boolean.FALSE; - } - - return Boolean.TRUE; + return valType == CacheObject.TOMBSTONE; } /** @@ -294,28 +248,9 @@ private Boolean isTombstone(ByteBuffer buf, int offset) { * @return {@code True} if stored value is tombstone. */ public boolean isTombstone(long addr) { - int off = 0; - - byte type = PageUtils.getByte(addr, off + 4); - - if (type != CacheObject.TYPE_REGULAR) - return false; + byte type = PageUtils.getByte(addr, 4); - int len = PageUtils.getInt(addr, off); - - if (len != tombstoneBytes.length) - return false; - - off += 5; - - for (int i = 0; i < len; i++) { - byte b = PageUtils.getByte(addr, off++); - - if (tombstoneBytes[i] != b) - return false; - } - - return true; + return type == CacheObject.TOMBSTONE; } /** From 919745d542698905038b089877224aaf0db4a338 Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Tue, 15 Oct 2019 17:32:04 +0300 Subject: [PATCH 29/30] IGNITE-11704 Avoid code duplication in clearAll and clearTombstones. --- .../dht/topology/GridDhtLocalPartition.java | 307 +++++++++--------- 1 file changed, 160 insertions(+), 147 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index 37aa0f37a6458..cd61492d15d7b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -65,6 +65,7 @@ import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.lang.GridCursor; import org.apache.ignite.internal.util.lang.GridIterator; +import org.apache.ignite.internal.util.lang.GridIteratorAdapter; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.S; @@ -943,7 +944,7 @@ public boolean isClearing() { * @return {@code false} if clearing is not started due to existing reservations. * @throws NodeStoppingException If node is stopping. */ - public boolean tryClear(EvictionContext evictionCtx) throws NodeStoppingException { + public boolean tryClear(EvictionContext evictionCtx) throws NodeStoppingException, IgniteCheckedException { if (clearFuture.isDone()) return true; @@ -954,8 +955,73 @@ public boolean tryClear(EvictionContext evictionCtx) throws NodeStoppingExceptio if (addEvicting()) { try { + GridCacheVersion clearVer = ctx.versions().next(); + + GridCacheObsoleteEntryExtras extras = new GridCacheObsoleteEntryExtras(clearVer); + + boolean rec = grp.eventRecordable(EVT_CACHE_REBALANCE_OBJECT_UNLOADED); + + if (grp.sharedGroup()) + cacheMaps.forEach((key, hld) -> clearOnheapEntries(hld.map, extras, rec)); + else + clearOnheapEntries(singleCacheEntryMap.map, extras, rec); + // Attempt to evict partition entries from cache. - long clearedEntities = clearAll(evictionCtx); + long clearedEntities = doClear( + evictionCtx, + 1000, + grp.offheap().partitionIterator(id, true), + (hld, row) -> { + // Do not clear fresh rows in case of partition reloading. + // This is required because normal updates are possible to moving partition which is currently cleared. + if (row.version().compareTo(clearVer) >= 0 && state() == MOVING) + return false; + + GridCacheMapEntry cached = putEntryIfObsoleteOrAbsent( + hld, + hld.cctx, + grp.affinity().lastVersion(), + row.key(), + true, + false); + + if (cached instanceof GridDhtCacheEntry && ((GridDhtCacheEntry)cached).clearInternal(clearVer, extras)) { + removeEntry(cached); + + if (rec && !hld.cctx.config().isEventsDisabled()) { + hld.cctx.events().addEvent(cached.partition(), + cached.key(), + ctx.localNodeId(), + null, + null, + null, + EVT_CACHE_REBALANCE_OBJECT_UNLOADED, + null, + false, + cached.rawGet(), + cached.hasValue(), + null, + null, + null, + false); + } + + return true; + } + + return false; + } + ); + + if (forceTestCheckpointOnEviction) { + if (partWhereTestCheckpointEnforced == null && clearedEntities >= fullSize()) { + ctx.database().forceCheckpoint("test").finishFuture().get(); + + log.warning("Forced checkpoint by test reasons for partition: " + this); + + partWhereTestCheckpointEnforced = id; + } + } if (log.isDebugEnabled()) log.debug("Partition has been cleared [grp=" + grp.cacheOrGroupName() @@ -1146,182 +1212,115 @@ public long fullSize() { * * @param evictionCtx Eviction context. */ - void clearTombstones(EvictionContext evictionCtx) { + void clearTombstones(EvictionContext evictionCtx) throws IgniteCheckedException { if (evictionCtx.shouldStop()) return; - final int stopCheckingFreq = 1000; - - CacheMapHolder hld = grp.sharedGroup() ? null : singleCacheEntryMap; + GridIterator iter; try { GridCursor cur = store.cursor(CacheDataRowAdapter.RowData.TOMBSTONES); - int cntr = 0; - - while (cur.next()) { - CacheDataRow row = cur.get(); - - assert row.key() != null; - assert row.version() != null; + iter = new GridIteratorAdapter() { + @Override public boolean hasNextX() throws IgniteCheckedException { + return cur.next(); + } - if (grp.sharedGroup() && (hld == null || hld.cctx.cacheId() != row.cacheId())) - hld = cacheMapHolder(ctx.cacheContext(row.cacheId())); + @Override public CacheDataRow nextX() throws IgniteCheckedException { + return cur.get(); + } - assert hld != null; + @Override public void removeX() throws IgniteCheckedException { + throw new UnsupportedOperationException(); + } + }; + } + catch (IgniteCheckedException e) { + throw new IgniteCheckedException("Failed to get iterator for partition: " + id, e); + } - ctx.database().checkpointReadLock(); + doClear( + evictionCtx, + 10, + iter, + (hld, row) -> { + while (true) { + GridCacheMapEntry cached = null; - try { - while (true) { - GridCacheMapEntry cached = null; - - try { - cached = putEntryIfObsoleteOrAbsent( - hld, - hld.cctx, - grp.affinity().lastVersion(), - row.key(), - true, - false); + try { + cached = putEntryIfObsoleteOrAbsent( + hld, + hld.cctx, + grp.affinity().lastVersion(), + row.key(), + true, + false); - cached.removeTombstone(row.version()); + cached.removeTombstone(row.version()); - break; - } - catch (GridCacheEntryRemovedException e) { - cached = null; - } - finally { - if (cached != null) - cached.touch(); - } + return true; + } + catch (GridCacheEntryRemovedException e) { + cached = null; + } + finally { + if (cached != null) + cached.touch(); } - } - finally { - ctx.database().checkpointReadUnlock(); - } - - cntr++; - - if (cntr % stopCheckingFreq == 0) { - if (evictionCtx.shouldStop() || state() != OWNING) - break; } } - } - catch (IgniteCheckedException e) { - U.error(log, "Failed clear tombstone entries for partition: " + id, e); - } + ); } /** - * Removes all entries and rows from this partition. + * Runs abstract clear operation over partition data rows. * - * @return Number of rows cleared from page memory. - * @throws NodeStoppingException If node stopping. - */ - private long clearAll(EvictionContext evictionCtx) throws NodeStoppingException { - GridCacheVersion clearVer = ctx.versions().next(); - - GridCacheObsoleteEntryExtras extras = new GridCacheObsoleteEntryExtras(clearVer); - - boolean rec = grp.eventRecordable(EVT_CACHE_REBALANCE_OBJECT_UNLOADED); - - if (grp.sharedGroup()) - cacheMaps.forEach((key, hld) -> clear(hld.map, extras, rec)); - else - clear(singleCacheEntryMap.map, extras, rec); - + * @param evictionCtx Eviction context. + * @param stopCheckingFreq Frequency to check stopping eviction/clearing. + * @param rowIter Rows iterator. + * @param clearOp Clear operation. + * @return Number of cleared rows. + * @throws IgniteCheckedException If failed. + */ + private long doClear( + EvictionContext evictionCtx, + int stopCheckingFreq, + GridIterator rowIter, + ClearRowOperation clearOp + ) throws IgniteCheckedException { long cleared = 0; - final int stopCheckingFreq = 1000; - CacheMapHolder hld = grp.sharedGroup() ? null : singleCacheEntryMap; - try { - GridIterator it0 = grp.offheap().partitionIterator(id, true); - - while (it0.hasNext()) { - ctx.database().checkpointReadLock(); - - try { - CacheDataRow row = it0.next(); - - // Do not clear fresh rows in case of partition reloading. - // This is required because normal updates are possible to moving partition which is currently cleared. - if (row.version().compareTo(clearVer) >= 0 && state() == MOVING) - continue; - - if (grp.sharedGroup() && (hld == null || hld.cctx.cacheId() != row.cacheId())) - hld = cacheMapHolder(ctx.cacheContext(row.cacheId())); - - assert hld != null; + while (rowIter.hasNext()) { + ctx.database().checkpointReadLock(); - GridCacheMapEntry cached = putEntryIfObsoleteOrAbsent( - hld, - hld.cctx, - grp.affinity().lastVersion(), - row.key(), - true, - false); + try { + CacheDataRow row = rowIter.next(); - if (cached instanceof GridDhtCacheEntry && ((GridDhtCacheEntry)cached).clearInternal(clearVer, extras)) { - removeEntry(cached); + assert row.key() != null : row; + assert row.version() != null : row; - if (rec && !hld.cctx.config().isEventsDisabled()) { - hld.cctx.events().addEvent(cached.partition(), - cached.key(), - ctx.localNodeId(), - null, - null, - null, - EVT_CACHE_REBALANCE_OBJECT_UNLOADED, - null, - false, - cached.rawGet(), - cached.hasValue(), - null, - null, - null, - false); - } + if (grp.sharedGroup() && (hld == null || hld.cctx.cacheId() != row.cacheId())) + hld = cacheMapHolder(ctx.cacheContext(row.cacheId())); - cleared++; - } + assert hld != null; - // For each 'stopCheckingFreq' cleared entities check clearing process to stop. - if (cleared % stopCheckingFreq == 0 && evictionCtx.shouldStop()) - return cleared; - } - catch (GridDhtInvalidPartitionException e) { - assert isEmpty() && state() == EVICTED : "Invalid error [e=" + e + ", part=" + this + ']'; + if (clearOp.apply(hld, row)) + cleared++; - break; // Partition is already concurrently cleared and evicted. - } - finally { - ctx.database().checkpointReadUnlock(); - } + // For each 'stopCheckingFreq' cleared entities check clearing process to stop. + if (cleared % stopCheckingFreq == 0 && evictionCtx.shouldStop()) + return cleared; } + catch (GridDhtInvalidPartitionException e) { + assert isEmpty() && state() == EVICTED : "Invalid error [e=" + e + ", part=" + this + ']'; - if (forceTestCheckpointOnEviction) { - if (partWhereTestCheckpointEnforced == null && cleared >= fullSize()) { - ctx.database().forceCheckpoint("test").finishFuture().get(); - - log.warning("Forced checkpoint by test reasons for partition: " + this); - - partWhereTestCheckpointEnforced = id; - } + break; // Partition is already concurrently cleared and evicted. + } + finally { + ctx.database().checkpointReadUnlock(); } - } - catch (NodeStoppingException e) { - if (log.isDebugEnabled()) - log.debug("Failed to get iterator for evicted partition: " + id); - - throw e; - } - catch (IgniteCheckedException e) { - U.error(log, "Failed to get iterator for evicted partition: " + id, e); } return cleared; @@ -1335,9 +1334,11 @@ private long clearAll(EvictionContext evictionCtx) throws NodeStoppingException * @param evt Unload event flag. * @throws NodeStoppingException If current node is stopping. */ - private void clear(ConcurrentMap map, + private void clearOnheapEntries( + ConcurrentMap map, GridCacheObsoleteEntryExtras extras, - boolean evt) throws NodeStoppingException { + boolean evt + ) throws NodeStoppingException { Iterator it = map.values().iterator(); while (it.hasNext()) { @@ -1629,6 +1630,18 @@ long expireTime() { } } + /** + * Abstract operation to clear row. + */ + @FunctionalInterface + private static interface ClearRowOperation { + /** + * @param hld Hld. + * @param row Row. + */ + boolean apply(CacheMapHolder hld, CacheDataRow row) throws IgniteCheckedException; + } + /** * Future is needed to control partition clearing process. * Future can be used both for single clearing or eviction processes. From 0babc69b1d15a31aea65786fde7658c89ae2a9ac Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 16 Oct 2019 13:52:19 +0300 Subject: [PATCH 30/30] IGNITE-11704 Enhanced tombstone with historical rebalance test. --- .../internal/processors/cache/GridCacheMapEntry.java | 2 +- .../cache/distributed/CacheRemoveWithTombstonesTest.java | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 3be252a768ffb..6936b980434bb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -3389,7 +3389,7 @@ private boolean skipInterceptor(@Nullable GridCacheVersion explicitVer) { else update0 = isStartVer; - // Such combination may exist during datastreamer udpates. + // Such combination may exist during datastreamer first update. update0 |= (!preload && val == null); return update0; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index c6a550cf14808..c8b0a7f56f28d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -166,7 +166,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea IgniteCache cache0 = ignite0.createCache(cacheConfiguration(atomicityMode)); - final int KEYS = 1024 * 256; + final int KEYS = histRebalance ? 1024 : 1024 * 256; if (histRebalance) { // Preload initial data to have start point for WAL rebalance. @@ -174,7 +174,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea streamer.allowOverwrite(true); for (int i = 0; i < KEYS; i++) - streamer.addData(i, 0); + streamer.addData(-i, 0); } forceCheckpoint(); @@ -182,6 +182,7 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea stopGrid(1); } + // This data will be rebalanced. try (IgniteDataStreamer streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) { streamer.allowOverwrite(true); @@ -204,7 +205,8 @@ private void testRemoveAndRebalanceRace(CacheAtomicityMode atomicityMode, boolea Set keysWithTombstone = new HashSet<>(); // Do removes while rebalance is in progress. - for (int i = 0; i < KEYS; i += 64) { + // All keys are removed during historical rebalance. + for (int i = 0, step = histRebalance ? 1 : 64; i < KEYS; i += step) { keysWithTombstone.add(i); cache0.remove(i);