From 69f6404a05473221e3dba0c13990e521a19c61f9 Mon Sep 17 00:00:00 2001 From: Alexey Stelmak Date: Tue, 13 Nov 2018 19:12:12 +0300 Subject: [PATCH 1/2] Metasotage pages fix --- .../internal/pagemem/PageIdAllocator.java | 3 + .../freelist/AbstractFreeList.java | 7 +- .../persistence/metastorage/MetaStorage.java | 7 ++ .../metastorage/MetastorageTree.java | 9 +++ .../IgniteMetaStorageBasicTest.java | 78 +++++++++++++++++++ 5 files changed, 103 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java index c6aeabe087975..b3e4b07b9fb67 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java @@ -35,6 +35,9 @@ public interface PageIdAllocator { /** Special partition reserved for index space. */ public static final int INDEX_PARTITION = 0xFFFF; + /** Special partition reserved for metastore space. */ + public static final int METASTORE_PARTITION = 0x0; + /** * Allocates a page from the space for the given partition ID and the given flags. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 4e1f783b2c287..04cd656c1e7af 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -144,7 +144,12 @@ private final class WriteRowHandler extends PageHandler { T row, int written) throws IgniteCheckedException { - AbstractDataPageIO io = (AbstractDataPageIO)iox; + AbstractDataPageIO io; + try { + io = (AbstractDataPageIO)iox; + } catch (ClassCastException e) { + throw e; + } int rowSize = row.size(); int oldFreeSpace = io.getFreeSpace(pageAddr); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java index 4a243aaea005a..91fbec096029d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java @@ -28,6 +28,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.pagemem.FullPageId; +import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; @@ -62,6 +63,7 @@ import org.apache.ignite.marshaller.jdk.JdkMarshaller; import org.jetbrains.annotations.NotNull; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId; import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId; @@ -507,6 +509,11 @@ public static class FreeListImpl extends AbstractFreeList { return SimpleDataPageIO.VERSIONS; } + /** {@inheritDoc} */ + @Override protected long allocatePageNoReuse() throws IgniteCheckedException { + return pageMem.allocatePage(grpId, PageIdAllocator.METASTORE_PARTITION, FLAG_DATA); + } + /** * Read row from data pages. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java index 00db5cd791163..27de6841c9084 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java @@ -19,6 +19,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.PageUtils; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; @@ -31,6 +32,9 @@ import org.apache.ignite.internal.processors.failure.FailureProcessor; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX; + /** * */ @@ -94,6 +98,11 @@ public MetastorageRowStore rowStore() { return rowStore; } + /** {@inheritDoc} */ + @Override protected long allocatePageNoReuse() throws IgniteCheckedException { + return pageMem.allocatePage(grpId, PageIdAllocator.METASTORE_PARTITION, FLAG_DATA); + } + /** * */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java index e5a53fac1833b..b6dfc12826177 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java @@ -17,6 +17,9 @@ package org.apache.ignite.internal.processors.cache.persistence.metastorage; import java.io.Serializable; +import java.util.HashSet; +import java.util.List; +import java.util.Random; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.DataRegionConfiguration; @@ -78,6 +81,81 @@ public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest { cleanPersistenceDir(); } + /** + * + */ + public void testMetaStorageMassivePutFixed() throws Exception { + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database(); + + MetaStorage metaStorage = db.metaStorage(); + + assertNotNull(metaStorage); + + Random rnd = new Random(); + + db.checkpointReadLock(); + + int size; + try { + for (int i = 0; i < 10_000; i++) { + size = rnd.nextBoolean() ? 3500 : 2 * 3500; + String key = "TEST_KEY_" + (i % 1000); + + byte[] arr = new byte[size]; + rnd.nextBytes(arr); + + metaStorage.remove(key); + + metaStorage.putData(key, arr/*b.toString().getBytes()*/); + } + } + finally { + db.checkpointReadUnlock(); + } + } + + /** + * + */ + public void testMetaStorageMassivePutRandom() throws Exception { + IgniteEx ig = startGrid(0); + + ig.cluster().active(true); + + IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database(); + + MetaStorage metaStorage = db.metaStorage(); + + assertNotNull(metaStorage); + + Random rnd = new Random(); + + db.checkpointReadLock(); + + int size; + try { + for (int i = 0; i < 50_000; i++) { + size = 100 + rnd.nextInt(9000); + + String key = "TEST_KEY_" + (i % 2_000); + + byte[] arr = new byte[size]; + rnd.nextBytes(arr); + + metaStorage.remove(key); + + metaStorage.putData(key, arr/*b.toString().getBytes()*/); + } + } + finally { + db.checkpointReadUnlock(); + } + } + /** * Verifies that MetaStorage after massive amounts of keys stored and updated keys restores its state successfully * after restart. From cfd52023a588a66fe89b726f828715c902505c2f Mon Sep 17 00:00:00 2001 From: Alexey Stelmak Date: Thu, 15 Nov 2018 13:01:40 +0300 Subject: [PATCH 2/2] Remove debug code --- .../cache/persistence/freelist/AbstractFreeList.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java index 04cd656c1e7af..4e1f783b2c287 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java @@ -144,12 +144,7 @@ private final class WriteRowHandler extends PageHandler { T row, int written) throws IgniteCheckedException { - AbstractDataPageIO io; - try { - io = (AbstractDataPageIO)iox; - } catch (ClassCastException e) { - throw e; - } + AbstractDataPageIO io = (AbstractDataPageIO)iox; int rowSize = row.size(); int oldFreeSpace = io.getFreeSpace(pageAddr);