IGNITE-7964 rmvId is stored to MetaStorage metapage during operations

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java

@@ -553,7 +553,7 @@ private void readMetastore() throws IgniteCheckedException {
             try {
                 restoreMemory(status, true, storePageMem);
 
-                metaStorage = new MetaStorage(cctx.wal(), regCfg, memMetrics, true);
+                metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true);
 
                 metaStorage.init(this);
 
@@ -724,7 +724,7 @@ private void unRegistrateMetricsMBean() {
 
             cctx.pageStore().initializeForMetastorage();
 
-            metaStorage = new MetaStorage(cctx.wal(), dataRegionMap.get(METASTORE_DATA_REGION_NAME),
+            metaStorage = new MetaStorage(cctx, dataRegionMap.get(METASTORE_DATA_REGION_NAME),
                 (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME));
 
             WALPointer restore = restoreMemory(status);

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java

@@ -24,13 +24,15 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.IncompleteObject;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
@@ -78,6 +80,9 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     /** */
     private final DataRegion dataRegion;
 
+    /** */
+    private final IgniteLogger log;
+
     /** */
     private MetastorageTree tree;
 
@@ -109,17 +114,18 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     private final Marshaller marshaller = new JdkMarshaller();
 
     /** */
-    public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
+    public MetaStorage(GridCacheSharedContext cctx, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
         boolean readOnly) {
-        this.wal = wal;
+        wal = cctx.wal();
         this.dataRegion = dataRegion;
         this.regionMetrics = regionMetrics;
         this.readOnly = readOnly;
+        log = cctx.logger(getClass());
     }
 
     /** */
-    public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
-        this(wal, memPlc, memMetrics, false);
+    public MetaStorage(GridCacheSharedContext cctx, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
+        this(cctx, memPlc, memMetrics, false);
     }
 
     /** */
@@ -309,6 +315,8 @@ private void getOrAllocateMetas() throws IgniteCheckedException {
 
                     treeRoot = new RootPage(new FullPageId(io.getTreeRoot(pageAddr), METASTORAGE_CACHE_ID), false);
                     reuseListRoot = new RootPage(new FullPageId(io.getReuseListRoot(pageAddr), METASTORAGE_CACHE_ID), false);
+
+                    rmvId.set(io.getGlobalRemoveId(pageAddr));
                 }
                 finally {
                     pageMem.readUnlock(METASTORAGE_CACHE_ID, partId, partMetaPage);
@@ -354,6 +362,8 @@ private void getOrAllocateMetas() throws IgniteCheckedException {
                         treeRoot = io.getTreeRoot(pageAddr);
                         reuseListRoot = io.getReuseListRoot(pageAddr);
 
+                        rmvId.set(io.getGlobalRemoveId(pageAddr));
+
                         assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA :
                             U.hexLong(treeRoot) + ", part=" + partId + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID;
                         assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA :
@@ -384,21 +394,46 @@ public PageMemory pageMemory() {
     @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
         freeList.saveMetadata();
 
-        MetastorageRowStore rowStore = tree.rowStore();
-
-        saveStoreMetadata(rowStore, ctx);
+        saveStoreMetadata();
     }
 
     /**
-     * @param rowStore Store to save metadata.
      * @throws IgniteCheckedException If failed.
      */
-    private void saveStoreMetadata(MetastorageRowStore rowStore, Context ctx) throws IgniteCheckedException {
-        FreeListImpl freeList = (FreeListImpl)rowStore.freeList();
+    private void saveStoreMetadata() throws IgniteCheckedException {
+        PageMemoryEx pageMem = (PageMemoryEx) pageMemory();
 
-        freeList.saveMetadata();
+        int partId = 0;
+
+        long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId);
+        long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId);
+
+        try {
+            long partMetaPageAddr = pageMem.writeLock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+
+            if (partMetaPageAddr == 0L) {
+                U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage + ']');
+
+                return;
+            }
+
+            boolean changed = false;
+
+            try {
+                PagePartitionMetaIO io = PageIO.getPageIO(partMetaPageAddr);
+
+                changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId.get());
+            }
+            finally {
+                pageMem.writeUnlock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage, null, changed);
+            }
+        }
+        finally {
+            pageMem.releasePage(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+        }
     }
 
+    /** */
     public void applyUpdate(String key, byte[] value) throws IgniteCheckedException {
         if (readOnly) {
             if (lastUpdates == null)

modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java

@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ * Single place to add for basic MetaStorage tests.
+ */
+public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        DataStorageConfiguration storageCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(100 * 1024 * 1024)
+                    .setPersistenceEnabled(true)
+            )
+            .setWalMode(WALMode.LOG_ONLY);
+
+        cfg.setDataStorageConfiguration(storageCfg);
+
+        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+        discoSpi.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(discoSpi);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Verifies that MetaStorage after massive amounts of keys stored and updated keys restores its state successfully
+     * after restart.
+     *
+     * See <a hfer="https://issues.apache.org/jira/browse/IGNITE-7964" target="_top">IGNITE-7964</a> for more context
+     * about this test.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMetaStorageMassivePutUpdateRestart() throws Exception {
+        IgniteEx ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        final byte KEYS_CNT = 100;
+        final String KEY_PREFIX = "test.key.";
+        final String NEW_VAL_PREFIX = "new.val.";
+        final String UPDATED_VAL_PREFIX = "updated.val.";
+
+        loadKeys(ig, KEYS_CNT, KEY_PREFIX, NEW_VAL_PREFIX, UPDATED_VAL_PREFIX);
+
+        stopGrid(0);
+
+        ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        verifyKeys(ig, KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX);
+    }
+
+    /** */
+    private void loadKeys(IgniteEx ig,
+        byte keysCnt,
+        String keyPrefix,
+        String newValPrefix,
+        String updatedValPrefix
+    ) throws IgniteCheckedException {
+        IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database();
+
+        MetaStorage metaStorage = db.metaStorage();
+
+        db.checkpointReadLock();
+        try {
+            for (byte i = 0; i < keysCnt; i++)
+                metaStorage.write(keyPrefix + i, newValPrefix + i);
+
+            for (byte i = 0; i < keysCnt; i++)
+                metaStorage.write(keyPrefix + i, updatedValPrefix + i);
+        }
+        finally {
+            db.checkpointReadUnlock();
+        }
+    }
+
+    /** */
+    private void verifyKeys(IgniteEx ig,
+        byte keysCnt,
+        String keyPrefix,
+        String valPrefix
+    ) throws IgniteCheckedException {
+        MetaStorage metaStorage = ig.context().cache().context().database().metaStorage();
+
+        for (byte i = 0; i < keysCnt; i++) {
+            Serializable val = metaStorage.read(keyPrefix + i);
+
+            Assert.assertEquals(valPrefix + i, val);
+        }
+    }
+}

modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java

@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.file.DefaultPageSizeBackwardsCompatibilityTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimulationWithRealCpDisabledTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsPageReplacementTest;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.IgniteMetaStorageBasicTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreePageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreeReuseListPageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.FillFactorMetricTest;
@@ -114,5 +115,8 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class);
 
         suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
+
+        //MetaStorage
+        suite.addTestSuite(IgniteMetaStorageBasicTest.class);
     }
 }