From fa0e0c89c2a40d702607e4421682ea4f954e572c Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Mon, 18 Sep 2017 19:22:24 +0300 Subject: [PATCH 1/4] IGNITE-6316 Added test --- .../index/DynamicColumnsPersistenceTest.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsPersistenceTest.java diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsPersistenceTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsPersistenceTest.java new file mode 100644 index 0000000000000..a77c61509a15e --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsPersistenceTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.index; + +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.PersistentStoreConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.query.QueryField; + +/** + * Created by apaschenko on 18.09.17. + */ +public class DynamicColumnsPersistenceTest extends DynamicColumnsAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration() throws Exception { + // Client configuration has SQL cache added to it, therefore it's used a basis. + IgniteConfiguration cfg = clientConfiguration(0).setClientMode(false); + + cfg.setPersistentStoreConfiguration( + new PersistentStoreConfiguration() + .setWalMode(WALMode.LOG_ONLY) + ); + + return cfg; + } + + /** + * Test that dynamically added column is persisted along with the rest of cache configuration. + * @throws Exception + */ + public void testDynamicColumnsMetadataPersistence() throws Exception { + IgniteEx node = startGrid(getConfiguration()); + + node.active(true); + + node.cache("idx"); + + run(node, "ALTER TABLE \"idx\".Person ADD COLUMN age int"); + + doSleep(500); + + QueryField c = c("AGE", Integer.class.getName()); + + checkNodeState(node, "idx", "PERSON", c); + + stopGrid(0); + + node = startGrid(getConfiguration()); + + node.active(true); + + checkNodeState(node, "idx", "PERSON", c); + } +} From 02f03c7c7540a7ff5efecc5461e2bf3b014cfa69 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Tue, 19 Sep 2017 15:16:23 +0300 Subject: [PATCH 2/4] IgnitePersistentStoreSchemaLoadTest fix --- .../IgnitePersistentStoreSchemaLoadTest.java | 100 ++++++++++-------- 1 file changed, 54 insertions(+), 46 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java index 349f062c6945c..87b29b7975904 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java @@ -18,15 +18,12 @@ package org.apache.ignite.internal.processors.database; import java.io.Serializable; -import java.util.Arrays; -import java.util.LinkedHashMap; import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheRebalanceMode; import org.apache.ignite.cache.QueryEntity; -import org.apache.ignite.cache.QueryIndex; -import org.apache.ignite.cache.QueryIndexType; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -36,6 +33,8 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; +import org.apache.ignite.internal.processors.query.QuerySchema; +import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -53,20 +52,15 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest /** */ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); - /** Index name. */ - private static final String IDX_NAME = "my_idx"; - /** Cache name. */ private static final String TMPL_NAME = "test_cache*"; /** Table name. */ private static final String TBL_NAME = Person.class.getSimpleName(); - /** Schema name. */ - private static final String SCHEMA_NAME = "PUBLIC"; - /** Cache name. */ - private static final String CACHE_NAME = TBL_NAME; + private static final String CACHE_NAME = QueryUtils.createTableCacheName(QueryUtils.DFLT_SCHEMA, + TBL_NAME); /** {@inheritDoc} */ @@ -83,6 +77,8 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest cfg.setPersistentStoreConfiguration(pCfg); + cfg.setActiveOnStart(true); + return cfg; } @@ -99,20 +95,6 @@ private CacheConfiguration cacheCfg(String name) { return cfg; } - /** */ - private QueryEntity getEntity() { - LinkedHashMap fields = new LinkedHashMap<>(); - - fields.put("id", Integer.class.getName()); - fields.put("name", String.class.getName()); - - QueryEntity entity = new QueryEntity(Integer.class.getName(), Person.class.getName()); - entity.setFields(fields); - entity.setTableName(TBL_NAME); - - return entity; - } - /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { System.setProperty(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK, "true"); @@ -134,7 +116,6 @@ private QueryEntity getEntity() { /** */ public void testPersistIndex() throws Exception { IgniteEx ig0 = startGrid(0); - startGrid(1); final AtomicInteger cnt = new AtomicInteger(); @@ -146,26 +127,25 @@ public void testPersistIndex() throws Exception { } }); - QueryIndex idx = new QueryIndex("name"); - - idx.setName(IDX_NAME); + ig0.active(true); - ig0.context().query().dynamicTableCreate(SCHEMA_NAME, getEntity(), TMPL_NAME, null, null, null, - null, 1, true); + ig0.context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("create table \"Person\" (\"id\" int primary key, \"name\" varchar)"), false).getAll(); - assert indexCnt(ig0, CACHE_NAME) == 0; + assertEquals(0, indexCnt(ig0, CACHE_NAME)); - ig0.context().query().dynamicIndexCreate(CACHE_NAME, SCHEMA_NAME, TBL_NAME, idx, false).get(); + ig0.context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("Create index \"my_idx\" on \"Person\" (\"name\")"), false).getAll(); - assert indexCnt(ig0, CACHE_NAME) == 1; + assertEquals(1, indexCnt(ig0, CACHE_NAME)); waitForCheckpoint(cnt); - stopGrid(1); + stopGrid(0); - IgniteEx ig1 = startGrid(1); + IgniteEx ig1 = startGrid(0); - assert indexCnt(ig1, CACHE_NAME) == 1; + assertEquals(1, indexCnt(ig1, CACHE_NAME)); } /** */ @@ -173,6 +153,8 @@ public void testPersistCompositeIndex() throws Exception { IgniteEx ig0 = startGrid(0); startGrid(1); + ig0.active(true); + final AtomicInteger cnt = new AtomicInteger(); GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database(); @@ -183,17 +165,20 @@ public void testPersistCompositeIndex() throws Exception { } }); - ig0.context().query().dynamicTableCreate(SCHEMA_NAME, getEntity(), TMPL_NAME, null, null, null, null, 1, true); + ig0.context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("create table \"Person\" (\"id\" int primary key, \"name\" varchar)"), false).getAll(); - assert indexCnt(ig0, CACHE_NAME) == 0; + assertEquals(0, indexCnt(ig0, CACHE_NAME)); - QueryIndex idx = new QueryIndex(Arrays.asList("id", "name"), QueryIndexType.SORTED); + ig0.context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("Create index \"my_idx\" on \"Person\" (\"id\", \"name\")"), false).getAll(); - idx.setName(IDX_NAME); + assertEquals(1, indexCnt(ig0, CACHE_NAME)); - ig0.context().query().dynamicIndexCreate(CACHE_NAME, SCHEMA_NAME, TBL_NAME, idx, false).get(); + ig0.context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("alter table \"Person\" add column \"age\" int"), false).getAll(); - assert indexCnt(ig0, CACHE_NAME) == 1; + assertEquals(3, colsCnt(ig0, CACHE_NAME)); waitForCheckpoint(cnt); @@ -201,7 +186,9 @@ public void testPersistCompositeIndex() throws Exception { IgniteEx ig1 = startGrid(1); - assert indexCnt(ig1, CACHE_NAME) == 1; + assertEquals(1, indexCnt(ig1, CACHE_NAME)); + + assertEquals(3, colsCnt(ig0, CACHE_NAME)); } /** */ @@ -217,13 +204,34 @@ private void waitForCheckpoint(final AtomicInteger cnt) throws IgniteInterrupted /** */ private int indexCnt(IgniteEx node, String cacheName) { + DynamicCacheDescriptor desc = node.context().cache().cacheDescriptor(cacheName); + + int cnt = 0; + if (desc != null) { + QuerySchema schema = desc.schema(); + if (schema != null) { + for (QueryEntity entity : schema.entities()) + cnt += entity.getIndexes().size(); + } + } + return cnt; + } + + /** */ + private int colsCnt(IgniteEx node, String cacheName) { DynamicCacheDescriptor desc = node.context().cache().cacheDescriptor(cacheName); int cnt = 0; - for (QueryEntity entity : desc.schema().entities()) - cnt += entity.getIndexes().size(); + if (desc != null) { + QuerySchema schema = desc.schema(); + if (schema != null) { + + for (QueryEntity entity : schema.entities()) + cnt += entity.getFields().size(); + } + } return cnt; } From 9da99e696085d337d2ddfb9e4e60b3d4ed73bd60 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Tue, 19 Sep 2017 23:03:04 +0300 Subject: [PATCH 3/4] IGNITE-6316 StoredCacheData persistence fix --- .../pagemem/store/IgnitePageStoreManager.java | 4 +- .../cache/CacheAffinitySharedManager.java | 4 +- .../file/FilePageStoreManager.java | 7 +- .../processors/query/GridQueryProcessor.java | 30 ++++++- .../pagemem/NoOpPageStoreManager.java | 2 +- .../index/DynamicColumnsPersistenceTest.java | 70 ----------------- .../IgnitePersistentStoreSchemaLoadTest.java | 78 +++++-------------- 7 files changed, 58 insertions(+), 137 deletions(-) delete mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsPersistenceTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java index eaa85adebe0cb..64c5927c4c2a8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java @@ -20,7 +20,6 @@ import java.nio.ByteBuffer; import java.util.Map; import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; @@ -185,9 +184,10 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac /** * @param cacheData Cache configuration. + * @param overwrite Whether stored configuration should be overwritten if it exists. * @throws IgniteCheckedException If failed. */ - public void storeCacheData(StoredCacheData cacheData) throws IgniteCheckedException; + public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException; /** * @param grpId Cache group ID. * @return {@code True} if index store for given cache group existed before node started. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java index beb7a7dbfea90..741e204449aad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java @@ -2596,8 +2596,8 @@ private void saveCacheConfiguration(CacheConfiguration cfg) { if (cctx.pageStore() != null && cctx.database().persistenceEnabled() && !cctx.kernalContext().clientNode()) { try { cctx.pageStore().storeCacheData( - new StoredCacheData(cfg) - ); + new StoredCacheData(cfg), + false); } catch (IgniteCheckedException e) { U.error(log(), "Error while saving cache configuration on disk, cfg = " + cfg, e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index d60151af2ea5c..b3eb74a770d24 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -207,9 +207,7 @@ public FilePageStoreManager(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public void storeCacheData( - StoredCacheData cacheData - ) throws IgniteCheckedException { + @Override public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException { File cacheWorkDir = cacheWorkDirectory(cacheData.config()); File file; @@ -222,10 +220,11 @@ public FilePageStoreManager(GridKernalContext ctx) { else file = new File(cacheWorkDir, CACHE_DATA_FILENAME); - if (!file.exists() || file.length() == 0) { + if (overwrite || !file.exists() || file.length() == 0) { try { file.createNewFile(); + // Pre-existing file will be truncated upon stream open. try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(file))) { marshaller.marshal(cacheData, stream); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java index 593a3b26fbab6..7a41304b7e567 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java @@ -64,7 +64,9 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.StoredCacheData; import org.apache.ignite.internal.processors.cache.query.CacheQueryFuture; import org.apache.ignite.internal.processors.cache.query.CacheQueryType; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType; @@ -503,8 +505,11 @@ private void onSchemaFinishDiscovery(SchemaFinishDiscoveryMessage msg) { if (!msg.hasError()) { DynamicCacheDescriptor cacheDesc = ctx.cache().cacheDescriptor(msg.operation().cacheName()); - if (cacheDesc != null && F.eq(cacheDesc.deploymentId(), proposeMsg.deploymentId())) + if (cacheDesc != null && F.eq(cacheDesc.deploymentId(), proposeMsg.deploymentId())) { cacheDesc.schemaChangeFinish(msg); + + saveCacheConfiguration(cacheDesc); + } } // Propose message will be used from exchange thread to @@ -2514,6 +2519,29 @@ private void processStatusMessage(SchemaOperationStatusMessage msg) { ", sndNodeId=" + msg.senderNodeId() + ']'); } } + /** + * @param desc cache descriptor. + */ + private void saveCacheConfiguration(DynamicCacheDescriptor desc) { + GridCacheSharedContext cctx = ctx.cache().context(); + + if (cctx.pageStore() != null && cctx.database().persistenceEnabled() && !cctx.kernalContext().clientNode()) { + CacheConfiguration cfg = desc.cacheConfiguration(); + + try { + StoredCacheData data = new StoredCacheData(cfg); + + if (desc.schema() != null) + data.queryEntities(desc.schema().entities()); + + cctx.pageStore().storeCacheData(data, true); + } + catch (IgniteCheckedException e) { + U.error(log, "Error while saving cache configuration on disk, cfg = " + cfg, e); + } + } + } + /** * Unwind pending messages for particular operation. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java index 8fc2bdb98f77d..40887e8c26fe9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java @@ -172,7 +172,7 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager { } /** {@inheritDoc} */ - @Override public void storeCacheData(StoredCacheData cacheData) throws IgniteCheckedException { + @Override public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException { // No-op. } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsPersistenceTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsPersistenceTest.java deleted file mode 100644 index a77c61509a15e..0000000000000 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsPersistenceTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.index; - -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.PersistentStoreConfiguration; -import org.apache.ignite.configuration.WALMode; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.processors.query.QueryField; - -/** - * Created by apaschenko on 18.09.17. - */ -public class DynamicColumnsPersistenceTest extends DynamicColumnsAbstractTest { - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration() throws Exception { - // Client configuration has SQL cache added to it, therefore it's used a basis. - IgniteConfiguration cfg = clientConfiguration(0).setClientMode(false); - - cfg.setPersistentStoreConfiguration( - new PersistentStoreConfiguration() - .setWalMode(WALMode.LOG_ONLY) - ); - - return cfg; - } - - /** - * Test that dynamically added column is persisted along with the rest of cache configuration. - * @throws Exception - */ - public void testDynamicColumnsMetadataPersistence() throws Exception { - IgniteEx node = startGrid(getConfiguration()); - - node.active(true); - - node.cache("idx"); - - run(node, "ALTER TABLE \"idx\".Person ADD COLUMN age int"); - - doSleep(500); - - QueryField c = c("AGE", Integer.class.getName()); - - checkNodeState(node, "idx", "PERSON", c); - - stopGrid(0); - - node = startGrid(getConfiguration()); - - node.active(true); - - checkNodeState(node, "idx", "PERSON", c); - } -} diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java index 87b29b7975904..f25187c0f2416 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java @@ -113,82 +113,48 @@ private CacheConfiguration cacheCfg(String name) { System.clearProperty(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK); } - /** */ - public void testPersistIndex() throws Exception { - IgniteEx ig0 = startGrid(0); - - final AtomicInteger cnt = new AtomicInteger(); - - GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database(); - - db.addCheckpointListener(new DbCheckpointListener() { - @Override public void onCheckpointBegin(Context context) { - cnt.incrementAndGet(); - } - }); - - ig0.active(true); - - ig0.context().query().querySqlFieldsNoCache( - new SqlFieldsQuery("create table \"Person\" (\"id\" int primary key, \"name\" varchar)"), false).getAll(); - - assertEquals(0, indexCnt(ig0, CACHE_NAME)); - - ig0.context().query().querySqlFieldsNoCache( - new SqlFieldsQuery("Create index \"my_idx\" on \"Person\" (\"name\")"), false).getAll(); - - assertEquals(1, indexCnt(ig0, CACHE_NAME)); - - waitForCheckpoint(cnt); - - stopGrid(0); - - IgniteEx ig1 = startGrid(0); - - assertEquals(1, indexCnt(ig1, CACHE_NAME)); - } - /** */ public void testPersistCompositeIndex() throws Exception { - IgniteEx ig0 = startGrid(0); - startGrid(1); + IgniteEx node = startGrid(0); - ig0.active(true); + node.active(true); final AtomicInteger cnt = new AtomicInteger(); - GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database(); + GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)node.context().cache().context().database(); db.addCheckpointListener(new DbCheckpointListener() { - @Override public void onCheckpointBegin(Context context) { + @Override public void onCheckpointBegin(Context ctx) { cnt.incrementAndGet(); } }); - ig0.context().query().querySqlFieldsNoCache( + node.context().query().querySqlFieldsNoCache( new SqlFieldsQuery("create table \"Person\" (\"id\" int primary key, \"name\" varchar)"), false).getAll(); - assertEquals(0, indexCnt(ig0, CACHE_NAME)); + assertEquals(0, indexCnt(node, CACHE_NAME)); - ig0.context().query().querySqlFieldsNoCache( + node.context().query().querySqlFieldsNoCache( new SqlFieldsQuery("Create index \"my_idx\" on \"Person\" (\"id\", \"name\")"), false).getAll(); - assertEquals(1, indexCnt(ig0, CACHE_NAME)); + assertEquals(1, indexCnt(node, CACHE_NAME)); - ig0.context().query().querySqlFieldsNoCache( + node.context().query().querySqlFieldsNoCache( new SqlFieldsQuery("alter table \"Person\" add column \"age\" int"), false).getAll(); - assertEquals(3, colsCnt(ig0, CACHE_NAME)); + assertEquals(3, colsCnt(node, CACHE_NAME)); waitForCheckpoint(cnt); - stopGrid(1); + stopGrid(0); + + node = startGrid(0); - IgniteEx ig1 = startGrid(1); + node.active(true); - assertEquals(1, indexCnt(ig1, CACHE_NAME)); + assertEquals(1, indexCnt(node, CACHE_NAME)); - assertEquals(3, colsCnt(ig0, CACHE_NAME)); + assertEquals(3, colsCnt(node, CACHE_NAME)); } /** */ @@ -277,21 +243,19 @@ public Person(int id) { if (o == null || getClass() != o.getClass()) return false; - IgnitePersistentStoreSchemaLoadTest.Person person = (IgnitePersistentStoreSchemaLoadTest.Person)o; + IgnitePersistentStoreSchemaLoadTest.Person person = (IgnitePersistentStoreSchemaLoadTest.Person) o; - if (id != person.id) - return false; + return id == person.id && (name != null ? name.equals(person.name) : person.name == null); - return name != null ? name.equals(person.name) : person.name == null; } /** {@inheritDoc} */ @Override public int hashCode() { - int result = id; + int res = id; - result = 31 * result + (name != null ? name.hashCode() : 0); + res = 31 * res + (name != null ? name.hashCode() : 0); - return result; + return res; } } } From b60666cc5aa8d18fc145677f90cda361393d8d08 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Wed, 20 Sep 2017 23:31:03 +0300 Subject: [PATCH 4/4] StoredCacheData persistence tests. --- .../IgnitePersistentStoreSchemaLoadTest.java | 163 +++++++++++++----- 1 file changed, 124 insertions(+), 39 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java index f25187c0f2416..b4c08b23fee70 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java @@ -18,7 +18,8 @@ package org.apache.ignite.internal.processors.database; import java.io.Serializable; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.CountDownLatch; +import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheRebalanceMode; @@ -29,18 +30,15 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.PersistentStoreConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.query.QuerySchema; import org.apache.ignite.internal.processors.query.QueryUtils; -import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK; @@ -58,10 +56,11 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest /** Table name. */ private static final String TBL_NAME = Person.class.getSimpleName(); - /** Cache name. */ - private static final String CACHE_NAME = QueryUtils.createTableCacheName(QueryUtils.DFLT_SCHEMA, - TBL_NAME); + /** Name of the cache created with {@code CREATE TABLE}. */ + private static final String SQL_CACHE_NAME = QueryUtils.createTableCacheName(QueryUtils.DFLT_SCHEMA, TBL_NAME); + /** Name of the cache created upon cluster start. */ + private static final String STATIC_CACHE_NAME = TBL_NAME; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { @@ -82,6 +81,26 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest return cfg; } + /** + * Create node configuration with a cache pre-configured. + * @param gridName Node name. + * @return Node configuration with a cache pre-configured. + * @throws Exception if failed. + */ + @SuppressWarnings("unchecked") + private IgniteConfiguration getConfigurationWithStaticCache(String gridName) throws Exception { + IgniteConfiguration cfg = getConfiguration(gridName); + + CacheConfiguration ccfg = cacheCfg(STATIC_CACHE_NAME); + + ccfg.setIndexedTypes(Integer.class, Person.class); + ccfg.setSqlEscapeAll(true); + + cfg.setCacheConfiguration(ccfg); + + return optimize(cfg); + } + /** */ private CacheConfiguration cacheCfg(String name) { CacheConfiguration cfg = new CacheConfiguration<>(); @@ -114,58 +133,80 @@ private CacheConfiguration cacheCfg(String name) { } /** */ - public void testPersistCompositeIndex() throws Exception { - IgniteEx node = startGrid(0); - - node.active(true); + public void testDynamicSchemaChangesPersistence() throws Exception { + checkSchemaStateAfterNodeRestart(false); + } - final AtomicInteger cnt = new AtomicInteger(); + /** */ + public void testDynamicSchemaChangesPersistenceWithAliveCluster() throws Exception { + checkSchemaStateAfterNodeRestart(true); + } - GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)node.context().cache().context().database(); + /** */ + @SuppressWarnings("unchecked") + public void testDynamicSchemaChangesPersistenceWithStaticCache() throws Exception { + IgniteEx node = startGrid(getConfigurationWithStaticCache(getTestIgniteInstanceName(0))); - db.addCheckpointListener(new DbCheckpointListener() { - @Override public void onCheckpointBegin(Context ctx) { - cnt.incrementAndGet(); - } - }); + node.active(true); - node.context().query().querySqlFieldsNoCache( - new SqlFieldsQuery("create table \"Person\" (\"id\" int primary key, \"name\" varchar)"), false).getAll(); + IgniteCache cache = node.cache(STATIC_CACHE_NAME); - assertEquals(0, indexCnt(node, CACHE_NAME)); + assertNotNull(cache); - node.context().query().querySqlFieldsNoCache( - new SqlFieldsQuery("Create index \"my_idx\" on \"Person\" (\"id\", \"name\")"), false).getAll(); + CountDownLatch cnt = checkpointLatch(node); - assertEquals(1, indexCnt(node, CACHE_NAME)); + assertEquals(0, indexCnt(node, STATIC_CACHE_NAME)); - node.context().query().querySqlFieldsNoCache( - new SqlFieldsQuery("alter table \"Person\" add column \"age\" int"), false).getAll(); + makeDynamicSchemaChanges(node, STATIC_CACHE_NAME); - assertEquals(3, colsCnt(node, CACHE_NAME)); + checkDynamicSchemaChanges(node, STATIC_CACHE_NAME); - waitForCheckpoint(cnt); + cnt.await(); stopGrid(0); + // Restarting with no-cache configuration - otherwise stored configurations + // will be ignored due to cache names duplication. node = startGrid(0); node.active(true); - assertEquals(1, indexCnt(node, CACHE_NAME)); - - assertEquals(3, colsCnt(node, CACHE_NAME)); + checkDynamicSchemaChanges(node, STATIC_CACHE_NAME); } - /** */ - private void waitForCheckpoint(final AtomicInteger cnt) throws IgniteInterruptedCheckedException { - final int i = cnt.get(); + /** + * Perform test with cache created with {@code CREATE TABLE}. + * @param aliveCluster Whether there should remain an alive node when tested node is restarted. + * @throws Exception if failed. + */ + private void checkSchemaStateAfterNodeRestart(boolean aliveCluster) throws Exception { + IgniteEx node = startGrid(0); - GridTestUtils.waitForCondition(new GridAbsPredicate() { - @Override public boolean apply() { - return cnt.get() > i; - } - }, 2000); + node.active(true); + + if (aliveCluster) + startGrid(1); + + CountDownLatch cnt = checkpointLatch(node); + + node.context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("create table \"Person\" (\"id\" int primary key, \"name\" varchar)"), false).getAll(); + + assertEquals(0, indexCnt(node, SQL_CACHE_NAME)); + + makeDynamicSchemaChanges(node, QueryUtils.DFLT_SCHEMA); + + checkDynamicSchemaChanges(node, SQL_CACHE_NAME); + + cnt.await(); + + stopGrid(0); + + node = startGrid(0); + + node.active(true); + + checkDynamicSchemaChanges(node, SQL_CACHE_NAME); } /** */ @@ -209,6 +250,50 @@ private void deleteWorkFiles() throws IgniteCheckedException { deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false)); } + /** + * @param node Node whose checkpoint to wait for. + * @return Latch released when checkpoint happens. + */ + private CountDownLatch checkpointLatch(IgniteEx node) { + final CountDownLatch cnt = new CountDownLatch(1); + + GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)node.context().cache().context().database(); + + db.addCheckpointListener(new DbCheckpointListener() { + @Override public void onCheckpointBegin(Context ctx) { + cnt.countDown(); + } + }); + + return cnt; + } + + /** + * Create dynamic index and column. + * @param node Node. + * @param schema Schema name. + */ + private void makeDynamicSchemaChanges(IgniteEx node, String schema) { + node.context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("create index \"my_idx\" on \"Person\" (\"id\", \"name\")").setSchema(schema), false) + .getAll(); + + node.context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("alter table \"Person\" add column \"age\" int").setSchema(schema), false) + .getAll(); + } + + /** + * Check that dynamically created schema objects are in place. + * @param node Node. + * @param cacheName Cache name. + */ + private void checkDynamicSchemaChanges(IgniteEx node, String cacheName) { + assertEquals(1, indexCnt(node, cacheName)); + + assertEquals(3, colsCnt(node, cacheName)); + } + /** * */