Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.internal.ClusterPerTestIntegrationTest;
import org.apache.ignite.internal.app.IgniteImpl;
import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
import org.apache.ignite.internal.catalog.events.CatalogEvent;
import org.apache.ignite.internal.lang.ByteArray;
import org.apache.ignite.internal.metastorage.Entry;
Expand All @@ -52,7 +53,8 @@
import org.apache.ignite.internal.testframework.IgniteTestUtils;
import org.intellij.lang.annotations.Language;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;

/**
* Integration test for data nodes' filters functionality.
Expand Down Expand Up @@ -111,15 +113,20 @@ protected int initialNodes() {
* @throws Exception If failed.
*/
@Disabled("https://issues.apache.org/jira/browse/IGNITE-21387")
void testFilteredDataNodesPropagatedToStable() throws Exception {
@ParameterizedTest
@EnumSource(ConsistencyMode.class)
void testFilteredDataNodesPropagatedToStable(ConsistencyMode consistencyMode) throws Exception {
String filter = "$[?(@.region == \"US\" && @.storage == \"SSD\")]";

// This node do not pass the filter
@Language("HOCON") String firstNodeAttributes = "{region: EU, storage: SSD}";

Ignite node = startNode(1, createStartConfig(firstNodeAttributes, STORAGE_PROFILES_CONFIGS));
Ignite node = unwrapIgniteImpl(startNode(1, createStartConfig(firstNodeAttributes, STORAGE_PROFILES_CONFIGS)));

node.sql().execute(null, createZoneSql(2, 3, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, filter, STORAGE_PROFILES));
node.sql().execute(
null,
createZoneSql(2, 3, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, filter, STORAGE_PROFILES, consistencyMode)
);

node.sql().execute(null, createTableSql());

Expand Down Expand Up @@ -182,13 +189,17 @@ void testFilteredDataNodesPropagatedToStable() throws Exception {
*
* @throws Exception If failed.
*/
@Test
void testAlteringFiltersPropagatedDataNodesToStableImmediately() throws Exception {
@ParameterizedTest
@EnumSource(ConsistencyMode.class)
void testAlteringFiltersPropagatedDataNodesToStableImmediately(ConsistencyMode consistencyMode) throws Exception {
String filter = "$[?(@.region == \"US\" && @.storage == \"SSD\")]";

Ignite node0 = unwrapIgniteImpl(node(0));

node0.sql().execute(null, createZoneSql(2, 3, 10_000, 10_000, filter, STORAGE_PROFILES));
node0.sql().execute(
null,
createZoneSql(2, 3, 10_000, 10_000, filter, STORAGE_PROFILES, consistencyMode)
);

node0.sql().execute(null, createTableSql());

Expand Down Expand Up @@ -237,13 +248,17 @@ void testAlteringFiltersPropagatedDataNodesToStableImmediately() throws Exceptio
*
* @throws Exception If failed.
*/
@Test
void testEmptyDataNodesDoNotPropagatedToStableAfterAlteringFilter() throws Exception {
@ParameterizedTest
@EnumSource(ConsistencyMode.class)
void testEmptyDataNodesDoNotPropagatedToStableAfterAlteringFilter(ConsistencyMode consistencyMode) throws Exception {
String filter = "$[?(@.region == \"US\" && @.storage == \"SSD\")]";

Ignite node0 = unwrapIgniteImpl(node(0));

node0.sql().execute(null, createZoneSql(2, 3, 10_000, 10_000, filter, STORAGE_PROFILES));
node0.sql().execute(
null,
createZoneSql(2, 3, 10_000, 10_000, filter, STORAGE_PROFILES, consistencyMode)
);

node0.sql().execute(null, createTableSql());

Expand Down Expand Up @@ -298,8 +313,9 @@ void testEmptyDataNodesDoNotPropagatedToStableAfterAlteringFilter() throws Excep
*
* @throws Exception If failed.
*/
@Test
void testFilteredEmptyDataNodesDoNotTriggerRebalance() throws Exception {
@ParameterizedTest
@EnumSource(ConsistencyMode.class)
void testFilteredEmptyDataNodesDoNotTriggerRebalance(ConsistencyMode consistencyMode) throws Exception {
String filter = "$[?(@.region == \"EU\" && @.storage == \"HDD\")]";

// This node do not pass the filter.
Expand All @@ -310,7 +326,10 @@ void testFilteredEmptyDataNodesDoNotTriggerRebalance() throws Exception {

Ignite node1 = startNode(1, createStartConfig(firstNodeAttributes, STORAGE_PROFILES_CONFIGS));

node1.sql().execute(null, createZoneSql(1, 1, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, filter, STORAGE_PROFILES));
node1.sql().execute(
null,
createZoneSql(1, 1, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, filter, STORAGE_PROFILES, consistencyMode)
);

MetaStorageManager metaStorageManager = IgniteTestUtils.getFieldValue(
node0,
Expand Down Expand Up @@ -342,8 +361,9 @@ void testFilteredEmptyDataNodesDoNotTriggerRebalance() throws Exception {
assertPendingAssignmentsWereNeverExist(metaStorageManager, partId);
}

@Test
void testFilteredEmptyDataNodesDoNotTriggerRebalanceOnReplicaUpdate() throws Exception {
@ParameterizedTest
@EnumSource(ConsistencyMode.class)
void testFilteredEmptyDataNodesDoNotTriggerRebalanceOnReplicaUpdate(ConsistencyMode consistencyMode) throws Exception {
String filter = "$[?(@.region == \"EU\" && @.storage == \"HDD\")]";

// This node do not pass the filter.
Expand All @@ -354,7 +374,10 @@ void testFilteredEmptyDataNodesDoNotTriggerRebalanceOnReplicaUpdate() throws Exc

startNode(1, createStartConfig(firstNodeAttributes, STORAGE_PROFILES_CONFIGS));

node0.sql().execute(null, createZoneSql(1, 1, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, filter, STORAGE_PROFILES));
node0.sql().execute(
null,
createZoneSql(1, 1, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, filter, STORAGE_PROFILES, consistencyMode)
);

MetaStorageManager metaStorageManager = IgniteTestUtils.getFieldValue(
unwrapIgniteImpl(node0),
Expand Down Expand Up @@ -444,16 +467,25 @@ private static void assertPendingAssignmentsWereNeverExist(
assertTrue(metaStorageManager.get(pendingPartAssignmentsKey(partId)).get().empty());
}

private static String createZoneSql(int partitions, int replicas, int scaleUp, int scaleDown, String filter, String storageProfiles) {
private static String createZoneSql(
int partitions,
int replicas,
int scaleUp,
int scaleDown,
String filter,
String storageProfiles,
ConsistencyMode consistencyMode
) {
String sqlFormat = "CREATE ZONE \"%s\" WITH "
+ "\"REPLICAS\" = %s, "
+ "\"PARTITIONS\" = %s, "
+ "\"DATA_NODES_FILTER\" = '%s', "
+ "\"DATA_NODES_AUTO_ADJUST_SCALE_UP\" = %s, "
+ "\"DATA_NODES_AUTO_ADJUST_SCALE_DOWN\" = %s, "
+ "\"STORAGE_PROFILES\" = %s";
+ "\"STORAGE_PROFILES\" = %s, "
+ "\"CONSISTENCY_MODE\" = '%s'";

return String.format(sqlFormat, ZONE_NAME, replicas, partitions, filter, scaleUp, scaleDown, storageProfiles);
return String.format(sqlFormat, ZONE_NAME, replicas, partitions, filter, scaleUp, scaleDown, storageProfiles, consistencyMode);
}

private static String alterZoneSql(String filter) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
import org.apache.ignite.internal.catalog.CatalogManagerImpl;
import org.apache.ignite.internal.catalog.CatalogTestUtils;
import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
import org.apache.ignite.internal.catalog.storage.UpdateLogImpl;
import org.apache.ignite.internal.cluster.management.ClusterManagementGroupManager;
import org.apache.ignite.internal.cluster.management.raft.TestClusterStateStorage;
Expand Down Expand Up @@ -139,7 +140,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.EnumSource;

/**
* Tests for checking {@link DistributionZoneManager} behavior after node's restart.
Expand Down Expand Up @@ -406,14 +408,18 @@ public void testNodeAttributesRestoredAfterRestart() throws Exception {
assertEquals(nodeAttributesBeforeRestart, nodeAttributesAfterRestart);
}

@ParameterizedTest(name = "defaultZone={0}")
@ValueSource(booleans = {true, false})
public void testTopologyAugmentationMapRestoredAfterRestart(boolean defaultZone) throws Exception {
@ParameterizedTest(name = "defaultZone={0},consistencyMode={1}")
@CsvSource({
"true,",
"false, HIGH_AVAILABILITY",
"false, STRONG_CONSISTENCY",
})
public void testTopologyAugmentationMapRestoredAfterRestart(boolean defaultZone, ConsistencyMode consistencyMode) throws Exception {
PartialNode node = startPartialNode(0);

node.logicalTopology().putNode(A);

String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE);
String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, consistencyMode);

node.logicalTopology().putNode(B);
node.logicalTopology().putNode(C);
Expand Down Expand Up @@ -613,8 +619,9 @@ public void testFirstLogicalTopologyUpdateInterruptedEventRestoredAfterRestart()
);
}

@Test
public void testCreationZoneWhenDataNodesAreDeletedIsNotSuccessful() throws Exception {
@ParameterizedTest
@EnumSource(ConsistencyMode.class)
public void testCreationZoneWhenDataNodesAreDeletedIsNotSuccessful(ConsistencyMode consistencyMode) throws Exception {
var imitateConcurrentRemoval = new AtomicBoolean();

var dataNodeKey = new AtomicReference<ByteArray>();
Expand Down Expand Up @@ -682,7 +689,15 @@ public void testCreationZoneWhenDataNodesAreDeletedIsNotSuccessful() throws Exce

imitateConcurrentRemoval.set(true);

createZone(getCatalogManager(node), "zone1", INFINITE_TIMER_VALUE, INFINITE_TIMER_VALUE, null, DEFAULT_STORAGE_PROFILE);
createZone(
getCatalogManager(node),
"zone1",
INFINITE_TIMER_VALUE,
INFINITE_TIMER_VALUE,
null,
consistencyMode,
DEFAULT_STORAGE_PROFILE
);

// Assert that after creation of a zone, data nodes are still tombstone, but not the logical topology, as for default zone.
assertThat(metastore.get(dataNodeKey.get()).thenApply(Entry::tombstone), willBe(true));
Expand All @@ -692,12 +707,16 @@ private Set<NodeWithAttributes> deserializeLogicalTopologySet(byte[] bytes) {
return DistributionZonesUtil.deserializeLogicalTopologySet(bytes);
}

@ParameterizedTest(name = "defaultZone={0}")
@ValueSource(booleans = {true, false})
public void testLocalDataNodesAreRestoredAfterRestart(boolean defaultZone) throws Exception {
@ParameterizedTest(name = "defaultZone={0},consistencyMode={1}")
@CsvSource({
"true,",
"false, HIGH_AVAILABILITY",
"false, STRONG_CONSISTENCY",
})
public void testLocalDataNodesAreRestoredAfterRestart(boolean defaultZone, ConsistencyMode consistencyMode) throws Exception {
PartialNode node = startPartialNode(0);

String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE);
String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, IMMEDIATE_TIMER_VALUE, IMMEDIATE_TIMER_VALUE, consistencyMode);

node.logicalTopology().putNode(A);
node.logicalTopology().putNode(B);
Expand Down Expand Up @@ -729,17 +748,21 @@ public void testLocalDataNodesAreRestoredAfterRestart(boolean defaultZone) throw
Set.of(A), TIMEOUT_MILLIS);
}

@ParameterizedTest(name = "defaultZone={0}")
@ValueSource(booleans = {true, false})
public void testScaleUpTimerIsRestoredAfterRestart(boolean defaultZone) throws Exception {
@ParameterizedTest(name = "defaultZone={0},consistencyMode={1}")
@CsvSource({
"true,",
"false, HIGH_AVAILABILITY",
"false, STRONG_CONSISTENCY",
})
public void testScaleUpTimerIsRestoredAfterRestart(boolean defaultZone, ConsistencyMode consistencyMode) throws Exception {
PartialNode node = startPartialNode(0);

node.logicalTopology().putNode(A);
node.logicalTopology().putNode(B);

assertLogicalTopologyInMetastorage(Set.of(A, B), metastore);

String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, 1, 1);
String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, 1, 1, consistencyMode);

int zoneId = getZoneId(node, zoneName);

Expand Down Expand Up @@ -797,9 +820,13 @@ public void testScaleUpTimerIsRestoredAfterRestart(boolean defaultZone) throws E
);
}

@ParameterizedTest(name = "defaultZone={0}")
@ValueSource(booleans = {true, false})
public void testScaleDownTimerIsRestoredAfterRestart(boolean defaultZone) throws Exception {
@ParameterizedTest(name = "defaultZone={0},consistencyMode={1}")
@CsvSource({
"true,",
"false, HIGH_AVAILABILITY",
"false, STRONG_CONSISTENCY",
})
public void testScaleDownTimerIsRestoredAfterRestart(boolean defaultZone, ConsistencyMode consistencyMode) throws Exception {
PartialNode node = startPartialNode(0);

node.logicalTopology().putNode(A);
Expand All @@ -810,7 +837,7 @@ public void testScaleDownTimerIsRestoredAfterRestart(boolean defaultZone) throws
DistributionZoneManager distributionZoneManager = getDistributionZoneManager(node);
CatalogManager catalogManager = getCatalogManager(node);

String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, 1, 1);
String zoneName = createZoneOrAlterDefaultZone(node, defaultZone, 1, 1, consistencyMode);

int zoneId = getZoneId(node, zoneName);

Expand Down Expand Up @@ -860,7 +887,8 @@ private static String createZoneOrAlterDefaultZone(
PartialNode node,
boolean useDefaultZone,
int scaleUp,
int scaleDown
int scaleDown,
ConsistencyMode consistencyMode
) throws Exception {
String zoneName;

Expand All @@ -885,7 +913,7 @@ private static String createZoneOrAlterDefaultZone(
} else {
zoneName = ZONE_NAME;

createZone(getCatalogManager(node), zoneName, scaleUp, scaleDown, null, DEFAULT_STORAGE_PROFILE);
createZone(getCatalogManager(node), zoneName, scaleUp, scaleDown, null, consistencyMode, DEFAULT_STORAGE_PROFILE);
}

return zoneName;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.apache.ignite.internal.catalog.CatalogManager;
import org.apache.ignite.internal.catalog.CatalogTestUtils;
import org.apache.ignite.internal.catalog.descriptors.CatalogZoneDescriptor;
import org.apache.ignite.internal.catalog.descriptors.ConsistencyMode;
import org.apache.ignite.internal.cluster.management.ClusterManagementGroupManager;
import org.apache.ignite.internal.cluster.management.raft.ClusterStateStorage;
import org.apache.ignite.internal.cluster.management.raft.TestClusterStateStorage;
Expand Down Expand Up @@ -185,6 +186,7 @@ protected void createZone(
@Nullable Integer dataNodesAutoAdjustScaleUp,
@Nullable Integer dataNodesAutoAdjustScaleDown,
@Nullable String filter,
@Nullable ConsistencyMode consistencyMode,
String storageProfiles
) {
DistributionZonesTestUtil.createZone(
Expand All @@ -193,6 +195,7 @@ protected void createZone(
dataNodesAutoAdjustScaleUp,
dataNodesAutoAdjustScaleDown,
filter,
consistencyMode,
storageProfiles
);
}
Expand Down
Loading