From 08c0c1aa2c9d611715c424fa48053a24b3d42f11 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Sun, 21 Feb 2021 01:44:33 +0300 Subject: [PATCH 01/21] IGNITE-13508: added two-phase rebalance test --- .../ducktest/tests/DeleteDataApplication.java | 92 ++++++ .../snapshot_test/DataLoaderApplication.java | 19 +- .../ignitetest/services/utils/ignite_aware.py | 25 +- .../utils/ignite_configuration/cache.py | 13 + .../ignite_configuration/data_storage.py | 1 + .../services/utils/templates/cache_macro.j2 | 32 +- .../utils/templates/datastorage_macro.j2 | 4 + .../tests/ignitetest/services/utils/util.py | 31 ++ .../tests/ignitetest/tests/snapshot_test.py | 6 +- .../ignitetest/tests/suites/slow_suite.yml | 3 + .../tests/two_phased_rebalanced_test.py | 296 ++++++++++++++++++ 11 files changed, 498 insertions(+), 24 deletions(-) create mode 100644 modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java create mode 100644 modules/ducktests/tests/ignitetest/services/utils/util.py create mode 100644 modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java new file mode 100644 index 0000000000000..d7c74faadf855 --- /dev/null +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.ducktest.tests; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.TreeSet; +import javax.cache.Cache; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; +import org.apache.ignite.lang.IgniteFuture; + +/** + * Deleting data from the cache. + */ +public class DeleteDataApplication extends IgniteAwareApplication { + /** {@inheritDoc} */ + @Override public void run(JsonNode jNode) { + String cacheName = jNode.get("cacheName").asText(); + + int size = jNode.get("size").asInt(); + + int bachSize = Optional.ofNullable(jNode.get("bachSize")) + .map(JsonNode::asInt) + .orElse(1000); + + IgniteCache cache = ignite.getOrCreateCache(cacheName); + + log.info("Cache size before: " + cache.size()); + + markInitialized(); + + long start = System.currentTimeMillis(); + + Iterator> iter = cache.iterator(); + + ArrayList keys = new ArrayList<>(size); + + int cnt = 0; + + while (iter.hasNext() && cnt < size) { + keys.add(iter.next().getKey()); + + cnt++; + } + + log.info("Start removing: " + keys.size()); + + int listSize = keys.size(); + + List> futures = new LinkedList<>(); + + int fromIdx = 0; + int toIdx = 0; + + while (fromIdx < listSize) { + toIdx = Math.min(fromIdx + bachSize, listSize); + + futures.add(cache.removeAllAsync(new TreeSet<>(keys.subList(fromIdx, toIdx)))); + + fromIdx = toIdx + 1; + } + + futures.forEach(IgniteFuture::get); + + log.info("Cache size after: " + cache.size()); + + recordResult("DURATION", System.currentTimeMillis() - start); + + markFinished(); + } +} diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/snapshot_test/DataLoaderApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/snapshot_test/DataLoaderApplication.java index b56241158da4f..6e96bd7ac6037 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/snapshot_test/DataLoaderApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/snapshot_test/DataLoaderApplication.java @@ -17,13 +17,8 @@ package org.apache.ignite.internal.ducktest.tests.snapshot_test; -import java.util.Collections; import com.fasterxml.jackson.databind.JsonNode; import org.apache.ignite.IgniteDataStreamer; -import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.QueryEntity; -import org.apache.ignite.cache.QueryIndex; -import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication; /** @@ -42,19 +37,7 @@ public class DataLoaderApplication extends IgniteAwareApplication { markInitialized(); - QueryEntity qryEntity = new QueryEntity() - .setKeyFieldName("id") - .setKeyType(Long.class.getName()) - .setTableName("TEST_TABLE") - .setValueType(byte[].class.getName()) - .addQueryField("id", Long.class.getName(), null) - .setIndexes(Collections.singletonList(new QueryIndex("id"))); - - CacheConfiguration cacheCfg = new CacheConfiguration<>(cacheName); - cacheCfg.setCacheMode(CacheMode.REPLICATED); - cacheCfg.setQueryEntities(Collections.singletonList(qryEntity)); - - ignite.getOrCreateCache(cacheCfg); + ignite.getOrCreateCache(cacheName); byte[] data = new byte[valSize]; diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index fa4c5c4a362fc..e3a1fec879555 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -22,7 +22,8 @@ import sys import time from abc import abstractmethod, ABCMeta -from datetime import datetime +from datetime import datetime, timedelta +from distutils.util import strtobool from enum import IntEnum from threading import Thread @@ -32,7 +33,7 @@ from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue from ignitetest.services.utils.path import IgnitePathAware from ignitetest.services.utils.ignite_spec import resolve_spec -from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin +from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin, JmxClient from ignitetest.services.utils.log_utils import monitor_log from ignitetest.utils.enum import constructible @@ -459,3 +460,23 @@ def restore_from_snapshot(self, snapshot_name: str): node.account.ssh(f'rm -rf {self.database_dir}', allow_fail=False) node.account.ssh(f'cp -r {snapshot_db} {self.work_dir}', allow_fail=False) + + def await_rebalance(self, timeout_sec=180): + """ + Waiting for the rebalance to complete. + :param timeout_sec: timeout to wait the rebalance to complete. + """ + delta_time = datetime.now() + timedelta(seconds=timeout_sec) + + rebalanced = False + for node in self.nodes: + rebalanced = False + mbean = JmxClient(node).find_mbean('.*name=cluster') + + while datetime.now() < delta_time and not rebalanced: + rebalanced = bool(strtobool(next(mbean.Rebalanced))) + + if rebalanced: + return + + raise TimeoutError(f'Rebalancing was not completed within the time: {timeout_sec} seconds.') diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py index cc58a69bb3670..176931e24b8cd 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py @@ -18,6 +18,17 @@ """ from typing import NamedTuple +AFFINITY_BACKUP_FILTER = 'BACKUP_FILTER' +CELL = 'CELL' + + +class Affinity(NamedTuple): + """ + Affinity. + """ + name: str = AFFINITY_BACKUP_FILTER + constructor_arg: str = CELL + class CacheConfiguration(NamedTuple): """ @@ -27,3 +38,5 @@ class CacheConfiguration(NamedTuple): cache_mode: str = 'PARTITIONED' atomicity_mode: str = 'ATOMIC' backups: int = 0 + indexed_types: list = None + affinity: Affinity = None diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py index 7b2999d82e4b8..b455ef186b61b 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py @@ -36,3 +36,4 @@ class DataStorageConfiguration(NamedTuple): """ default: DataRegionConfiguration = DataRegionConfiguration() regions: list = [] + checkpoint_frequency: int = None diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 index d227369abe3d7..2ac952523b144 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 @@ -15,6 +15,17 @@ limitations under the License. #} + +{% macro rendezvous_backup_filter(affinity) %} + + + + + + + +{% endmacro %} + {% macro cache_configs(caches) %} {% if caches %} @@ -22,10 +33,25 @@ {% for cache in caches %} - {% if cache.cache_mode == 'PARTITIONED' %} - - {% endif %} + + + + {% if cache.indexed_types %} + + + {% for class in cache.indexed_types %} + {{ class }} + {% endfor %} + + + {% endif %} + + {% if cache.affinity and cache.affinity.name == 'BACKUP_FILTER' %} + + {{ rendezvous_backup_filter(cache.affinity) }} + + {% endif %} {% endfor %} diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 index 1c2b4627b10e4..44abee31ef3b8 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 @@ -31,6 +31,10 @@ {% endif %} + + {% if config.checkpoint_frequency %} + + {% endif %} {% endif %} diff --git a/modules/ducktests/tests/ignitetest/services/utils/util.py b/modules/ducktests/tests/ignitetest/services/utils/util.py new file mode 100644 index 0000000000000..a3530bb8c9736 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/util.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains utility methods. +""" +import os + + +def copy_file_to_dest(node, file_path: str, dest_dir: str): + """ + Move file to logs directory. + :return new path to file. + """ + node.account.ssh_output(f'cp {file_path} {dest_dir}') + + file_name = os.path.basename(file_path) + + return os.path.join(dest_dir, file_name) diff --git a/modules/ducktests/tests/ignitetest/tests/snapshot_test.py b/modules/ducktests/tests/ignitetest/tests/snapshot_test.py index edb8012b627e1..d5a6f27415985 100644 --- a/modules/ducktests/tests/ignitetest/tests/snapshot_test.py +++ b/modules/ducktests/tests/ignitetest/tests/snapshot_test.py @@ -16,8 +16,8 @@ """ Module contains snapshot test. """ - from ducktape.mark.resource import cluster +from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration from ignitetest.services.ignite import IgniteService from ignitetest.services.ignite_app import IgniteApplicationService @@ -47,9 +47,13 @@ def snapshot_test(self, ignite_version): """ version = IgniteVersion(ignite_version) + cache_cfg = CacheConfiguration(name=self.CACHE_NAME, cache_mode='REPLICATED', + indexed_types=['java.lang.Long', 'byte[]']) + ignite_config = IgniteConfiguration( version=version, data_storage=DataStorageConfiguration(default=DataRegionConfiguration(persistent=True)), + caches=[cache_cfg], metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi' ) diff --git a/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml b/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml index 863f9a4b26841..134e7366247a5 100644 --- a/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml +++ b/modules/ducktests/tests/ignitetest/tests/suites/slow_suite.yml @@ -18,3 +18,6 @@ discovery: snapshot: - ../snapshot_test.py + +two_phased_rebalance: + - ../two_phased_rebalanced_test.py.py diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py new file mode 100644 index 0000000000000..9a9858ec95fca --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -0,0 +1,296 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains Cellular Affinity tests. +""" +import ducktape +from ducktape.mark.resource import cluster +from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration, Affinity + +from ignitetest.services.ignite import IgniteService +from ignitetest.services.ignite_app import IgniteApplicationService +from ignitetest.services.utils.control_utility import ControlUtility +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration +from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration +from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster +from ignitetest.services.utils.util import copy_file_to_dest +from ignitetest.utils import ignite_versions +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + + +# pylint: disable=W0223 +class TwoPhasedRebalancedTest(IgniteTest): + """ + Two-phase rebalancing test case. + """ + NUM_NODES_CELL = 4 + + NUM_CELL = 4 + + ATTRIBUTE = "CELL" + + CACHE_NAME = "test-cache" + + # pylint: disable=R0914 + @cluster(num_nodes=NUM_NODES_CELL * NUM_CELL + 2) + @ignite_versions(str(DEV_BRANCH)) + def two_phased_rebalancing_test(self, ignite_version): + """ + Test case of two-phase rebalancing. + Preparations. + 1. Start 4 cells. + 2. Load data to cache with the mentioned above affinity function and fix PDS size on all nodes. + 3. Delete 80% of data and fix PDS size on all nodes. + Phase 1. + 1. Stop two nodes in each cell, total a half of all nodes and clean PDS. + 2. Start cleaned node with preservance of consistent id and cell attributes. + 3. Wait for the rebalance to complete. + Phase 2. + Run steps 1-3 of Phase 2 on the other half of the cluster. + Verifications. + 1. Check that PDS size reduced (compare to step 3) + 2. Check data consistency (idle_verify --dump) + """ + data_storage = DataStorageConfiguration(default=DataRegionConfiguration(persistent=True), + checkpoint_frequency=30000) + + cells = self.start_cells(ignite_version=ignite_version, + cells_cnt=self.NUM_CELL, + cell_nodes_cnt=self.NUM_NODES_CELL, + cache_name=self.CACHE_NAME, + data_storage=data_storage) + + control_utility = ControlUtility(cells[0]) + control_utility.activate() + + client_config = IgniteConfiguration(client_mode=True, + version=IgniteVersion(ignite_version), + discovery_spi=from_ignite_cluster(cells[0])) + + streamer = IgniteApplicationService( + self.test_context, + client_config, + java_class_name="org.apache.ignite.internal.ducktest.tests.snapshot_test.DataLoaderApplication", + params={"start": 0, + "cacheName": "test-cache", + "interval": 500 * 1024, + "valueSizeKb": 1} + ) + + deleter = IgniteApplicationService( + self.test_context, + client_config, + java_class_name="org.apache.ignite.internal.ducktest.tests.DeleteDataApplication", + shutdown_timeout_sec=15 * 60, + params={"cacheName": "test-cache", + "size": 400 * 1024}) + + streamer.run() + + node = cells[0].nodes[0] + try: + cells[0].await_event_on_node('Skipping checkpoint', node, timeout_sec=60) + except ducktape.errors.TimeoutError as ex: + self.logger.warn(ex) + + self.fix_pds_size(cells, "Step prepare, load data. PDS.") + + deleter.run() + + try: + cells[0].await_event_on_node('Skipping checkpoint', node, timeout_sec=60) + except ducktape.errors.TimeoutError as ex: + self.logger.warn(ex) + + dump_1 = fix_data(control_utility, node, cells[0].log_dir) + + pds_before = self.fix_pds_size(cells, "After Delete 80%, PDS.") + + restart_with_clean_idx_node_on_cell(cells, [0, 1]) + + for cell in cells: + cell.await_rebalance(timeout_sec=15 * 60) + + self.fix_pds_size(cells, "After rebalancing complate on nodes 0, 1. PDS.") + + restart_with_clean_idx_node_on_cell(cells, [2, 3]) + + try: + cells[0].await_event_on_node('Skipping checkpoint', node, timeout_sec=60) + except ducktape.errors.TimeoutError as ex: + self.logger.warn(ex) + + for cell in cells: + cell.await_rebalance(timeout_sec=15 * 60) + + pds_after = self.fix_pds_size(cells, "After rebalancing complate on nodes 2, 3. PDS.") + + check_pds_size(pds_before, pds_after) + + dump_2 = fix_data(control_utility, node, cells[0].log_dir) + + diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True) + assert not diff + + # pylint: disable=R0913 + def start_cells(self, ignite_version: str, cells_cnt: int, cell_nodes_cnt: int, cache_name: str, + data_storage: DataStorageConfiguration = None): + """ + Start cells. + """ + assert cells_cnt > 0 + + cells = [] + + cache_cfg = CacheConfiguration(name=cache_name, backups=self.NUM_NODES_CELL-1, affinity=Affinity(), + atomicity_mode='TRANSACTIONAL') + + config = IgniteConfiguration(version=IgniteVersion(ignite_version), data_storage=data_storage, + caches=[cache_cfg], + metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi') + + cell = start_cell(self.test_context, config, [f'-D{self.ATTRIBUTE}=0'], num_nodes=cell_nodes_cnt,) + + discovery_spi = from_ignite_cluster(cell) + config = config._replace(discovery_spi=discovery_spi) + + cells.append(cell) + + if cells_cnt > 1: + for i in range(1, cells_cnt): + cells.append(start_cell(self.test_context, config, [f'-D{self.ATTRIBUTE}={i}'], + num_nodes=cell_nodes_cnt)) + + return cells + + def fix_pds_size(self, cells: [IgniteService], msg: str): + """ + Pds size in megabytes. + """ + assert len(cells) > 0 + + res = {} + + for cell in cells: + for node in cell.nodes: + consistent_id = str(node.account.hostname).replace('.', '_').replace('-', '_') + cmd = f'du -sm {cell.database_dir}/{consistent_id} | ' + "awk '{print $1}'" + res[node.account.hostname] = int(node.account.ssh_output(cmd).decode("utf-8").rstrip()) + + self.logger.warn(msg) + + for item in res.items(): + self.logger.warn(f'Host: {item[0]}, PDS {item[1]}mb') + + return res + + +def restart_with_clean_idx_node_on_cell(cells: [IgniteService], idxs: [int]): + """ + Restart idxs nodes on cells with cleaning working directory. + """ + stop_idx_node_on_cell(cells, idxs) + clean_work_idx_node_on_cell(cells, idxs) + start_idx_node_on_cell(cells, idxs) + + +def stop_idx_node_on_cell(cells: [IgniteService], idxs: [int]): + """ + Stop idxs nodes on cells. + """ + for cell in cells: + size = len(cell.nodes) + + for i in idxs: + assert i < size + + node = cell.nodes[i] + + cell.stop_node(node) + + for i in idxs: + node = cell.nodes[i] + cell.wait_node(node) + + +def clean_work_idx_node_on_cell(cells: [IgniteService], idxs: [int]): + """ + Cleaning the working directory on idxs nodes in cells. + """ + for cell in cells: + size = len(cell.nodes) + + for i in idxs: + assert i < size + + node = cell.nodes[i] + + assert not cell.pids(node) + + node.account.ssh(f'rm -rf {cell.work_dir}') + + +def start_idx_node_on_cell(cells: [IgniteService], idxs: [int]): + """ + Start idxs nodes on cells. + """ + for cell in cells: + size = len(cell.nodes) + + for i in idxs: + assert i < size + + node = cell.nodes[i] + + cell.start_node(node) + + for cell in cells: + cell.await_started() + + +def check_pds_size(pds_before, pds_after): + """ + Checks that the size of the pds has become smaller. + """ + for host in pds_after: + assert pds_after[host] < pds_before[host], f'Host {host}: size after = {pds_after[host]}, ' \ + f'size before = {pds_before[host]}.' + + +def fix_data(control_utility: ControlUtility, node, log_dir): + """ + :return: Path to idle-verify dump file. + """ + control_utility.validate_indexes() + control_utility.idle_verify() + + dump = control_utility.idle_verify_dump(node) + + return copy_file_to_dest(node, dump, log_dir) + + +# pylint: disable=R0913 +def start_cell(test_context, config, jvm_opts: None, modules=None, num_nodes=4): + """ + Starts cell. + """ + ignites = IgniteService(test_context, config, modules=modules, num_nodes=num_nodes, jvm_opts=jvm_opts, + startup_timeout_sec=180) + + ignites.start() + + return ignites From 600c0baf827ab4779b6eb1279db083420864a5f6 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Wed, 24 Feb 2021 17:23:22 +0300 Subject: [PATCH 02/21] IGNITE-13508: fix --- .../ducktest/tests/DeleteDataApplication.java | 2 +- .../ignitetest/services/utils/ignite_aware.py | 2 +- .../services/utils/templates/cache_macro.j2 | 2 +- .../tests/two_phased_rebalanced_test.py | 150 +++++++++--------- 4 files changed, 76 insertions(+), 80 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java index d7c74faadf855..f42042220f9fb 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java @@ -78,7 +78,7 @@ public class DeleteDataApplication extends IgniteAwareApplication { futures.add(cache.removeAllAsync(new TreeSet<>(keys.subList(fromIdx, toIdx)))); - fromIdx = toIdx + 1; + fromIdx = toIdx; } futures.forEach(IgniteFuture::get); diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index e3a1fec879555..0c9a57599db58 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -474,7 +474,7 @@ def await_rebalance(self, timeout_sec=180): mbean = JmxClient(node).find_mbean('.*name=cluster') while datetime.now() < delta_time and not rebalanced: - rebalanced = bool(strtobool(next(mbean.Rebalanced))) + rebalanced = strtobool(next(mbean.Rebalanced)) if rebalanced: return diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 index 2ac952523b144..9fbb4d96c563e 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 @@ -41,7 +41,7 @@ {% for class in cache.indexed_types %} - {{ class }} + {{ class }} {% endfor %} diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 9a9858ec95fca..0f0dee591a3ab 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -16,8 +16,12 @@ """ This module contains Cellular Affinity tests. """ -import ducktape +from typing import List +from ducktape import errors +from ducktape.cluster.cluster import ClusterNode from ducktape.mark.resource import cluster +from ignitetest.services.utils.ignite_aware import IgniteAwareService + from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration, Affinity from ignitetest.services.ignite import IgniteService @@ -29,7 +33,15 @@ from ignitetest.services.utils.util import copy_file_to_dest from ignitetest.utils import ignite_versions from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import DEV_BRANCH, IgniteVersion +from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST_2_9 + +NUM_NODES_CELL = 4 + +NUM_CELL = 1 + +ATTRIBUTE = "CELL" + +CACHE_NAME = "test-cache" # pylint: disable=W0223 @@ -37,17 +49,12 @@ class TwoPhasedRebalancedTest(IgniteTest): """ Two-phase rebalancing test case. """ - NUM_NODES_CELL = 4 - - NUM_CELL = 4 - - ATTRIBUTE = "CELL" - - CACHE_NAME = "test-cache" - # pylint: disable=R0914 @cluster(num_nodes=NUM_NODES_CELL * NUM_CELL + 2) - @ignite_versions(str(DEV_BRANCH)) + @ignite_versions( + str(DEV_BRANCH), + str(LATEST_2_9) + ) def two_phased_rebalancing_test(self, ignite_version): """ Test case of two-phase rebalancing. @@ -65,14 +72,14 @@ def two_phased_rebalancing_test(self, ignite_version): 1. Check that PDS size reduced (compare to step 3) 2. Check data consistency (idle_verify --dump) """ - data_storage = DataStorageConfiguration(default=DataRegionConfiguration(persistent=True), - checkpoint_frequency=30000) + config = IgniteConfiguration(version=IgniteVersion(ignite_version), + data_storage=DataStorageConfiguration( + default=DataRegionConfiguration(persistent=True), checkpoint_frequency=30000), + caches=[CacheConfiguration( + name=CACHE_NAME, backups=NUM_NODES_CELL-1, affinity=Affinity())], + metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi') - cells = self.start_cells(ignite_version=ignite_version, - cells_cnt=self.NUM_CELL, - cell_nodes_cnt=self.NUM_NODES_CELL, - cache_name=self.CACHE_NAME, - data_storage=data_storage) + cells = self.start_cells(config) control_utility = ControlUtility(cells[0]) control_utility.activate() @@ -102,68 +109,51 @@ def two_phased_rebalancing_test(self, ignite_version): streamer.run() node = cells[0].nodes[0] + try: - cells[0].await_event_on_node('Skipping checkpoint', node, timeout_sec=60) - except ducktape.errors.TimeoutError as ex: + IgniteAwareService.await_event_on_node('Skipping checkpoint', node, timeout_sec=60) + except errors.TimeoutError as ex: self.logger.warn(ex) self.fix_pds_size(cells, "Step prepare, load data. PDS.") deleter.run() - try: - cells[0].await_event_on_node('Skipping checkpoint', node, timeout_sec=60) - except ducktape.errors.TimeoutError as ex: - self.logger.warn(ex) - dump_1 = fix_data(control_utility, node, cells[0].log_dir) pds_before = self.fix_pds_size(cells, "After Delete 80%, PDS.") restart_with_clean_idx_node_on_cell(cells, [0, 1]) - for cell in cells: - cell.await_rebalance(timeout_sec=15 * 60) - self.fix_pds_size(cells, "After rebalancing complate on nodes 0, 1. PDS.") restart_with_clean_idx_node_on_cell(cells, [2, 3]) - try: - cells[0].await_event_on_node('Skipping checkpoint', node, timeout_sec=60) - except ducktape.errors.TimeoutError as ex: - self.logger.warn(ex) - - for cell in cells: - cell.await_rebalance(timeout_sec=15 * 60) - pds_after = self.fix_pds_size(cells, "After rebalancing complate on nodes 2, 3. PDS.") - check_pds_size(pds_before, pds_after) + for host in pds_after: + assert pds_after[host] < pds_before[host], f'Host {host}: size after = {pds_after[host]}, ' \ + f'size before = {pds_before[host]}.' dump_2 = fix_data(control_utility, node, cells[0].log_dir) diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True) - assert not diff + assert not diff, diff - # pylint: disable=R0913 - def start_cells(self, ignite_version: str, cells_cnt: int, cell_nodes_cnt: int, cache_name: str, - data_storage: DataStorageConfiguration = None): + def start_cells(self, config: IgniteConfiguration, cells_cnt: int = NUM_CELL, cell_nodes_cnt: int = NUM_NODES_CELL)\ + -> List[IgniteService]: """ Start cells. + :param config IgniteConfiguration. + :param cells_cnt Cells size. + :param cell_nodes_cnt Nodes on cell. + :return List of IgniteServices """ assert cells_cnt > 0 cells = [] - cache_cfg = CacheConfiguration(name=cache_name, backups=self.NUM_NODES_CELL-1, affinity=Affinity(), - atomicity_mode='TRANSACTIONAL') - - config = IgniteConfiguration(version=IgniteVersion(ignite_version), data_storage=data_storage, - caches=[cache_cfg], - metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi') - - cell = start_cell(self.test_context, config, [f'-D{self.ATTRIBUTE}=0'], num_nodes=cell_nodes_cnt,) + cell = start_cell(self.test_context, config, cell_nodes_cnt, [f'-D{ATTRIBUTE}=0']) discovery_spi = from_ignite_cluster(cell) config = config._replace(discovery_spi=discovery_spi) @@ -172,45 +162,52 @@ def start_cells(self, ignite_version: str, cells_cnt: int, cell_nodes_cnt: int, if cells_cnt > 1: for i in range(1, cells_cnt): - cells.append(start_cell(self.test_context, config, [f'-D{self.ATTRIBUTE}={i}'], - num_nodes=cell_nodes_cnt)) + cells.append(start_cell(self.test_context, config, cell_nodes_cnt, [f'-D{ATTRIBUTE}={i}'])) return cells - def fix_pds_size(self, cells: [IgniteService], msg: str): + def fix_pds_size(self, cells: [IgniteService], msg: str) -> dict: """ Pds size in megabytes. + :param cells List of IgniteService + :param msg Information message """ assert len(cells) > 0 res = {} - for cell in cells: for node in cell.nodes: consistent_id = str(node.account.hostname).replace('.', '_').replace('-', '_') cmd = f'du -sm {cell.database_dir}/{consistent_id} | ' + "awk '{print $1}'" res[node.account.hostname] = int(node.account.ssh_output(cmd).decode("utf-8").rstrip()) - self.logger.warn(msg) + self.logger.info(msg) for item in res.items(): - self.logger.warn(f'Host: {item[0]}, PDS {item[1]}mb') + self.logger.info(f'Host: {item[0]}, PDS {item[1]}mb') return res def restart_with_clean_idx_node_on_cell(cells: [IgniteService], idxs: [int]): """ - Restart idxs nodes on cells with cleaning working directory. + Restart idxs nodes on cells with cleaning working directory and await rebalance. + :param cells List of IgniteService + :param idxs List the index nodes that need to be restarted with cleanup """ stop_idx_node_on_cell(cells, idxs) clean_work_idx_node_on_cell(cells, idxs) start_idx_node_on_cell(cells, idxs) + for cell in cells: + cell.await_rebalance(timeout_sec=10 * 60) + def stop_idx_node_on_cell(cells: [IgniteService], idxs: [int]): """ Stop idxs nodes on cells. + :param cells List of IgniteService + :param idxs List of index nodes to stop """ for cell in cells: size = len(cell.nodes) @@ -218,18 +215,18 @@ def stop_idx_node_on_cell(cells: [IgniteService], idxs: [int]): for i in idxs: assert i < size - node = cell.nodes[i] - - cell.stop_node(node) + cell.stop_node(cell.nodes[i]) + for cell in cells: for i in idxs: - node = cell.nodes[i] - cell.wait_node(node) + cell.wait_node(cell.nodes[i]) def clean_work_idx_node_on_cell(cells: [IgniteService], idxs: [int]): """ Cleaning the working directory on idxs nodes in cells. + :param cells List of IgniteService + :param idxs List of index nodes to clean """ for cell in cells: size = len(cell.nodes) @@ -247,6 +244,8 @@ def clean_work_idx_node_on_cell(cells: [IgniteService], idxs: [int]): def start_idx_node_on_cell(cells: [IgniteService], idxs: [int]): """ Start idxs nodes on cells. + :param cells List of IgniteService + :param idxs List of index nodes to start """ for cell in cells: size = len(cell.nodes) @@ -262,20 +261,14 @@ def start_idx_node_on_cell(cells: [IgniteService], idxs: [int]): cell.await_started() -def check_pds_size(pds_before, pds_after): - """ - Checks that the size of the pds has become smaller. - """ - for host in pds_after: - assert pds_after[host] < pds_before[host], f'Host {host}: size after = {pds_after[host]}, ' \ - f'size before = {pds_before[host]}.' - - -def fix_data(control_utility: ControlUtility, node, log_dir): +def fix_data(control_utility: ControlUtility, node: ClusterNode, log_dir: str) -> str: """ + Start cell. + :param control_utility ControlUtility. + :param node ClusterNode. + :param log_dir Path to log directory. :return: Path to idle-verify dump file. """ - control_utility.validate_indexes() control_utility.idle_verify() dump = control_utility.idle_verify_dump(node) @@ -283,13 +276,16 @@ def fix_data(control_utility: ControlUtility, node, log_dir): return copy_file_to_dest(node, dump, log_dir) -# pylint: disable=R0913 -def start_cell(test_context, config, jvm_opts: None, modules=None, num_nodes=4): +def start_cell(test_context, config: IgniteConfiguration, num_nodes: int, jvm_opts: list) -> IgniteService: """ - Starts cell. + Start cell. + :param test_context Context. + :param config IgniteConfig. + :param num_nodes Number nodes. + :param jvm_opts: List JVM options. + :return IgniteService. """ - ignites = IgniteService(test_context, config, modules=modules, num_nodes=num_nodes, jvm_opts=jvm_opts, - startup_timeout_sec=180) + ignites = IgniteService(test_context, config, num_nodes=num_nodes, jvm_opts=jvm_opts) ignites.start() From ecf4b1ddffdcb1f2fd363c433438b046a02da8de Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Wed, 24 Feb 2021 19:10:32 +0300 Subject: [PATCH 03/21] IGNITE-13508: fix --- .../tests/two_phased_rebalanced_test.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 0f0dee591a3ab..05ebe7172ed07 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -37,7 +37,7 @@ NUM_NODES_CELL = 4 -NUM_CELL = 1 +NUM_CELL = 4 ATTRIBUTE = "CELL" @@ -51,10 +51,7 @@ class TwoPhasedRebalancedTest(IgniteTest): """ # pylint: disable=R0914 @cluster(num_nodes=NUM_NODES_CELL * NUM_CELL + 2) - @ignite_versions( - str(DEV_BRANCH), - str(LATEST_2_9) - ) + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_9)) def two_phased_rebalancing_test(self, ignite_version): """ Test case of two-phase rebalancing. @@ -110,15 +107,14 @@ def two_phased_rebalancing_test(self, ignite_version): node = cells[0].nodes[0] - try: - IgniteAwareService.await_event_on_node('Skipping checkpoint', node, timeout_sec=60) - except errors.TimeoutError as ex: - self.logger.warn(ex) + self.await_skipping_checkpoint(node) self.fix_pds_size(cells, "Step prepare, load data. PDS.") deleter.run() + self.await_skipping_checkpoint(node) + dump_1 = fix_data(control_utility, node, cells[0].log_dir) pds_before = self.fix_pds_size(cells, "After Delete 80%, PDS.") @@ -188,6 +184,16 @@ def fix_pds_size(self, cells: [IgniteService], msg: str) -> dict: return res + def await_skipping_checkpoint(self, node: ClusterNode): + """ + Await Skipping checkpoint. + :param node ClusterNode + """ + try: + IgniteAwareService.await_event_on_node('Skipping checkpoint', node, timeout_sec=60) + except errors.TimeoutError as ex: + self.logger.warn(ex) + def restart_with_clean_idx_node_on_cell(cells: [IgniteService], idxs: [int]): """ From 9a642b2f11eafa751e24302e3f8b420c828f1d84 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Thu, 25 Feb 2021 12:59:36 +0300 Subject: [PATCH 04/21] IGNITE-13508: fix pydoc --- .../ducktests/tests/ignitetest/services/utils/ignite_aware.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 0c9a57599db58..c02b4a4d2de03 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -464,6 +464,9 @@ def restore_from_snapshot(self, snapshot_name: str): def await_rebalance(self, timeout_sec=180): """ Waiting for the rebalance to complete. + For the method, you need to set the + metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi' + to the config. :param timeout_sec: timeout to wait the rebalance to complete. """ delta_time = datetime.now() + timedelta(seconds=timeout_sec) From 7257a290e3b7afc30553d53671b75120b811cf3b Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Thu, 25 Feb 2021 13:49:46 +0300 Subject: [PATCH 05/21] IGNITE-13508: fix pydoc --- .../tests/ignitetest/tests/two_phased_rebalanced_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 05ebe7172ed07..50d596f57d8a2 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -269,7 +269,7 @@ def start_idx_node_on_cell(cells: [IgniteService], idxs: [int]): def fix_data(control_utility: ControlUtility, node: ClusterNode, log_dir: str) -> str: """ - Start cell. + Creates a dump file and copies it to the log directory. :param control_utility ControlUtility. :param node ClusterNode. :param log_dir Path to log directory. From df0aef6d3a0bc64bde32fd2f45653c906945cf08 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Tue, 2 Mar 2021 13:48:43 +0300 Subject: [PATCH 06/21] IGNITE-13508: fix comments --- .../tests/ignitetest/services/utils/ignite_aware.py | 3 +-- modules/ducktests/tests/ignitetest/services/utils/util.py | 2 +- .../tests/ignitetest/tests/two_phased_rebalanced_test.py | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index c02b4a4d2de03..f4ebfc8172a6e 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -23,7 +23,6 @@ import time from abc import abstractmethod, ABCMeta from datetime import datetime, timedelta -from distutils.util import strtobool from enum import IntEnum from threading import Thread @@ -477,7 +476,7 @@ def await_rebalance(self, timeout_sec=180): mbean = JmxClient(node).find_mbean('.*name=cluster') while datetime.now() < delta_time and not rebalanced: - rebalanced = strtobool(next(mbean.Rebalanced)) + rebalanced = next(mbean.Rebalanced) == 'true' if rebalanced: return diff --git a/modules/ducktests/tests/ignitetest/services/utils/util.py b/modules/ducktests/tests/ignitetest/services/utils/util.py index a3530bb8c9736..088a4a4eb008f 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/util.py +++ b/modules/ducktests/tests/ignitetest/services/utils/util.py @@ -21,7 +21,7 @@ def copy_file_to_dest(node, file_path: str, dest_dir: str): """ - Move file to logs directory. + Copy file to destination directory. :return new path to file. """ node.account.ssh_output(f'cp {file_path} {dest_dir}') diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 50d596f57d8a2..1b36400c5f259 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -33,7 +33,7 @@ from ignitetest.services.utils.util import copy_file_to_dest from ignitetest.utils import ignite_versions from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST_2_9 +from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST_2_9, LATEST_2_8 NUM_NODES_CELL = 4 @@ -51,7 +51,7 @@ class TwoPhasedRebalancedTest(IgniteTest): """ # pylint: disable=R0914 @cluster(num_nodes=NUM_NODES_CELL * NUM_CELL + 2) - @ignite_versions(str(DEV_BRANCH), str(LATEST_2_9)) + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_9), str(LATEST_2_8)) def two_phased_rebalancing_test(self, ignite_version): """ Test case of two-phase rebalancing. @@ -134,7 +134,7 @@ def two_phased_rebalancing_test(self, ignite_version): dump_2 = fix_data(control_utility, node, cells[0].log_dir) diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True) - assert not diff, diff + assert not diff, f"Validation error, files are different. Difference:: \n {diff}" def start_cells(self, config: IgniteConfiguration, cells_cnt: int = NUM_CELL, cell_nodes_cnt: int = NUM_NODES_CELL)\ -> List[IgniteService]: @@ -206,7 +206,7 @@ def restart_with_clean_idx_node_on_cell(cells: [IgniteService], idxs: [int]): start_idx_node_on_cell(cells, idxs) for cell in cells: - cell.await_rebalance(timeout_sec=10 * 60) + cell.await_rebalance() def stop_idx_node_on_cell(cells: [IgniteService], idxs: [int]): From 233943fe022319624635b24487aa9cf26e7c3bba Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Tue, 2 Mar 2021 18:24:59 +0300 Subject: [PATCH 07/21] IGNITE-13508: fix comments --- .../tests/two_phased_rebalanced_test.py | 90 +++++++++---------- 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 1b36400c5f259..229839b4ef51e 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -64,7 +64,7 @@ def two_phased_rebalancing_test(self, ignite_version): 2. Start cleaned node with preservance of consistent id and cell attributes. 3. Wait for the rebalance to complete. Phase 2. - Run steps 1-3 of Phase 2 on the other half of the cluster. + Run steps 1-3 of Phase 1 on the other half of the cluster. Verifications. 1. Check that PDS size reduced (compare to step 3) 2. Check data consistency (idle_verify --dump) @@ -76,6 +76,7 @@ def two_phased_rebalancing_test(self, ignite_version): name=CACHE_NAME, backups=NUM_NODES_CELL-1, affinity=Affinity())], metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi') + # Start 4 cells. cells = self.start_cells(config) control_utility = ControlUtility(cells[0]) @@ -85,7 +86,8 @@ def two_phased_rebalancing_test(self, ignite_version): version=IgniteVersion(ignite_version), discovery_spi=from_ignite_cluster(cells[0])) - streamer = IgniteApplicationService( + # Load data to cache. + IgniteApplicationService( self.test_context, client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.snapshot_test.DataLoaderApplication", @@ -93,80 +95,70 @@ def two_phased_rebalancing_test(self, ignite_version): "cacheName": "test-cache", "interval": 500 * 1024, "valueSizeKb": 1} - ) + ).run() - deleter = IgniteApplicationService( + # Delete 80% of data and fix PDS size on all nodes. + IgniteApplicationService( self.test_context, client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.DeleteDataApplication", shutdown_timeout_sec=15 * 60, params={"cacheName": "test-cache", - "size": 400 * 1024}) - - streamer.run() + "size": 400 * 1024} + ).run() node = cells[0].nodes[0] - self.await_skipping_checkpoint(node) - - self.fix_pds_size(cells, "Step prepare, load data. PDS.") - - deleter.run() - - self.await_skipping_checkpoint(node) - - dump_1 = fix_data(control_utility, node, cells[0].log_dir) - - pds_before = self.fix_pds_size(cells, "After Delete 80%, PDS.") + self.await_cluster_idle(node) - restart_with_clean_idx_node_on_cell(cells, [0, 1]) + dump_1 = create_idle_dump_and_copy_to_log_dir(control_utility, node, cells[0].log_dir) - self.fix_pds_size(cells, "After rebalancing complate on nodes 0, 1. PDS.") + pds_before = self.get_pds_size(cells, "After Delete 80%, PDS.") - restart_with_clean_idx_node_on_cell(cells, [2, 3]) + # Restarting the cluster in twos nodes per a cell with a cleanup and waiting for rebalancing. + restart_with_clean_idx_node_on_cell_and_await_rebalance(cells, [0, 1]) + restart_with_clean_idx_node_on_cell_and_await_rebalance(cells, [2, 3]) - pds_after = self.fix_pds_size(cells, "After rebalancing complate on nodes 2, 3. PDS.") + pds_after = self.get_pds_size(cells, "After rebalancing complete, PDS.") + # Check that PDS size reduced. for host in pds_after: assert pds_after[host] < pds_before[host], f'Host {host}: size after = {pds_after[host]}, ' \ f'size before = {pds_before[host]}.' - dump_2 = fix_data(control_utility, node, cells[0].log_dir) + dump_2 = create_idle_dump_and_copy_to_log_dir(control_utility, node, cells[0].log_dir) + # Check data consistency. diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True) - assert not diff, f"Validation error, files are different. Difference:: \n {diff}" + assert not diff, f"Validation error, files are different. Difference:\n {diff}" - def start_cells(self, config: IgniteConfiguration, cells_cnt: int = NUM_CELL, cell_nodes_cnt: int = NUM_NODES_CELL)\ - -> List[IgniteService]: + def start_cells(self, config: IgniteConfiguration) -> List[IgniteService]: """ Start cells. :param config IgniteConfiguration. - :param cells_cnt Cells size. - :param cell_nodes_cnt Nodes on cell. - :return List of IgniteServices + :return List of IgniteServices. """ - assert cells_cnt > 0 cells = [] - cell = start_cell(self.test_context, config, cell_nodes_cnt, [f'-D{ATTRIBUTE}=0']) + cell = start_cell(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}=0']) discovery_spi = from_ignite_cluster(cell) config = config._replace(discovery_spi=discovery_spi) cells.append(cell) - if cells_cnt > 1: - for i in range(1, cells_cnt): - cells.append(start_cell(self.test_context, config, cell_nodes_cnt, [f'-D{ATTRIBUTE}={i}'])) + for i in range(1, NUM_CELL): + cells.append(start_cell(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}={i}'])) return cells - def fix_pds_size(self, cells: [IgniteService], msg: str) -> dict: + def get_pds_size(self, cells: [IgniteService], msg: str) -> dict: """ Pds size in megabytes. - :param cells List of IgniteService - :param msg Information message + :param cells List of IgniteService. + :param msg Information message. + :return dict with hostname -> pds size in megabytes. """ assert len(cells) > 0 @@ -184,10 +176,10 @@ def fix_pds_size(self, cells: [IgniteService], msg: str) -> dict: return res - def await_skipping_checkpoint(self, node: ClusterNode): + def await_cluster_idle(self, node: ClusterNode): """ Await Skipping checkpoint. - :param node ClusterNode + :param node ClusterNode. """ try: IgniteAwareService.await_event_on_node('Skipping checkpoint', node, timeout_sec=60) @@ -195,11 +187,11 @@ def await_skipping_checkpoint(self, node: ClusterNode): self.logger.warn(ex) -def restart_with_clean_idx_node_on_cell(cells: [IgniteService], idxs: [int]): +def restart_with_clean_idx_node_on_cell_and_await_rebalance(cells: [IgniteService], idxs: [int]): """ Restart idxs nodes on cells with cleaning working directory and await rebalance. - :param cells List of IgniteService - :param idxs List the index nodes that need to be restarted with cleanup + :param cells List of IgniteService. + :param idxs List the index nodes that need to be restarted with cleanup. """ stop_idx_node_on_cell(cells, idxs) clean_work_idx_node_on_cell(cells, idxs) @@ -212,8 +204,8 @@ def restart_with_clean_idx_node_on_cell(cells: [IgniteService], idxs: [int]): def stop_idx_node_on_cell(cells: [IgniteService], idxs: [int]): """ Stop idxs nodes on cells. - :param cells List of IgniteService - :param idxs List of index nodes to stop + :param cells List of IgniteService. + :param idxs List of index nodes to stop. """ for cell in cells: size = len(cell.nodes) @@ -231,8 +223,8 @@ def stop_idx_node_on_cell(cells: [IgniteService], idxs: [int]): def clean_work_idx_node_on_cell(cells: [IgniteService], idxs: [int]): """ Cleaning the working directory on idxs nodes in cells. - :param cells List of IgniteService - :param idxs List of index nodes to clean + :param cells List of IgniteService. + :param idxs List of index nodes to clean. """ for cell in cells: size = len(cell.nodes) @@ -250,8 +242,8 @@ def clean_work_idx_node_on_cell(cells: [IgniteService], idxs: [int]): def start_idx_node_on_cell(cells: [IgniteService], idxs: [int]): """ Start idxs nodes on cells. - :param cells List of IgniteService - :param idxs List of index nodes to start + :param cells List of IgniteService. + :param idxs List of index nodes to start. """ for cell in cells: size = len(cell.nodes) @@ -267,7 +259,7 @@ def start_idx_node_on_cell(cells: [IgniteService], idxs: [int]): cell.await_started() -def fix_data(control_utility: ControlUtility, node: ClusterNode, log_dir: str) -> str: +def create_idle_dump_and_copy_to_log_dir(control_utility: ControlUtility, node: ClusterNode, log_dir: str) -> str: """ Creates a dump file and copies it to the log directory. :param control_utility ControlUtility. From 5eb455bad2397f4f17bfc7eaf062a0862a273b51 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Wed, 10 Mar 2021 00:19:57 +0300 Subject: [PATCH 08/21] IGNITE-13508: added indexes --- .../tests/ignitetest/tests/two_phased_rebalanced_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 229839b4ef51e..095375b696b58 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -73,7 +73,8 @@ def two_phased_rebalancing_test(self, ignite_version): data_storage=DataStorageConfiguration( default=DataRegionConfiguration(persistent=True), checkpoint_frequency=30000), caches=[CacheConfiguration( - name=CACHE_NAME, backups=NUM_NODES_CELL-1, affinity=Affinity())], + name=CACHE_NAME, backups=NUM_NODES_CELL-1, affinity=Affinity(), + indexed_types=['java.lang.Long', 'byte[]'])], metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi') # Start 4 cells. @@ -126,6 +127,7 @@ def two_phased_rebalancing_test(self, ignite_version): assert pds_after[host] < pds_before[host], f'Host {host}: size after = {pds_after[host]}, ' \ f'size before = {pds_before[host]}.' + control_utility.validate_indexes() dump_2 = create_idle_dump_and_copy_to_log_dir(control_utility, node, cells[0].log_dir) # Check data consistency. From 46ce2eb08e71ecfd20efa652ff2894a886a7ace9 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Wed, 10 Mar 2021 16:17:19 +0300 Subject: [PATCH 09/21] IGNITE-13508: added await_cluster_idle after rebalance --- .../tests/ignitetest/tests/two_phased_rebalanced_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 095375b696b58..817a337894433 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -120,6 +120,8 @@ def two_phased_rebalancing_test(self, ignite_version): restart_with_clean_idx_node_on_cell_and_await_rebalance(cells, [0, 1]) restart_with_clean_idx_node_on_cell_and_await_rebalance(cells, [2, 3]) + self.await_cluster_idle(node) + pds_after = self.get_pds_size(cells, "After rebalancing complete, PDS.") # Check that PDS size reduced. From 3a9ef2b56b640328c76b0b2f2b9b4e4b610dcfde Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Mon, 15 Mar 2021 10:38:37 +0300 Subject: [PATCH 10/21] IGNITE-13508: fix comments --- .../ducktest/tests/DeleteDataApplication.java | 10 +- modules/ducktests/tests/docker/run_tests.sh | 2 +- .../utils/ignite_configuration/cache.py | 2 +- .../services/utils/templates/cache_macro.j2 | 2 +- .../tests/two_phased_rebalanced_test.py | 94 ++++++++----------- 5 files changed, 46 insertions(+), 64 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java index f42042220f9fb..4f72cb1a88f62 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java @@ -21,8 +21,8 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Optional; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import javax.cache.Cache; import com.fasterxml.jackson.databind.JsonNode; @@ -40,9 +40,7 @@ public class DeleteDataApplication extends IgniteAwareApplication { int size = jNode.get("size").asInt(); - int bachSize = Optional.ofNullable(jNode.get("bachSize")) - .map(JsonNode::asInt) - .orElse(1000); + int batchSize = jNode.get("batchSize").asInt(); IgniteCache cache = ignite.getOrCreateCache(cacheName); @@ -74,7 +72,7 @@ public class DeleteDataApplication extends IgniteAwareApplication { int toIdx = 0; while (fromIdx < listSize) { - toIdx = Math.min(fromIdx + bachSize, listSize); + toIdx = Math.min(fromIdx + batchSize, listSize); futures.add(cache.removeAllAsync(new TreeSet<>(keys.subList(fromIdx, toIdx)))); @@ -85,7 +83,7 @@ public class DeleteDataApplication extends IgniteAwareApplication { log.info("Cache size after: " + cache.size()); - recordResult("DURATION", System.currentTimeMillis() - start); + recordResult("DURATION_SECONDS", TimeUnit.MILLISECONDS.toSeconds( System.currentTimeMillis() - start)); markFinished(); } diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh index 6ff723cadccd7..2c06c007d5b07 100755 --- a/modules/ducktests/tests/docker/run_tests.sh +++ b/modules/ducktests/tests/docker/run_tests.sh @@ -21,7 +21,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # DuckerUp parameters are specified with env variables # Num of cotainers that ducktape will prepare for tests -IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-13} +IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-18} # Image name to run nodes JDK_VERSION="${JDK_VERSION:-8}" diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py index 176931e24b8cd..b094d7bba2a67 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py @@ -27,7 +27,7 @@ class Affinity(NamedTuple): Affinity. """ name: str = AFFINITY_BACKUP_FILTER - constructor_arg: str = CELL + attr_name: str = CELL class CacheConfiguration(NamedTuple): diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 index 9fbb4d96c563e..abab4631bec39 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 @@ -20,7 +20,7 @@ - + diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 817a337894433..ce5daa73eb68f 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -50,15 +50,15 @@ class TwoPhasedRebalancedTest(IgniteTest): Two-phase rebalancing test case. """ # pylint: disable=R0914 - @cluster(num_nodes=NUM_NODES_CELL * NUM_CELL + 2) + @cluster(num_nodes=(NUM_NODES_CELL * NUM_CELL) + 1) @ignite_versions(str(DEV_BRANCH), str(LATEST_2_9), str(LATEST_2_8)) def two_phased_rebalancing_test(self, ignite_version): """ Test case of two-phase rebalancing. Preparations. 1. Start 4 cells. - 2. Load data to cache with the mentioned above affinity function and fix PDS size on all nodes. - 3. Delete 80% of data and fix PDS size on all nodes. + 2. Load data to cache with the mentioned above affinity function. + 3. Delete 80% of data and measure PDS size on all nodes. Phase 1. 1. Stop two nodes in each cell, total a half of all nodes and clean PDS. 2. Start cleaned node with preservance of consistent id and cell attributes. @@ -73,7 +73,7 @@ def two_phased_rebalancing_test(self, ignite_version): data_storage=DataStorageConfiguration( default=DataRegionConfiguration(persistent=True), checkpoint_frequency=30000), caches=[CacheConfiguration( - name=CACHE_NAME, backups=NUM_NODES_CELL-1, affinity=Affinity(), + name=CACHE_NAME, backups=2, affinity=Affinity(), indexed_types=['java.lang.Long', 'byte[]'])], metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi') @@ -88,7 +88,7 @@ def two_phased_rebalancing_test(self, ignite_version): discovery_spi=from_ignite_cluster(cells[0])) # Load data to cache. - IgniteApplicationService( + app = IgniteApplicationService( self.test_context, client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.snapshot_test.DataLoaderApplication", @@ -96,21 +96,26 @@ def two_phased_rebalancing_test(self, ignite_version): "cacheName": "test-cache", "interval": 500 * 1024, "valueSizeKb": 1} - ).run() + ) + app.run() + app.free() - # Delete 80% of data and fix PDS size on all nodes. - IgniteApplicationService( + # Delete 80% of data and measure PDS size on all nodes. + app = IgniteApplicationService( self.test_context, client_config, java_class_name="org.apache.ignite.internal.ducktest.tests.DeleteDataApplication", - shutdown_timeout_sec=15 * 60, + shutdown_timeout_sec=5 * 60, params={"cacheName": "test-cache", - "size": 400 * 1024} - ).run() + "size": 400 * 1024, + "batchSize": 1000} + ) + app.start(clean=False) + app.stop() node = cells[0].nodes[0] - self.await_cluster_idle(node) + await_cluster_idle(node) dump_1 = create_idle_dump_and_copy_to_log_dir(control_utility, node, cells[0].log_dir) @@ -120,7 +125,7 @@ def two_phased_rebalancing_test(self, ignite_version): restart_with_clean_idx_node_on_cell_and_await_rebalance(cells, [0, 1]) restart_with_clean_idx_node_on_cell_and_await_rebalance(cells, [2, 3]) - self.await_cluster_idle(node) + await_cluster_idle(node) pds_after = self.get_pds_size(cells, "After rebalancing complete, PDS.") @@ -145,15 +150,21 @@ def start_cells(self, config: IgniteConfiguration) -> List[IgniteService]: cells = [] - cell = start_cell(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}=0']) + first = IgniteService(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}=0'], + startup_timeout_sec=180) + first.start() - discovery_spi = from_ignite_cluster(cell) + discovery_spi = from_ignite_cluster(first) config = config._replace(discovery_spi=discovery_spi) - cells.append(cell) + cells.append(first) for i in range(1, NUM_CELL): - cells.append(start_cell(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}={i}'])) + cell = IgniteService(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}={i}'], + startup_timeout_sec=180,) + cell.start() + + cells.append(cell) return cells @@ -164,7 +175,6 @@ def get_pds_size(self, cells: [IgniteService], msg: str) -> dict: :param msg Information message. :return dict with hostname -> pds size in megabytes. """ - assert len(cells) > 0 res = {} for cell in cells: @@ -180,15 +190,17 @@ def get_pds_size(self, cells: [IgniteService], msg: str) -> dict: return res - def await_cluster_idle(self, node: ClusterNode): - """ - Await Skipping checkpoint. - :param node ClusterNode. - """ - try: - IgniteAwareService.await_event_on_node('Skipping checkpoint', node, timeout_sec=60) - except errors.TimeoutError as ex: - self.logger.warn(ex) + +def await_cluster_idle(node: ClusterNode, timeout_sec=30): + """ + Await Skipping checkpoint. + :param node ClusterNode. + :param timeout_sec Number of seconds to await event. + """ + try: + IgniteAwareService.await_event_on_node('Skipping checkpoint', node, timeout_sec=timeout_sec) + except errors.TimeoutError: + pass def restart_with_clean_idx_node_on_cell_and_await_rebalance(cells: [IgniteService], idxs: [int]): @@ -212,11 +224,7 @@ def stop_idx_node_on_cell(cells: [IgniteService], idxs: [int]): :param idxs List of index nodes to stop. """ for cell in cells: - size = len(cell.nodes) - for i in idxs: - assert i < size - cell.stop_node(cell.nodes[i]) for cell in cells: @@ -231,11 +239,7 @@ def clean_work_idx_node_on_cell(cells: [IgniteService], idxs: [int]): :param idxs List of index nodes to clean. """ for cell in cells: - size = len(cell.nodes) - for i in idxs: - assert i < size - node = cell.nodes[i] assert not cell.pids(node) @@ -250,11 +254,7 @@ def start_idx_node_on_cell(cells: [IgniteService], idxs: [int]): :param idxs List of index nodes to start. """ for cell in cells: - size = len(cell.nodes) - for i in idxs: - assert i < size - node = cell.nodes[i] cell.start_node(node) @@ -276,19 +276,3 @@ def create_idle_dump_and_copy_to_log_dir(control_utility: ControlUtility, node: dump = control_utility.idle_verify_dump(node) return copy_file_to_dest(node, dump, log_dir) - - -def start_cell(test_context, config: IgniteConfiguration, num_nodes: int, jvm_opts: list) -> IgniteService: - """ - Start cell. - :param test_context Context. - :param config IgniteConfig. - :param num_nodes Number nodes. - :param jvm_opts: List JVM options. - :return IgniteService. - """ - ignites = IgniteService(test_context, config, num_nodes=num_nodes, jvm_opts=jvm_opts) - - ignites.start() - - return ignites From bb49d938680bd025e588e444d9f6e40ac714523c Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Mon, 15 Mar 2021 11:00:17 +0300 Subject: [PATCH 11/21] IGNITE-13508: set NUM_CELL=2 --- modules/ducktests/tests/docker/run_tests.sh | 2 +- .../tests/ignitetest/tests/two_phased_rebalanced_test.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ducktests/tests/docker/run_tests.sh b/modules/ducktests/tests/docker/run_tests.sh index 2c06c007d5b07..6ff723cadccd7 100755 --- a/modules/ducktests/tests/docker/run_tests.sh +++ b/modules/ducktests/tests/docker/run_tests.sh @@ -21,7 +21,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # DuckerUp parameters are specified with env variables # Num of cotainers that ducktape will prepare for tests -IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-18} +IGNITE_NUM_CONTAINERS=${IGNITE_NUM_CONTAINERS:-13} # Image name to run nodes JDK_VERSION="${JDK_VERSION:-8}" diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index ce5daa73eb68f..cb9dcc6a44637 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -37,7 +37,7 @@ NUM_NODES_CELL = 4 -NUM_CELL = 4 +NUM_CELL = 2 ATTRIBUTE = "CELL" @@ -56,7 +56,7 @@ def two_phased_rebalancing_test(self, ignite_version): """ Test case of two-phase rebalancing. Preparations. - 1. Start 4 cells. + 1. Start 2 cells. 2. Load data to cache with the mentioned above affinity function. 3. Delete 80% of data and measure PDS size on all nodes. Phase 1. @@ -77,7 +77,7 @@ def two_phased_rebalancing_test(self, ignite_version): indexed_types=['java.lang.Long', 'byte[]'])], metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi') - # Start 4 cells. + # Start 2 cells. cells = self.start_cells(config) control_utility = ControlUtility(cells[0]) From 40d61a5a32b26e9e8d02c5b1a1fd7607f48fa05a Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Mon, 15 Mar 2021 12:50:47 +0300 Subject: [PATCH 12/21] IGNITE-13508: optimize cache_macro a bit --- .../tests/ignitetest/services/utils/templates/cache_macro.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 index abab4631bec39..d10cf127e8fee 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2 @@ -33,9 +33,9 @@ {% for cache in caches %} - + - + {% if cache.indexed_types %} From 33562ed8b684768c97a47ef3181f7da0578de184 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Mon, 15 Mar 2021 19:20:26 +0300 Subject: [PATCH 13/21] IGNITE-13508: added use globals cluster_size --- .../tests/two_phased_rebalanced_test.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index cb9dcc6a44637..dad1b89740f78 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -16,6 +16,7 @@ """ This module contains Cellular Affinity tests. """ +import math from typing import List from ducktape import errors from ducktape.cluster.cluster import ClusterNode @@ -56,7 +57,7 @@ def two_phased_rebalancing_test(self, ignite_version): """ Test case of two-phase rebalancing. Preparations. - 1. Start 2 cells. + 1. Start cells. 2. Load data to cache with the mentioned above affinity function. 3. Delete 80% of data and measure PDS size on all nodes. Phase 1. @@ -77,8 +78,12 @@ def two_phased_rebalancing_test(self, ignite_version): indexed_types=['java.lang.Long', 'byte[]'])], metric_exporter='org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi') - # Start 2 cells. - cells = self.start_cells(config) + cluster_size = len(self.test_context.cluster) + + num_cell = math.floor((cluster_size - 1) / NUM_NODES_CELL) + + # Start cells. + cells = self.start_cells(config, num_cell) control_utility = ControlUtility(cells[0]) control_utility.activate() @@ -141,10 +146,11 @@ def two_phased_rebalancing_test(self, ignite_version): diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True) assert not diff, f"Validation error, files are different. Difference:\n {diff}" - def start_cells(self, config: IgniteConfiguration) -> List[IgniteService]: + def start_cells(self, config: IgniteConfiguration, num_cell: int) -> List[IgniteService]: """ Start cells. :param config IgniteConfiguration. + :param num_cell Number of cell. :return List of IgniteServices. """ @@ -159,7 +165,7 @@ def start_cells(self, config: IgniteConfiguration) -> List[IgniteService]: cells.append(first) - for i in range(1, NUM_CELL): + for i in range(1, num_cell): cell = IgniteService(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}={i}'], startup_timeout_sec=180,) cell.start() From 0290e1c0e40150975420164253129d3ce62746cd Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Tue, 16 Mar 2021 01:54:30 +0300 Subject: [PATCH 14/21] IGNITE-13508: fix comments --- .../ducktest/tests/DeleteDataApplication.java | 25 +++++++------------ .../tests/two_phased_rebalanced_test.py | 6 ++--- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java index 4f72cb1a88f62..b38ee660bd7e5 100644 --- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java +++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DeleteDataApplication.java @@ -21,6 +21,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Optional; import java.util.TreeSet; import java.util.concurrent.TimeUnit; import javax.cache.Cache; @@ -40,7 +41,9 @@ public class DeleteDataApplication extends IgniteAwareApplication { int size = jNode.get("size").asInt(); - int batchSize = jNode.get("batchSize").asInt(); + int batchSize = Optional.ofNullable(jNode.get("batchSize")) + .map(JsonNode::asInt) + .orElse(1000); IgniteCache cache = ignite.getOrCreateCache(cacheName); @@ -54,32 +57,22 @@ public class DeleteDataApplication extends IgniteAwareApplication { ArrayList keys = new ArrayList<>(size); - int cnt = 0; - - while (iter.hasNext() && cnt < size) { + for (int cnt = 0; iter.hasNext() && cnt < size; cnt++) keys.add(iter.next().getKey()); - cnt++; - } - log.info("Start removing: " + keys.size()); int listSize = keys.size(); List> futures = new LinkedList<>(); - int fromIdx = 0; - int toIdx = 0; - - while (fromIdx < listSize) { - toIdx = Math.min(fromIdx + batchSize, listSize); - - futures.add(cache.removeAllAsync(new TreeSet<>(keys.subList(fromIdx, toIdx)))); + for (int from = 0; from < listSize; from += batchSize) { + int to = Math.min(from + batchSize, listSize); - fromIdx = toIdx; + futures.add(cache.removeAllAsync(new TreeSet<>(keys.subList(from, to)))); } - futures.forEach(IgniteFuture::get); + futures.forEach(f -> f.get(TimeUnit.MINUTES.toMillis(5))); log.info("Cache size after: " + cache.size()); diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index dad1b89740f78..8b4f4fdc8da48 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -20,7 +20,6 @@ from typing import List from ducktape import errors from ducktape.cluster.cluster import ClusterNode -from ducktape.mark.resource import cluster from ignitetest.services.utils.ignite_aware import IgniteAwareService from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration, Affinity @@ -32,7 +31,7 @@ from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.services.utils.util import copy_file_to_dest -from ignitetest.utils import ignite_versions +from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST_2_9, LATEST_2_8 @@ -112,8 +111,7 @@ def two_phased_rebalancing_test(self, ignite_version): java_class_name="org.apache.ignite.internal.ducktest.tests.DeleteDataApplication", shutdown_timeout_sec=5 * 60, params={"cacheName": "test-cache", - "size": 400 * 1024, - "batchSize": 1000} + "size": 400 * 1024} ) app.start(clean=False) app.stop() From 2d2b88e62e6c12d6b48bcbd0a6eddf7876b39fcb Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Thu, 18 Mar 2021 11:26:39 +0300 Subject: [PATCH 15/21] IGNITE-13508: fix comments --- .../ducktests/tests/ignitetest/services/utils/ignite_aware.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index f4ebfc8172a6e..f128087756c03 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -468,6 +468,8 @@ def await_rebalance(self, timeout_sec=180): to the config. :param timeout_sec: timeout to wait the rebalance to complete. """ + assert self.nodes, 'Node list is empty.' + delta_time = datetime.now() + timedelta(seconds=timeout_sec) rebalanced = False From 1f396d2f695f9de14b0ed143cb0ef0bd8f00d34d Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Thu, 18 Mar 2021 12:01:33 +0300 Subject: [PATCH 16/21] IGNITE-13508: fix comments --- .../ducktests/tests/ignitetest/services/utils/ignite_aware.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index f128087756c03..ee0989fa1e5fb 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -472,7 +472,6 @@ def await_rebalance(self, timeout_sec=180): delta_time = datetime.now() + timedelta(seconds=timeout_sec) - rebalanced = False for node in self.nodes: rebalanced = False mbean = JmxClient(node).find_mbean('.*name=cluster') From 987caba81596836591b3c44df8dd6475a8716b18 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Mon, 22 Mar 2021 02:33:05 +0300 Subject: [PATCH 17/21] IGNITE-13508: fix comments --- .../tests/ignitetest/services/utils/util.py | 31 ------------------- .../tests/two_phased_rebalanced_test.py | 6 ++-- 2 files changed, 4 insertions(+), 33 deletions(-) delete mode 100644 modules/ducktests/tests/ignitetest/services/utils/util.py diff --git a/modules/ducktests/tests/ignitetest/services/utils/util.py b/modules/ducktests/tests/ignitetest/services/utils/util.py deleted file mode 100644 index 088a4a4eb008f..0000000000000 --- a/modules/ducktests/tests/ignitetest/services/utils/util.py +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This module contains utility methods. -""" -import os - - -def copy_file_to_dest(node, file_path: str, dest_dir: str): - """ - Copy file to destination directory. - :return new path to file. - """ - node.account.ssh_output(f'cp {file_path} {dest_dir}') - - file_name = os.path.basename(file_path) - - return os.path.join(dest_dir, file_name) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 8b4f4fdc8da48..b19eb57f6eb93 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -17,6 +17,7 @@ This module contains Cellular Affinity tests. """ import math +import os from typing import List from ducktape import errors from ducktape.cluster.cluster import ClusterNode @@ -30,7 +31,6 @@ from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, DataStorageConfiguration from ignitetest.services.utils.ignite_configuration.data_storage import DataRegionConfiguration from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster -from ignitetest.services.utils.util import copy_file_to_dest from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.ignite_test import IgniteTest from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST_2_9, LATEST_2_8 @@ -279,4 +279,6 @@ def create_idle_dump_and_copy_to_log_dir(control_utility: ControlUtility, node: dump = control_utility.idle_verify_dump(node) - return copy_file_to_dest(node, dump, log_dir) + node.account.ssh_output(f'cp {dump} {log_dir}') + + return os.path.join(log_dir, os.path.basename(dump)) From 623409024b56a758d04797086bd674ee4096bc8c Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Mon, 22 Mar 2021 18:42:47 +0300 Subject: [PATCH 18/21] IGNITE-13508: fix comments --- .../ignite_configuration/data_storage.py | 2 +- .../utils/templates/datastorage_macro.j2 | 5 +--- .../tests/two_phased_rebalanced_test.py | 29 ++++++------------- 3 files changed, 11 insertions(+), 25 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py index b455ef186b61b..5f18fc83ce549 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py @@ -36,4 +36,4 @@ class DataStorageConfiguration(NamedTuple): """ default: DataRegionConfiguration = DataRegionConfiguration() regions: list = [] - checkpoint_frequency: int = None + wal_mode: str = "LOG_ONLY" diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 index 44abee31ef3b8..f780d64b3d999 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 +++ b/modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2 @@ -31,10 +31,7 @@ {% endif %} - - {% if config.checkpoint_frequency %} - - {% endif %} + {% endif %} diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index b19eb57f6eb93..1c59ae7bb3374 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -19,9 +19,7 @@ import math import os from typing import List -from ducktape import errors from ducktape.cluster.cluster import ClusterNode -from ignitetest.services.utils.ignite_aware import IgniteAwareService from ignitetest.services.utils.ignite_configuration.cache import CacheConfiguration, Affinity @@ -33,11 +31,11 @@ from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST_2_9, LATEST_2_8 +from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST_2_10, LATEST_2_9, LATEST_2_8 NUM_NODES_CELL = 4 -NUM_CELL = 2 +NUM_CELL = 1 ATTRIBUTE = "CELL" @@ -51,7 +49,7 @@ class TwoPhasedRebalancedTest(IgniteTest): """ # pylint: disable=R0914 @cluster(num_nodes=(NUM_NODES_CELL * NUM_CELL) + 1) - @ignite_versions(str(DEV_BRANCH), str(LATEST_2_9), str(LATEST_2_8)) + @ignite_versions(str(DEV_BRANCH), str(LATEST_2_10), str(LATEST_2_9), str(LATEST_2_8)) def two_phased_rebalancing_test(self, ignite_version): """ Test case of two-phase rebalancing. @@ -71,7 +69,8 @@ def two_phased_rebalancing_test(self, ignite_version): """ config = IgniteConfiguration(version=IgniteVersion(ignite_version), data_storage=DataStorageConfiguration( - default=DataRegionConfiguration(persistent=True), checkpoint_frequency=30000), + wal_mode='NONE', + default=DataRegionConfiguration(persistent=True)), caches=[CacheConfiguration( name=CACHE_NAME, backups=2, affinity=Affinity(), indexed_types=['java.lang.Long', 'byte[]'])], @@ -118,7 +117,8 @@ def two_phased_rebalancing_test(self, ignite_version): node = cells[0].nodes[0] - await_cluster_idle(node) + control_utility.deactivate() # flush dirty pages on disk + control_utility.activate() dump_1 = create_idle_dump_and_copy_to_log_dir(control_utility, node, cells[0].log_dir) @@ -128,7 +128,8 @@ def two_phased_rebalancing_test(self, ignite_version): restart_with_clean_idx_node_on_cell_and_await_rebalance(cells, [0, 1]) restart_with_clean_idx_node_on_cell_and_await_rebalance(cells, [2, 3]) - await_cluster_idle(node) + control_utility.deactivate() # flush dirty pages on disk + control_utility.activate() pds_after = self.get_pds_size(cells, "After rebalancing complete, PDS.") @@ -195,18 +196,6 @@ def get_pds_size(self, cells: [IgniteService], msg: str) -> dict: return res -def await_cluster_idle(node: ClusterNode, timeout_sec=30): - """ - Await Skipping checkpoint. - :param node ClusterNode. - :param timeout_sec Number of seconds to await event. - """ - try: - IgniteAwareService.await_event_on_node('Skipping checkpoint', node, timeout_sec=timeout_sec) - except errors.TimeoutError: - pass - - def restart_with_clean_idx_node_on_cell_and_await_rebalance(cells: [IgniteService], idxs: [int]): """ Restart idxs nodes on cells with cleaning working directory and await rebalance. From 78df8d195b23682eb7ccfcd44a89016823b4ae81 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Mon, 22 Mar 2021 19:41:48 +0300 Subject: [PATCH 19/21] IGNITE-13508: fix comments --- .../tests/ignitetest/tests/two_phased_rebalanced_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 1c59ae7bb3374..15c3b2e90047f 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -35,7 +35,7 @@ NUM_NODES_CELL = 4 -NUM_CELL = 1 +NUM_CELL = 2 ATTRIBUTE = "CELL" @@ -138,8 +138,8 @@ def two_phased_rebalancing_test(self, ignite_version): assert pds_after[host] < pds_before[host], f'Host {host}: size after = {pds_after[host]}, ' \ f'size before = {pds_before[host]}.' - control_utility.validate_indexes() dump_2 = create_idle_dump_and_copy_to_log_dir(control_utility, node, cells[0].log_dir) + control_utility.validate_indexes() # Check data consistency. diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True) From ca3976e0651eced52caac8a0b5fe71501402a361 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Tue, 23 Mar 2021 11:25:27 +0300 Subject: [PATCH 20/21] IGNITE-13508: fix versions --- .../tests/two_phased_rebalanced_test.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index 15c3b2e90047f..f16640663b50e 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -31,7 +31,7 @@ from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST_2_10, LATEST_2_9, LATEST_2_8 +from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST, LATEST_2_9 NUM_NODES_CELL = 4 @@ -49,7 +49,7 @@ class TwoPhasedRebalancedTest(IgniteTest): """ # pylint: disable=R0914 @cluster(num_nodes=(NUM_NODES_CELL * NUM_CELL) + 1) - @ignite_versions(str(DEV_BRANCH), str(LATEST_2_10), str(LATEST_2_9), str(LATEST_2_8)) + @ignite_versions(str(DEV_BRANCH), str(LATEST), str(LATEST_2_9)) def two_phased_rebalancing_test(self, ignite_version): """ Test case of two-phase rebalancing. @@ -138,8 +138,8 @@ def two_phased_rebalancing_test(self, ignite_version): assert pds_after[host] < pds_before[host], f'Host {host}: size after = {pds_after[host]}, ' \ f'size before = {pds_before[host]}.' - dump_2 = create_idle_dump_and_copy_to_log_dir(control_utility, node, cells[0].log_dir) control_utility.validate_indexes() + dump_2 = create_idle_dump_and_copy_to_log_dir(control_utility, node, cells[0].log_dir) # Check data consistency. diff = node.account.ssh_output(f'diff {dump_1} {dump_2}', allow_fail=True) @@ -155,22 +155,23 @@ def start_cells(self, config: IgniteConfiguration, num_cell: int) -> List[Ignite cells = [] - first = IgniteService(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}=0'], - startup_timeout_sec=180) - first.start() + first = IgniteService(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}=0']) + first.start_async() + + cells.append(first) discovery_spi = from_ignite_cluster(first) config = config._replace(discovery_spi=discovery_spi) - cells.append(first) - for i in range(1, num_cell): - cell = IgniteService(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}={i}'], - startup_timeout_sec=180,) - cell.start() + cell = IgniteService(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}={i}']) + cell.start_async() cells.append(cell) + for cell in cells: + cell.await_started() + return cells def get_pds_size(self, cells: [IgniteService], msg: str) -> dict: From 31de1d48ec579c702eeba20d0dfca3a2ac402477 Mon Sep 17 00:00:00 2001 From: Ryzhov Sergei Date: Thu, 25 Mar 2021 18:51:29 +0300 Subject: [PATCH 21/21] IGNITE-13508: remove LATEST_2_9 --- .../tests/ignitetest/tests/two_phased_rebalanced_test.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py index f16640663b50e..6b477a9d3b7d6 100644 --- a/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py +++ b/modules/ducktests/tests/ignitetest/tests/two_phased_rebalanced_test.py @@ -31,7 +31,7 @@ from ignitetest.services.utils.ignite_configuration.discovery import from_ignite_cluster from ignitetest.utils import cluster, ignite_versions from ignitetest.utils.ignite_test import IgniteTest -from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST, LATEST_2_9 +from ignitetest.utils.version import IgniteVersion, DEV_BRANCH, LATEST NUM_NODES_CELL = 4 @@ -49,7 +49,7 @@ class TwoPhasedRebalancedTest(IgniteTest): """ # pylint: disable=R0914 @cluster(num_nodes=(NUM_NODES_CELL * NUM_CELL) + 1) - @ignite_versions(str(DEV_BRANCH), str(LATEST), str(LATEST_2_9)) + @ignite_versions(str(DEV_BRANCH), str(LATEST)) def two_phased_rebalancing_test(self, ignite_version): """ Test case of two-phase rebalancing. @@ -152,7 +152,6 @@ def start_cells(self, config: IgniteConfiguration, num_cell: int) -> List[Ignite :param num_cell Number of cell. :return List of IgniteServices. """ - cells = [] first = IgniteService(self.test_context, config, NUM_NODES_CELL, [f'-D{ATTRIBUTE}=0']) @@ -181,7 +180,6 @@ def get_pds_size(self, cells: [IgniteService], msg: str) -> dict: :param msg Information message. :return dict with hostname -> pds size in megabytes. """ - res = {} for cell in cells: for node in cell.nodes: