diff --git a/add_new_dc_test.py b/add_new_dc_test.py index ed5ec15145d..d34e37bf536 100644 --- a/add_new_dc_test.py +++ b/add_new_dc_test.py @@ -1,5 +1,4 @@ import warnings -from typing import Tuple, List from cassandra import ConsistencyLevel from cassandra.query import SimpleStatement # pylint: disable=no-name-in-module @@ -56,7 +55,7 @@ def test_add_new_dc(self) -> None: # pylint: disable=too-many-locals self.verify_data_can_be_read_from_new_dc(new_node) self.log.info("Test completed.") - def reconfigure_keyspaces_to_use_network_topology_strategy(self, keyspaces: List[str], replication_factors: dict[str, int]) -> None: + def reconfigure_keyspaces_to_use_network_topology_strategy(self, keyspaces: list[str], replication_factors: dict[str, int]) -> None: node = self.db_cluster.nodes[0] self.log.info("Reconfiguring keyspace Replication Strategy") network_topology_strategy = NetworkTopologyReplicationStrategy( @@ -64,7 +63,7 @@ def reconfigure_keyspaces_to_use_network_topology_strategy(self, keyspaces: List for keyspace in keyspaces: cql = f"ALTER KEYSPACE {keyspace} WITH replication = {network_topology_strategy}" node.run_cqlsh(cql) - self.log.info("Replication Strategies for {} reconfigured".format(keyspaces)) + self.log.info(f"Replication Strategies for {keyspaces} reconfigured") def prewrite_db_with_data(self) -> None: self.log.info("Prewriting database...") @@ -73,7 +72,7 @@ def prewrite_db_with_data(self) -> None: self.verify_stress_thread(cs_thread_pool=pre_thread) self.log.info("Database pre write completed") - def start_stress_during_adding_new_dc(self) -> Tuple[CassandraStressThread, CassandraStressThread]: + def start_stress_during_adding_new_dc(self) -> tuple[CassandraStressThread, CassandraStressThread]: self.log.info("Running stress during adding new DC") stress_cmds = self.params.get('stress_cmd') read_thread = self.run_stress_thread(stress_cmd=stress_cmds[0], stats_aggregate_cmds=False, round_robin=False) diff --git a/admission_control_overload_test.py b/admission_control_overload_test.py index d08649b4c09..74ac1f05bfe 100644 --- a/admission_control_overload_test.py +++ b/admission_control_overload_test.py @@ -1,7 +1,9 @@ import time + from invoke import exceptions -from sdcm.tester import ClusterTester + from sdcm.db_stats import PrometheusDBStats +from sdcm.tester import ClusterTester class AdmissionControlOverloadTest(ClusterTester): @@ -53,7 +55,7 @@ def run_load(self, job_num, job_cmd, is_prepare=False): try: results.append(self.get_stress_results(stress)) except exceptions.CommandTimedOut as ex: - self.log.debug('some c-s timed out\n{}'.format(ex)) + self.log.debug(f'some c-s timed out\n{ex}') return is_ever_triggered diff --git a/artifacts_test.py b/artifacts_test.py index 05f1010ebf1..b9f6c229515 100644 --- a/artifacts_test.py +++ b/artifacts_test.py @@ -11,21 +11,21 @@ # # Copyright (c) 2020 ScyllaDB import datetime +import json import pprint import re import typing from functools import cached_property -import json -import yaml import requests +import yaml from sdcm.sct_events import Severity from sdcm.sct_events.database import ScyllaHousekeepingServiceEvent from sdcm.tester import ClusterTester from sdcm.utils.adaptive_timeouts import NodeLoadInfoServices +from sdcm.utils.common import ScyllaProduct, get_latest_scylla_release from sdcm.utils.housekeeping import HousekeepingDB -from sdcm.utils.common import get_latest_scylla_release, ScyllaProduct STRESS_CMD: str = "/usr/bin/cassandra-stress" @@ -285,7 +285,7 @@ def run_pre_create_schema(self, replication_factor=1): scylla_encryption_options = self.params.get('scylla_encryption_options') self.log.debug('Pre Creating Schema for c-s with %s keyspaces', keyspace_num) for i in range(1, keyspace_num+1): - keyspace_name = 'keyspace{}'.format(i) + keyspace_name = f'keyspace{i}' self.create_keyspace(keyspace_name=keyspace_name, replication_factor=replication_factor) self.log.debug('%s Created', keyspace_name) col_num = 5 diff --git a/big_cluster_test.py b/big_cluster_test.py index 81928df89e1..364f1beb29d 100644 --- a/big_cluster_test.py +++ b/big_cluster_test.py @@ -18,8 +18,7 @@ import logging -from sdcm.tester import ClusterTester -from sdcm.tester import teardown_on_exception +from sdcm.tester import ClusterTester, teardown_on_exception class HugeClusterTest(ClusterTester): diff --git a/cdc_replication_test.py b/cdc_replication_test.py index cd2bd61e2a7..440bddfa137 100644 --- a/cdc_replication_test.py +++ b/cdc_replication_test.py @@ -13,22 +13,21 @@ # # Copyright (c) 2020 ScyllaDB +import os +import random import shutil import sys -import os import time -import random from enum import Enum from textwrap import dedent -from typing import Optional, Tuple from cassandra import ConsistencyLevel from cassandra.query import SimpleStatement # pylint: disable=no-name-in-module from sdcm import cluster -from sdcm.tester import ClusterTester from sdcm.gemini_thread import GeminiStressThread from sdcm.nemesis import CategoricalMonkey +from sdcm.tester import ClusterTester class Mode(Enum): @@ -92,7 +91,7 @@ def test_replication_cs(self) -> None: self.test_replication(False, Mode.DELTA) def test_replication_gemini(self, mode: Mode) -> None: - self.log.info('Using gemini to generate workload. Mode: {}'.format(mode.name)) + self.log.info(f'Using gemini to generate workload. Mode: {mode.name}') self.test_replication(True, mode) def test_replication_gemini_delta(self) -> None: @@ -174,14 +173,14 @@ def test_replication_longevity(self) -> None: # One more round would cause the nodes to run out of disk space. no_rounds = 9 for rnd in range(no_rounds): - self.log.info('Starting round {}'.format(rnd)) + self.log.info(f'Starting round {rnd}') self.log.info('Starting nemesis') self.db_cluster.start_nemesis() self.log.info('Waiting for workload generation to finish (~30 minutes)...') stress_results = self.verify_gemini_results(queue=stress_thread) - self.log.info('gemini results: {}'.format(stress_results)) + self.log.info(f'gemini results: {stress_results}') self.log.info('Waiting for replicator to finish (sleeping 180s)...') time.sleep(180) @@ -279,10 +278,10 @@ def test_replication(self, is_gemini_test: bool, mode: Mode) -> None: # noqa: P self.log.info('Waiting for stressor to finish...') if is_gemini_test: stress_results = self.verify_gemini_results(queue=stress_thread) - self.log.info('gemini results: {}'.format(stress_results)) + self.log.info(f'gemini results: {stress_results}') else: stress_results = stress_thread.get_results() - self.log.info('cassandra-stress results: {}'.format(list(stress_results))) + self.log.info(f'cassandra-stress results: {list(stress_results)}') self.log.info('Waiting for replicator to finish (sleeping 60s)...') time.sleep(60) @@ -318,7 +317,7 @@ def test_replication(self, is_gemini_test: bool, mode: Mode) -> None: # noqa: P self.fail('Consistency check failed.') # Compares tables using the scylla-migrate tool. - def check_consistency(self, migrate_log_dst_path: str, compare_timestamps: bool = True) -> Tuple[bool, bool]: + def check_consistency(self, migrate_log_dst_path: str, compare_timestamps: bool = True) -> tuple[bool, bool]: loader_node = self.loaders.nodes[0] self.log.info('Comparing table contents using scylla-migrate...') res = loader_node.remoter.run(cmd='./scylla-migrate check --master-address {} --replica-address {}' @@ -331,7 +330,7 @@ def check_consistency(self, migrate_log_dst_path: str, compare_timestamps: bool migrate_ok = res.ok if not migrate_ok: - self.log.error('scylla-migrate command returned status {}'.format(res.exit_status)) + self.log.error(f'scylla-migrate command returned status {res.exit_status}') with open(migrate_log_dst_path, encoding="utf-8") as file: consistency_ok = 'Consistency check OK.\n' in (line for line in file) @@ -380,14 +379,14 @@ def start_replicator(self, mode: Mode) -> None: self.cs_db_cluster.nodes[0].external_address, mode_str(mode))) - self.log.info('Replicator script:\n{}'.format(replicator_script)) + self.log.info(f'Replicator script:\n{replicator_script}') self.log.info('Starting replicator.') res = self.loaders.nodes[0].remoter.run(cmd=replicator_script) if res.exit_status != 0: self.fail('Could not start CDC replicator.') - def start_gemini(self, seed: Optional[int] = None) -> GeminiStressThread: + def start_gemini(self, seed: int | None = None) -> GeminiStressThread: params = {'gemini_seed': seed} if seed else {} return GeminiStressThread( test_cluster=self.db_cluster, diff --git a/corrupt_then_rebuild_test.py b/corrupt_then_rebuild_test.py index b3da83b9112..0577bed6d7b 100644 --- a/corrupt_then_rebuild_test.py +++ b/corrupt_then_rebuild_test.py @@ -13,8 +13,8 @@ # # Copyright (c) 2017 ScyllaDB -from sdcm.tester import ClusterTester from sdcm import nemesis +from sdcm.tester import ClusterTester class CorruptThenRebuildTest(ClusterTester): diff --git a/destroy_data_then_repair_test.py b/destroy_data_then_repair_test.py index cfd56a33ef3..dfbd76d608e 100644 --- a/destroy_data_then_repair_test.py +++ b/destroy_data_then_repair_test.py @@ -13,8 +13,8 @@ # # Copyright (c) 2017 ScyllaDB -from sdcm.tester import ClusterTester from sdcm import nemesis +from sdcm.tester import ClusterTester class CorruptThenRepair(ClusterTester): diff --git a/enospc_test.py b/enospc_test.py index 2504ba7dfef..4b9efd78c38 100644 --- a/enospc_test.py +++ b/enospc_test.py @@ -13,8 +13,8 @@ # # Copyright (c) 2017 ScyllaDB -from sdcm.tester import ClusterTester from sdcm.nemesis import EnospcAllNodesMonkey +from sdcm.tester import ClusterTester class EnospcTest(ClusterTester): diff --git a/full_cluster_stop_start_test.py b/full_cluster_stop_start_test.py index 235cef1ffba..f6c660df95a 100644 --- a/full_cluster_stop_start_test.py +++ b/full_cluster_stop_start_test.py @@ -30,7 +30,7 @@ def test_full_cluster_stop_start(self): time.sleep(60) # making sure all nodes are up after RestartSecs are over for node in nodes: - self.log.info("making sure node '{}' is up".format(node)) + self.log.info(f"making sure node '{node}' is up") node.wait_db_up(verbose=True, timeout=300) stress_queue = self.run_stress_thread(stress_cmd=self.params.get('stress_read_cmd')) diff --git a/functional_tests/mocked/conftest.py b/functional_tests/mocked/conftest.py index c6129b004d0..fb839eb53b8 100644 --- a/functional_tests/mocked/conftest.py +++ b/functional_tests/mocked/conftest.py @@ -17,7 +17,6 @@ from sdcm.utils.aws_region import AwsRegion - AWS_REGION = "us-east-1" diff --git a/functional_tests/scylla_operator/conftest.py b/functional_tests/scylla_operator/conftest.py index 3d8b593b6d4..bf55c537642 100644 --- a/functional_tests/scylla_operator/conftest.py +++ b/functional_tests/scylla_operator/conftest.py @@ -13,23 +13,23 @@ # # Copyright (c) 2021 ScyllaDB +import contextlib import logging import os import time import traceback -import contextlib - -from typing import Optional import pytest from deepdiff import DeepDiff -from functional_tests.scylla_operator.libs.auxiliary import ScyllaOperatorFunctionalClusterTester, sct_abs_path +from functional_tests.scylla_operator.libs.auxiliary import ( + ScyllaOperatorFunctionalClusterTester, + sct_abs_path, +) from sdcm.cluster_k8s import ScyllaPodCluster from sdcm.utils import version_utils - -TESTER: Optional[ScyllaOperatorFunctionalClusterTester] = None +TESTER: ScyllaOperatorFunctionalClusterTester | None = None LOGGER = logging.getLogger(__name__) diff --git a/functional_tests/scylla_operator/libs/auxiliary.py b/functional_tests/scylla_operator/libs/auxiliary.py index c15fb1a4152..1cd2873e715 100644 --- a/functional_tests/scylla_operator/libs/auxiliary.py +++ b/functional_tests/scylla_operator/libs/auxiliary.py @@ -15,10 +15,9 @@ import os from sdcm.cluster_k8s import ScyllaPodCluster -from sdcm.tester import ClusterTester from sdcm.sct_events import Severity from sdcm.sct_events.system import TestFrameworkEvent - +from sdcm.tester import ClusterTester SCT_ROOT = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..')) diff --git a/functional_tests/scylla_operator/libs/helpers.py b/functional_tests/scylla_operator/libs/helpers.py index ab1b0ce7cda..b170455802b 100644 --- a/functional_tests/scylla_operator/libs/helpers.py +++ b/functional_tests/scylla_operator/libs/helpers.py @@ -12,17 +12,16 @@ # See LICENSE for more details. # # Copyright (c) 2021 ScyllaDB -from enum import Enum import logging import time -from typing import Union -import yaml +from enum import Enum +import yaml from kubernetes.client import exceptions as k8s_exceptions from sdcm.cluster import ( - DB_LOG_PATTERN_RESHARDING_START, DB_LOG_PATTERN_RESHARDING_FINISH, + DB_LOG_PATTERN_RESHARDING_START, ) from sdcm.cluster_k8s import ( SCYLLA_MANAGER_NAMESPACE, @@ -169,7 +168,7 @@ def reinstall_scylla_manager(db_cluster: ScyllaPodCluster, manager_version: str) log.info("Scylla Manager '%s' has successfully been installed", manager_version) -def verify_resharding_on_k8s(db_cluster: ScyllaPodCluster, cpus: Union[str, int, float]): +def verify_resharding_on_k8s(db_cluster: ScyllaPodCluster, cpus: str | int | float): nodes_data = [] for node in reversed(db_cluster.nodes): liveness_probe_failures = node.follow_system_log( diff --git a/functional_tests/scylla_operator/test_functional.py b/functional_tests/scylla_operator/test_functional.py index e6d8b5fb41c..df016738cfa 100644 --- a/functional_tests/scylla_operator/test_functional.py +++ b/functional_tests/scylla_operator/test_functional.py @@ -14,53 +14,52 @@ # Copyright (c) 2021 ScyllaDB # pylint: disable=too-many-lines +import base64 import logging import os import random +import ssl import threading import time -import ssl -import base64 -import path +import path import pytest import yaml from cassandra.cluster import ( # pylint: disable=no-name-in-module + EXEC_PROFILE_DEFAULT, Cluster, ExecutionProfile, - EXEC_PROFILE_DEFAULT, ) from cassandra.policies import WhiteListRoundRobinPolicy +from functional_tests.scylla_operator.libs.helpers import ( + PodStatuses, + get_orphaned_services, + get_pod_storage_capacity, + get_pods_and_statuses, + get_pods_without_probe, + get_scylla_sysctl_value, + reinstall_scylla_manager, + set_scylla_sysctl_value, + verify_resharding_on_k8s, + wait_for_resource_absence, +) from sdcm.cluster_k8s import ( - ScyllaPodCluster, - SCYLLA_NAMESPACE, SCYLLA_MANAGER_NAMESPACE, - SCYLLA_OPERATOR_NAMESPACE + SCYLLA_NAMESPACE, + SCYLLA_OPERATOR_NAMESPACE, + ScyllaPodCluster, ) from sdcm.mgmt import TaskStatus from sdcm.utils.common import ParallelObject from sdcm.utils.k8s import ( - convert_cpu_units_to_k8s_value, - convert_cpu_value_from_k8s_to_units, HelmValues, KubernetesOps, + convert_cpu_units_to_k8s_value, + convert_cpu_value_from_k8s_to_units, ) from sdcm.utils.k8s.chaos_mesh import PodFailureExperiment -from functional_tests.scylla_operator.libs.helpers import ( - get_scylla_sysctl_value, - get_orphaned_services, - get_pods_without_probe, - get_pods_and_statuses, - get_pod_storage_capacity, - PodStatuses, - reinstall_scylla_manager, - set_scylla_sysctl_value, - verify_resharding_on_k8s, - wait_for_resource_absence, -) - log = logging.getLogger() # TODO: add support for multiDC setups diff --git a/gemini_test.py b/gemini_test.py index b9d491d0688..5f0345ceb10 100644 --- a/gemini_test.py +++ b/gemini_test.py @@ -56,7 +56,7 @@ def test_load_random_with_nemesis(self): self.gemini_results["cmd"] = gemini_thread.gemini_commands # sleep before run nemesis test_duration * .25 sleep_before_start = float(self.params.get('test_duration')) * 60 * .1 - self.log.info('Sleep interval {}'.format(sleep_before_start)) + self.log.info(f'Sleep interval {sleep_before_start}') time.sleep(sleep_before_start) self.db_cluster.start_nemesis() @@ -85,7 +85,7 @@ def test_load_random_with_nemesis_cdc_reader(self): base_table_name="table1") # sleep before run nemesis test_duration * .1 sleep_before_start = float(self.params.get('test_duration')) * 60 * .1 - self.log.info('Sleep interval {}'.format(sleep_before_start)) + self.log.info(f'Sleep interval {sleep_before_start}') time.sleep(sleep_before_start) self.db_cluster.start_nemesis() diff --git a/grow_cluster_test.py b/grow_cluster_test.py index 9fc570ea9a0..15885545eb5 100644 --- a/grow_cluster_test.py +++ b/grow_cluster_test.py @@ -13,14 +13,13 @@ # # Copyright (c) 2016 ScyllaDB -import time import datetime import random +import time +from sdcm import nemesis, prometheus from sdcm.tester import ClusterTester from sdcm.utils.common import get_data_dir_path -from sdcm import nemesis -from sdcm import prometheus class GrowClusterTest(ClusterTester): @@ -65,11 +64,10 @@ def get_stress_cmd(self, mode='write', duration=None): duration = self.params.get('test_duration') threads = self.params.get('cassandra_stress_threads') - return ("cassandra-stress %s cl=QUORUM duration=%sm " + return (f"cassandra-stress {mode} cl=QUORUM duration={duration}m " "-schema 'replication(strategy=NetworkTopologyStrategy,replication_factor=3)' -col 'size=FIXED(2) n=FIXED(1)' " - "-mode cql3 native -rate threads=%s " - "-pop seq=1..%s -node %s" % - (mode, duration, threads, population_size, ip)) + f"-mode cql3 native -rate threads={threads} " + f"-pop seq=1..{population_size} -node {ip}") def add_nodes(self, add_node_cnt): self.metrics_srv.event_start('add_node') diff --git a/hinted_handoff_test.py b/hinted_handoff_test.py index 53e9e01e165..f40f658d87e 100644 --- a/hinted_handoff_test.py +++ b/hinted_handoff_test.py @@ -1,4 +1,5 @@ import time + from sdcm.tester import ClusterTester diff --git a/ics_space_amplification_goal_test.py b/ics_space_amplification_goal_test.py index d65fad57ce7..222b71c90b5 100644 --- a/ics_space_amplification_goal_test.py +++ b/ics_space_amplification_goal_test.py @@ -22,7 +22,7 @@ from sdcm.sct_events import Severity from sdcm.sct_events.system import InfoEvent, TestFrameworkEvent from sdcm.utils.common import ParallelObject -from test_lib.compaction import CompactionStrategy, LOGGER +from test_lib.compaction import LOGGER, CompactionStrategy KEYSPACE_NAME = 'keyspace1' TABLE_NAME = 'standard1' @@ -129,8 +129,8 @@ def _set_enforce_min_threshold_true(self): """.format(yaml_file, tmp_yaml_file)) for node in self.db_cluster.nodes: # set compaction_enforce_min_threshold on all nodes node.remoter.run('sudo bash -cxe "%s"' % set_enforce_min_threshold) - self.log.debug("Scylla YAML configuration read from: {} {} is:".format(node.public_ip_address, yaml_file)) - node.remoter.run('sudo cat {}'.format(yaml_file)) + self.log.debug(f"Scylla YAML configuration read from: {node.public_ip_address} {yaml_file} is:") + node.remoter.run(f'sudo cat {yaml_file}') node.stop_scylla_server() node.start_scylla_server() diff --git a/jepsen_test.py b/jepsen_test.py index c7e9fa286a1..00b222c8cac 100644 --- a/jepsen_test.py +++ b/jepsen_test.py @@ -89,7 +89,7 @@ def save_jepsen_report(self): sleep {JEPSEN_WEB_SERVER_START_DELAY} """), verbose=True) - with open(os.path.join(self.logdir, "jepsen_report.html"), "wt", encoding="utf-8") as jepsen_report: + with open(os.path.join(self.logdir, "jepsen_report.html"), "w", encoding="utf-8") as jepsen_report: jepsen_report.write(requests.get(url).text) self.log.info("Report has been saved to %s", jepsen_report.name) diff --git a/longevity_lwt_test.py b/longevity_lwt_test.py index 368ad263fd4..3a0b66ec6bc 100644 --- a/longevity_lwt_test.py +++ b/longevity_lwt_test.py @@ -21,9 +21,9 @@ from longevity_test import LongevityTest from sdcm.sct_events import Severity +from sdcm.sct_events.group_common_events import ignore_mutation_write_errors from sdcm.sct_events.health import DataValidatorEvent from sdcm.utils.data_validator import LongevityDataValidator -from sdcm.sct_events.group_common_events import ignore_mutation_write_errors class LWTLongevityTest(LongevityTest): diff --git a/longevity_sla_test.py b/longevity_sla_test.py index 8187cab9faf..5bab6035329 100644 --- a/longevity_sla_test.py +++ b/longevity_sla_test.py @@ -15,7 +15,7 @@ from longevity_test import LongevityTest from sdcm.sla.libs.sla_utils import SlaUtils from sdcm.utils import loader_utils -from sdcm.utils.adaptive_timeouts import adaptive_timeout, Operations +from sdcm.utils.adaptive_timeouts import Operations, adaptive_timeout from test_lib.sla import create_sla_auth diff --git a/longevity_test.py b/longevity_test.py index 5da0991aa1d..03f8791af9f 100644 --- a/longevity_test.py +++ b/longevity_test.py @@ -12,28 +12,29 @@ # See LICENSE for more details. # # Copyright (c) 2016 ScyllaDB +import contextlib +import itertools import json import os import re import string import tempfile -import itertools -import contextlib import yaml from cassandra import AlreadyExists, InvalidRequest from cassandra.query import SimpleStatement # pylint: disable=no-name-in-module -from sdcm.sct_events.group_common_events import \ - ignore_large_collection_warning, \ - ignore_max_memory_for_unlimited_query_soft_limit +from sdcm.cluster import MAX_TIME_WAIT_FOR_NEW_NODE_UP +from sdcm.sct_events import Severity +from sdcm.sct_events.group_common_events import ( + ignore_large_collection_warning, + ignore_max_memory_for_unlimited_query_soft_limit, +) +from sdcm.sct_events.system import InfoEvent from sdcm.tester import ClusterTester from sdcm.utils import loader_utils -from sdcm.utils.adaptive_timeouts import adaptive_timeout, Operations +from sdcm.utils.adaptive_timeouts import Operations, adaptive_timeout from sdcm.utils.operations_thread import ThreadParams -from sdcm.sct_events.system import InfoEvent -from sdcm.sct_events import Severity -from sdcm.cluster import MAX_TIME_WAIT_FOR_NEW_NODE_UP class LongevityTest(ClusterTester, loader_utils.LoaderUtilsMixin): @@ -97,7 +98,7 @@ def run_pre_create_keyspace(self): self._pre_create_keyspace() def _run_validate_large_collections_in_system(self, node, table='table_with_large_collection'): - self.log.info("Verifying large collections in system tables on node: {}".format(node)) + self.log.info(f"Verifying large collections in system tables on node: {node}") with self.db_cluster.cql_connection_exclusive(node=node) as session: query = "SELECT * from system.large_cells WHERE keyspace_name='large_collection_test'" \ f" AND table_name='{table}' ALLOW FILTERING" @@ -107,11 +108,11 @@ def _run_validate_large_collections_in_system(self, node, table='table_with_larg InfoEvent("Did not find expected row in system.large_cells", severity=Severity.ERROR) def _run_validate_large_collections_warning_in_logs(self, node): - self.log.info("Verifying warning for large collections in logs on node: {}".format(node)) + self.log.info(f"Verifying warning for large collections in logs on node: {node}") msg = "Writing large collection" res = list(node.follow_system_log(patterns=[msg], start_from_beginning=True)) if not res: - InfoEvent("Did not find expected log message warning: {}".format(msg), severity=Severity.ERROR) + InfoEvent(f"Did not find expected log message warning: {msg}", severity=Severity.ERROR) def test_custom_time(self): # noqa: PLR0912, PLR0915 """ @@ -173,8 +174,8 @@ def test_custom_time(self): # noqa: PLR0912, PLR0915 if customer_profiles: cs_duration = self.params.get('cs_duration') for cs_profile in customer_profiles: - assert os.path.exists(cs_profile), 'File not found: {}'.format(cs_profile) - self.log.debug('Run stress test with user profile {}, duration {}'.format(cs_profile, cs_duration)) + assert os.path.exists(cs_profile), f'File not found: {cs_profile}' + self.log.debug(f'Run stress test with user profile {cs_profile}, duration {cs_duration}') profile_dst = os.path.join('/tmp', os.path.basename(cs_profile)) with open(cs_profile, encoding="utf-8") as pconf: cont = pconf.readlines() @@ -184,7 +185,7 @@ def test_custom_time(self): # noqa: PLR0912, PLR0915 for cmd in [line.lstrip('#').strip() for line in cont if line.find('cassandra-stress') > 0]: stress_cmd = (cmd.format(profile_dst, cs_duration)) params = {'stress_cmd': stress_cmd, 'profile': cs_profile} - self.log.debug('Stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'Stress cmd: {stress_cmd}') self._run_all_stress_cmds(stress_queue, params) # Check if we shall wait for total_used_space or if nemesis wasn't started @@ -286,8 +287,8 @@ def test_user_batch_custom_time(self): duration = int(cs_duration.translate(str.maketrans('', '', string.ascii_letters))) for cs_profile in customer_profiles: - assert os.path.exists(cs_profile), 'File not found: {}'.format(cs_profile) - self.log.debug('Run stress test with user profile {}, duration {}'.format(cs_profile, cs_duration)) + assert os.path.exists(cs_profile), f'File not found: {cs_profile}' + self.log.debug(f'Run stress test with user profile {cs_profile}, duration {cs_duration}') user_profile_table_count = self.params.get('user_profile_table_count') # pylint: disable=invalid-name @@ -392,13 +393,13 @@ def _pre_create_schema(self, keyspace_num=1, scylla_encryption_options=None): cassandra-stress. """ - self.log.debug('Pre Creating Schema for c-s with {} keyspaces'.format(keyspace_num)) + self.log.debug(f'Pre Creating Schema for c-s with {keyspace_num} keyspaces') compaction_strategy = self.params.get('compaction_strategy') sstable_size = self.params.get('sstable_size') for i in range(1, keyspace_num+1): - keyspace_name = 'keyspace{}'.format(i) + keyspace_name = f'keyspace{i}' self.create_keyspace(keyspace_name=keyspace_name, replication_factor=3) - self.log.debug('{} Created'.format(keyspace_name)) + self.log.debug(f'{keyspace_name} Created') col_num = self._get_prepare_write_cmd_columns_num() or 5 columns = {} for col_idx in range(col_num): @@ -429,28 +430,28 @@ def _pre_create_templated_user_schema(self, batch_start=None, batch_end=None): try: session.execute(keyspace_definition) except AlreadyExists: - self.log.debug("keyspace [{}] exists".format(keyspace_name)) + self.log.debug(f"keyspace [{keyspace_name}] exists") if batch_start is not None and batch_end is not None: table_range = range(batch_start, batch_end) else: table_range = range(user_profile_table_count) - self.log.debug('Pre Creating Schema for c-s with {} user tables'.format(user_profile_table_count)) + self.log.debug(f'Pre Creating Schema for c-s with {user_profile_table_count} user tables') for i in table_range: - table_name = 'table{}'.format(i) + table_name = f'table{i}' query = table_template.substitute(table_name=table_name) try: session.execute(query) except AlreadyExists: - self.log.debug('table [{}] exists'.format(table_name)) - self.log.debug('{} Created'.format(table_name)) + self.log.debug(f'table [{table_name}] exists') + self.log.debug(f'{table_name} Created') for definition in profile_yaml.get('extra_definitions', []): query = string.Template(definition).substitute(table_name=table_name) try: session.execute(query) except (AlreadyExists, InvalidRequest) as exc: - self.log.debug('extra definition for [{}] exists [{}]'.format(table_name, str(exc))) + self.log.debug(f'extra definition for [{table_name}] exists [{str(exc)}]') def _flush_all_nodes(self): """ @@ -508,7 +509,7 @@ def create_templated_user_stress_params(self, idx, cs_profile): # pylint: disab for cmd in [line.lstrip('#').strip() for line in cont if line.find('cassandra-stress') > 0]: stress_cmd = (cmd.format(profile_dst, cs_duration)) params = {'stress_cmd': stress_cmd, 'profile': profile_dst} - self.log.debug('Stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'Stress cmd: {stress_cmd}') params_list.append(params) return params_list diff --git a/maintenance_test.py b/maintenance_test.py index 9e8fe29fd93..40e7254b0ea 100644 --- a/maintenance_test.py +++ b/maintenance_test.py @@ -15,10 +15,12 @@ import time +from sdcm.nemesis import ( + CorruptThenRebuildMonkey, + CorruptThenRepairMonkey, + DrainerMonkey, +) from sdcm.tester import ClusterTester -from sdcm.nemesis import DrainerMonkey -from sdcm.nemesis import CorruptThenRepairMonkey -from sdcm.nemesis import CorruptThenRebuildMonkey class MaintainanceTest(ClusterTester): diff --git a/mgmt_cli_test.py b/mgmt_cli_test.py index 49de9d53659..be0976843a1 100644 --- a/mgmt_cli_test.py +++ b/mgmt_cli_test.py @@ -16,34 +16,42 @@ # pylint: disable=too-many-lines import random -from pathlib import Path -from functools import cached_property import re import time -from textwrap import dedent from datetime import datetime +from functools import cached_property +from pathlib import Path +from textwrap import dedent import boto3 - from invoke import exceptions from pkg_resources import parse_version from sdcm import mgmt -from sdcm.mgmt import ScyllaManagerError, TaskStatus, HostStatus, HostSsl, HostRestStatus -from sdcm.mgmt.cli import ScyllaManagerTool -from sdcm.mgmt.common import reconfigure_scylla_manager, get_persistent_snapshots -from sdcm.remote import shell_script_cmd -from sdcm.tester import ClusterTester from sdcm.cluster import TestConfig +from sdcm.exceptions import FilesNotCorrupted +from sdcm.mgmt import ( + HostRestStatus, + HostSsl, + HostStatus, + ScyllaManagerError, + TaskStatus, +) +from sdcm.mgmt.cli import ScyllaManagerTool +from sdcm.mgmt.common import get_persistent_snapshots, reconfigure_scylla_manager from sdcm.nemesis import MgmtRepair -from sdcm.utils.adaptive_timeouts import adaptive_timeout, Operations -from sdcm.utils.common import reach_enospc_on_node, clean_enospc_on_node -from sdcm.utils.loader_utils import LoaderUtilsMixin +from sdcm.remote import shell_script_cmd +from sdcm.sct_events.group_common_events import ( + ignore_no_space_errors, + ignore_stream_mutation_fragments_errors, +) from sdcm.sct_events.system import InfoEvent -from sdcm.sct_events.group_common_events import ignore_no_space_errors, ignore_stream_mutation_fragments_errors -from sdcm.utils.gce_utils import get_gce_storage_client +from sdcm.tester import ClusterTester +from sdcm.utils.adaptive_timeouts import Operations, adaptive_timeout from sdcm.utils.azure_utils import AzureService -from sdcm.exceptions import FilesNotCorrupted +from sdcm.utils.common import clean_enospc_on_node, reach_enospc_on_node +from sdcm.utils.gce_utils import get_gce_storage_client +from sdcm.utils.loader_utils import LoaderUtilsMixin class BackupFunctionsMixIn(LoaderUtilsMixin): @@ -235,14 +243,14 @@ def _parse_stress_cmd(self, stress_cmd, params): if 'compression' in stress_cmd: if 'keyspace_name' not in params: compression_prefix = re.search('compression=(.*)Compressor', stress_cmd).group(1) - keyspace_name = "keyspace_{}".format(compression_prefix.lower()) + keyspace_name = f"keyspace_{compression_prefix.lower()}" params.update({'keyspace_name': keyspace_name}) return params @staticmethod def _get_keyspace_name(ks_number, keyspace_pref='keyspace'): - return '{}{}'.format(keyspace_pref, ks_number) + return f'{keyspace_pref}{ks_number}' def generate_background_read_load(self): self.log.info('Starting c-s read') @@ -299,7 +307,7 @@ def test_mgmt_cluster_crud(self): auth_token=self.monitors.mgmt_auth_token) # Test cluster attributes cluster_orig_name = mgr_cluster.name - mgr_cluster.update(name="{}_renamed".format(cluster_orig_name)) + mgr_cluster.update(name=f"{cluster_orig_name}_renamed") assert mgr_cluster.name == cluster_orig_name+"_renamed", "Cluster name wasn't changed after update command" mgr_cluster.delete() mgr_cluster = manager_tool.add_cluster(self.CLUSTER_NAME, db_cluster=self.db_cluster, @@ -323,11 +331,11 @@ def get_all_dcs_names(self): def _create_keyspace_and_basic_table(self, keyspace_name, table_name="example_table", replication_factor=1): - self.log.info("creating keyspace {}".format(keyspace_name)) + self.log.info(f"creating keyspace {keyspace_name}") keyspace_existence = self.create_keyspace(keyspace_name, replication_factor) assert keyspace_existence, "keyspace creation failed" # Keyspaces without tables won't appear in the repair, so the must have one - self.log.info("creating the table {} in the keyspace {}".format(table_name, keyspace_name)) + self.log.info(f"creating the table {table_name} in the keyspace {keyspace_name}") self.create_table(table_name, keyspace_name=keyspace_name) def test_manager_sanity(self): @@ -380,7 +388,7 @@ def test_repair_control(self): InfoEvent(message="Sleep ended - Starting tests").publish() self._create_repair_and_alter_it_with_repair_control() load_results = stress_read_thread.get_results() - self.log.info('load={}'.format(load_results)) + self.log.info(f'load={load_results}') def _create_repair_and_alter_it_with_repair_control(self): keyspace_to_be_repaired = "keyspace2" @@ -425,7 +433,7 @@ def _repair_intensity_feature(self, fault_multiple_nodes): with self.subTest('test_intensity_and_parallel'): self.test_intensity_and_parallel(fault_multiple_nodes=fault_multiple_nodes) load_results = stress_read_thread.get_results() - self.log.info('load={}'.format(load_results)) + self.log.info(f'load={load_results}') def get_email_data(self): self.log.info("Prepare data for email") @@ -563,7 +571,7 @@ def test_backup_multiple_ks_tables(self): or manager_tool.add_cluster(name=self.CLUSTER_NAME, db_cluster=self.db_cluster, auth_token=self.monitors.mgmt_auth_token) tables = self.create_ks_and_tables(10, 100) - self.log.debug('tables list = {}'.format(tables)) + self.log.debug(f'tables list = {tables}') # TODO: insert data to those tables backup_task = mgr_cluster.create_backup_task(location_list=self.locations) backup_task_status = backup_task.wait_and_get_final_status(timeout=1500) @@ -581,7 +589,7 @@ def test_backup_location_with_path(self): try: mgr_cluster.create_backup_task(location_list=[f'{location}/path_testing/' for location in self.locations]) except ScyllaManagerError as error: - self.log.info('Expected to fail - error: {}'.format(error)) + self.log.info(f'Expected to fail - error: {error}') self.log.info('finishing test_backup_location_with_path') def test_backup_rate_limit(self): @@ -591,12 +599,12 @@ def test_backup_rate_limit(self): or manager_tool.add_cluster(name=self.CLUSTER_NAME, db_cluster=self.db_cluster, auth_token=self.monitors.mgmt_auth_token) rate_limit_list = [f'{dc}:{random.randint(15, 25)}' for dc in self.get_all_dcs_names()] - self.log.info('rate limit will be {}'.format(rate_limit_list)) + self.log.info(f'rate limit will be {rate_limit_list}') backup_task = mgr_cluster.create_backup_task(location_list=self.locations, rate_limit_list=rate_limit_list) task_status = backup_task.wait_and_get_final_status(timeout=18000) assert task_status == TaskStatus.DONE, \ f"Task {backup_task.id} did not end successfully:\n{backup_task.detailed_progress}" - self.log.info('backup task finished with status {}'.format(task_status)) + self.log.info(f'backup task finished with status {task_status}') # TODO: verify that the rate limit is as set in the cmd self.verify_backup_success(mgr_cluster=mgr_cluster, backup_task=backup_task) self.log.info('finishing test_backup_rate_limit') @@ -704,9 +712,9 @@ def test_client_encryption(self): time.sleep(30) # Make sure healthcheck task is triggered healthcheck_task.wait_for_status(list_status=[TaskStatus.DONE], step=5, timeout=240) sleep = 40 - self.log.debug('Sleep {} seconds, waiting for health-check task to run by schedule on first time'.format(sleep)) + self.log.debug(f'Sleep {sleep} seconds, waiting for health-check task to run by schedule on first time') time.sleep(sleep) - self.log.debug("Health-check task history is: {}".format(healthcheck_task.history)) + self.log.debug(f"Health-check task history is: {healthcheck_task.history}") dict_host_health = mgr_cluster.get_hosts_health() for host_health in dict_host_health.values(): assert host_health.ssl == HostSsl.ON, "Not all hosts ssl is 'ON'" @@ -725,18 +733,18 @@ def test_mgmt_cluster_healthcheck(self): host_data for host_data in self.get_cluster_hosts_with_ips() if host_data[1] != self.get_cluster_hosts_ip()[0]][0] sleep = 40 - self.log.debug('Sleep {} seconds, waiting for health-check task to run by schedule on first time'.format(sleep)) + self.log.debug(f'Sleep {sleep} seconds, waiting for health-check task to run by schedule on first time') time.sleep(sleep) healthcheck_task = mgr_cluster.get_healthcheck_task() - self.log.debug("Health-check task history is: {}".format(healthcheck_task.history)) + self.log.debug(f"Health-check task history is: {healthcheck_task.history}") dict_host_health = mgr_cluster.get_hosts_health() for host_health in dict_host_health.values(): assert host_health.status == HostStatus.UP, "Not all hosts status is 'UP'" assert host_health.rest_status == HostRestStatus.UP, "Not all hosts REST status is 'UP'" # Check for sctool status change after scylla-server down other_host.stop_scylla_server() - self.log.debug("Health-check next run is: {}".format(healthcheck_task.next_run)) - self.log.debug('Sleep {} seconds, waiting for health-check task to run after node down'.format(sleep)) + self.log.debug(f"Health-check next run is: {healthcheck_task.next_run}") + self.log.debug(f'Sleep {sleep} seconds, waiting for health-check task to run after node down') time.sleep(sleep) dict_host_health = mgr_cluster.get_hosts_health() assert dict_host_health[other_host_ip].status == HostStatus.DOWN, "Host: {} status is not 'DOWN'".format( @@ -820,11 +828,11 @@ def test_manager_upgrade(self): assert manager_from_version[0] != manager_tool.sctool.version[0], "Manager version not changed after upgrade." # verify all repair tasks exist for repair_task in repair_task_list: - self.log.debug("{} status: {}".format(repair_task.id, repair_task.status)) + self.log.debug(f"{repair_task.id} status: {repair_task.status}") self.log.info('Running a new repair task after upgrade') repair_task = mgr_cluster.create_repair_task() - self.log.debug("{} status: {}".format(repair_task.id, repair_task.status)) + self.log.debug(f"{repair_task.id} status: {repair_task.status}") self.log.info('finishing test_manager_upgrade') def test_manager_rollback_upgrade(self): @@ -855,8 +863,8 @@ def test_repair_multiple_keyspace_types(self): # pylint: disable=invalid-name task_final_status = repair_task.wait_and_get_final_status(timeout=7200) assert task_final_status == TaskStatus.DONE, 'Task: {} final status is: {}.'.format(repair_task.id, str(repair_task.status)) - self.log.info('Task: {} is done.'.format(repair_task.id)) - self.log.debug("sctool version is : {}".format(manager_tool.sctool.version)) + self.log.info(f'Task: {repair_task.id} is done.') + self.log.debug(f"sctool version is : {manager_tool.sctool.version}") expected_keyspaces_to_be_repaired = ["system_auth", "system_distributed", # pylint: disable=invalid-name self.NETWORKSTRATEGY_KEYSPACE_NAME] @@ -865,7 +873,7 @@ def test_repair_multiple_keyspace_types(self): # pylint: disable=invalid-name for keyspace_name in expected_keyspaces_to_be_repaired: keyspace_repair_percentage = per_keyspace_progress.get(keyspace_name, None) assert keyspace_repair_percentage is not None, \ - "The keyspace {} was not included in the repair!".format(keyspace_name) + f"The keyspace {keyspace_name} was not included in the repair!" # noqa: PLR2004 assert keyspace_repair_percentage == 100, \ "The repair of the keyspace {} stopped at {}%".format( # noqa: PLR2004 @@ -953,7 +961,7 @@ def _delete_keyspace_directory(self, db_node, keyspace_name): result = db_node.remoter.sudo(f'rm -rf {directoy_path}') if result.stderr: raise FilesNotCorrupted('Files were not corrupted. CorruptThenRepair nemesis can\'t be run. ' - 'Error: {}'.format(result)) + f'Error: {result}') if directory_size_result.stdout: directory_size = directory_size_result.stdout[:directory_size_result.stdout.find("\t")] self.log.debug("Removed the directory of keyspace {} from node {}\nThe size of the directory is {}".format( @@ -978,7 +986,7 @@ def _insert_data_while_excluding_each_node(self, total_num_of_rows, keyspace_nam # We can't shut down node 1 since it's the default contact point of the stress command, and we have no way # of changing that. As such, we skip it. for node in self.db_cluster.nodes[1:]: - self.log.info("inserting {} rows to every node except {}".format(num_of_rows_per_insertion, node.name)) + self.log.info(f"inserting {num_of_rows_per_insertion} rows to every node except {node.name}") end_of_range = start_of_range + num_of_rows_per_insertion - 1 node.stop_scylla_server(verify_up=False, verify_down=True) stress_thread = self.run_stress_thread(stress_cmd=stress_command_template.format(num_of_rows_per_insertion, @@ -986,7 +994,7 @@ def _insert_data_while_excluding_each_node(self, total_num_of_rows, keyspace_nam start_of_range, end_of_range)) time.sleep(15) - self.log.info('load={}'.format(stress_thread.get_results())) + self.log.info(f'load={stress_thread.get_results()}') node.start_scylla_server(verify_up=True, verify_down=False) start_of_range = end_of_range + 1 with self.db_cluster.cql_connection_patient(self.db_cluster.nodes[0]) as session: @@ -1069,7 +1077,7 @@ def test_suspend_and_resume(self): def _suspend_and_resume_task_template(self, task_type): # task types: backup/repair - self.log.info('starting test_suspend_and_resume_{}'.format(task_type)) + self.log.info(f'starting test_suspend_and_resume_{task_type}') manager_tool = mgmt.get_scylla_manager_tool(manager_node=self.monitors.nodes[0]) mgr_cluster = manager_tool.get_cluster(cluster_name=self.CLUSTER_NAME) \ or manager_tool.add_cluster(name=self.CLUSTER_NAME, db_cluster=self.db_cluster, @@ -1087,12 +1095,12 @@ def _suspend_and_resume_task_template(self, task_type): f"task {suspendable_task.id} failed to reach status {TaskStatus.STOPPED}" assert suspendable_task.wait_for_status(list_status=[TaskStatus.DONE], timeout=1200, step=10), \ f"task {suspendable_task.id} failed to reach status {TaskStatus.DONE}" - self.log.info('finishing test_suspend_and_resume_{}'.format(task_type)) + self.log.info(f'finishing test_suspend_and_resume_{task_type}') def _template_suspend_with_on_resume_start_tasks_flag(self, wait_for_duration): suspension_duration = 75 test_name_filler = "after_duration_passed" if wait_for_duration else "before_duration_passed" - self.log.info('starting test_suspend_with_on_resume_start_tasks_flag_{}'.format(test_name_filler)) + self.log.info(f'starting test_suspend_with_on_resume_start_tasks_flag_{test_name_filler}') manager_tool = mgmt.get_scylla_manager_tool(manager_node=self.monitors.nodes[0]) mgr_cluster = manager_tool.get_cluster(cluster_name=self.CLUSTER_NAME) \ or manager_tool.add_cluster(name=self.CLUSTER_NAME, db_cluster=self.db_cluster, @@ -1123,7 +1131,7 @@ def _template_suspend_with_on_resume_start_tasks_flag(self, wait_for_duration): assert suspendable_task.status == TaskStatus.STOPPED, \ "After the cluster was resumed (while resuming BEFORE the suspend duration "\ f"has passed), task {suspendable_task.id} failed to stay in status STOPPED after suspension time ended" - self.log.info('finishing test_suspend_with_on_resume_start_tasks_flag_{}'.format(test_name_filler)) + self.log.info(f'finishing test_suspend_with_on_resume_start_tasks_flag_{test_name_filler}') def test_suspend_and_resume_without_starting_tasks(self): self.log.info('starting test_suspend_and_resume_without_starting_tasks') diff --git a/mgmt_upgrade_test.py b/mgmt_upgrade_test.py index 6f622d76667..53883db55a3 100644 --- a/mgmt_upgrade_test.py +++ b/mgmt_upgrade_test.py @@ -12,15 +12,17 @@ # Copyright (c) 2021 ScyllaDB import logging -from time import sleep from datetime import datetime, timedelta +from time import sleep -from sdcm.tester import ClusterTester -from sdcm.mgmt import get_scylla_manager_tool, TaskStatus -from sdcm.mgmt.cli import RepairTask -from sdcm.mgmt.common import get_manager_repo_from_defaults, create_cron_list_from_timedelta from mgmt_cli_test import BackupFunctionsMixIn - +from sdcm.mgmt import TaskStatus, get_scylla_manager_tool +from sdcm.mgmt.cli import RepairTask +from sdcm.mgmt.common import ( + create_cron_list_from_timedelta, + get_manager_repo_from_defaults, +) +from sdcm.tester import ClusterTester LOGGER = logging.getLogger(__name__) @@ -70,7 +72,7 @@ def test_upgrade(self): # pylint: disable=too-many-locals,too-many-statements node_ip = scylla_manager_yaml["http"].split(":", maxsplit=1)[0] scylla_manager_yaml["http"] = f"{node_ip}:{new_manager_http_port}" scylla_manager_yaml["prometheus"] = f"{node_ip}:{self.params.get('manager_prometheus_port')}" - LOGGER.info("The new Scylla Manager is:\n{}".format(scylla_manager_yaml)) + LOGGER.info(f"The new Scylla Manager is:\n{scylla_manager_yaml}") manager_node.restart_manager_server(port=new_manager_http_port) manager_tool = get_scylla_manager_tool(manager_node=manager_node) manager_tool.add_cluster(name="cluster_under_test", db_cluster=self.db_cluster, diff --git a/microbenchmarking_test.py b/microbenchmarking_test.py index 57b8573bfa3..bf244d4f57e 100644 --- a/microbenchmarking_test.py +++ b/microbenchmarking_test.py @@ -11,8 +11,11 @@ # # Copyright (c) 2023 ScyllaDB import json -from sdcm.tester import ClusterTester, teardown_on_exception, log_run_info -from sdcm.utils.microbenchmarking.perf_simple_query_reporter import PerfSimpleQueryAnalyzer + +from sdcm.tester import ClusterTester, log_run_info, teardown_on_exception +from sdcm.utils.microbenchmarking.perf_simple_query_reporter import ( + PerfSimpleQueryAnalyzer, +) class PerfSimpleQueryTest(ClusterTester): diff --git a/performance_regression_alternator_test.py b/performance_regression_alternator_test.py index 54b57b6d2e0..2b4634719dc 100644 --- a/performance_regression_alternator_test.py +++ b/performance_regression_alternator_test.py @@ -14,7 +14,10 @@ import contextlib from performance_regression_test import PerformanceRegressionTest -from sdcm.sct_events.group_common_events import ignore_operation_errors, ignore_alternator_client_errors +from sdcm.sct_events.group_common_events import ( + ignore_alternator_client_errors, + ignore_operation_errors, +) from sdcm.utils import alternator diff --git a/performance_regression_cdc_test.py b/performance_regression_cdc_test.py index b4da1cbebd5..00bc6c1f526 100644 --- a/performance_regression_cdc_test.py +++ b/performance_regression_cdc_test.py @@ -1,9 +1,8 @@ import pprint import time -from sdcm.cluster import BaseNode, UnexpectedExit, Failure from performance_regression_test import PerformanceRegressionTest - +from sdcm.cluster import BaseNode, Failure, UnexpectedExit PP = pprint.PrettyPrinter(indent=2) diff --git a/performance_regression_gradual_grow_throughput.py b/performance_regression_gradual_grow_throughput.py index 0e7cd9d7cb8..a7393bfd3e9 100644 --- a/performance_regression_gradual_grow_throughput.py +++ b/performance_regression_gradual_grow_throughput.py @@ -1,8 +1,8 @@ +import json import os -from enum import Enum from collections import defaultdict +from enum import Enum -import json from performance_regression_test import PerformanceRegressionTest from sdcm.results_analyze import ThroughputLatencyGradualGrowPayloadPerformanceAnalyzer @@ -120,7 +120,7 @@ def preload_data(self, compaction_strategy=None): params.update({'stress_cmd': stress_cmd}) # Run all stress commands params.update(dict(stats_aggregate_cmds=False)) - self.log.debug('RUNNING stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'RUNNING stress cmd: {stress_cmd}') stress_queue.append(self.run_stress_thread(**params)) for stress in stress_queue: diff --git a/performance_regression_lwt_test.py b/performance_regression_lwt_test.py index 6afe124fb17..eb8ab4e0cd6 100644 --- a/performance_regression_lwt_test.py +++ b/performance_regression_lwt_test.py @@ -13,12 +13,11 @@ import pprint -from invoke.exceptions import UnexpectedExit, Failure +from invoke.exceptions import Failure, UnexpectedExit from performance_regression_test import PerformanceRegressionTest -from sdcm.utils.decorators import log_run_info, retrying from sdcm.sct_events.group_common_events import ignore_operation_errors - +from sdcm.utils.decorators import log_run_info, retrying PP = pprint.PrettyPrinter(indent=2) diff --git a/performance_regression_row_level_repair_test.py b/performance_regression_row_level_repair_test.py index 00be4fcf354..39a23c4ebee 100644 --- a/performance_regression_row_level_repair_test.py +++ b/performance_regression_row_level_repair_test.py @@ -37,7 +37,7 @@ class PerformanceRegressionRowLevelRepairTest(ClusterTester): @measure_time def _run_repair(self, node): - self.log.info('Running nodetool repair on {}'.format(node.name)) + self.log.info(f'Running nodetool repair on {node.name}') node.run_nodetool(sub_cmd='repair') def _pre_create_schema_large_scale(self, keyspace_num=1, scylla_encryption_options=None): @@ -46,12 +46,12 @@ def _pre_create_schema_large_scale(self, keyspace_num=1, scylla_encryption_optio cassandra-stress. """ - self.log.debug('Pre Creating Schema for c-s with {} keyspaces'.format(keyspace_num)) + self.log.debug(f'Pre Creating Schema for c-s with {keyspace_num} keyspaces') for i in range(1, keyspace_num + 1): - keyspace_name = 'keyspace{}'.format(i) + keyspace_name = f'keyspace{i}' self.create_keyspace(keyspace_name=keyspace_name, replication_factor=3) - self.log.debug('{} Created'.format(keyspace_name)) - table_name = "{}.standard1".format(keyspace_name) + self.log.debug(f'{keyspace_name} Created') + table_name = f"{keyspace_name}.standard1" self.create_table(name=table_name, key_type='blob', read_repair=0.0, columns={'"C0"': 'blob'}, scylla_encryption_options=scylla_encryption_options) @@ -59,8 +59,8 @@ def _pre_create_schema_large_scale(self, keyspace_num=1, scylla_encryption_optio def _update_cl_in_stress_cmd(self, str_stress_cmd, consistency_level): for param in str_stress_cmd.split(): if param.startswith('cl='): - return str_stress_cmd.replace(param, "cl={}".format(consistency_level)) - self.log.debug("Could not find a 'cl' parameter in stress command: {}".format(str_stress_cmd)) + return str_stress_cmd.replace(param, f"cl={consistency_level}") + self.log.debug(f"Could not find a 'cl' parameter in stress command: {str_stress_cmd}") return str_stress_cmd def preload_data(self, consistency_level=None): @@ -86,7 +86,7 @@ def preload_data(self, consistency_level=None): # Run all stress commands params.update(dict(stats_aggregate_cmds=False)) - self.log.debug('RUNNING stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'RUNNING stress cmd: {stress_cmd}') stress_queue.append(self.run_stress_thread(**params)) # One stress cmd command @@ -125,8 +125,8 @@ def _disable_hinted_handoff(self): """.format(yaml_file, tmp_yaml_file)) for node in self.db_cluster.nodes: # disable hinted handoff on all nodes node.remoter.run('sudo bash -cxe "%s"' % disable_hinted_handoff) - self.log.debug("Scylla YAML configuration read from: {} {} is:".format(node.public_ip_address, yaml_file)) - node.remoter.run('sudo cat {}'.format(yaml_file)) + self.log.debug(f"Scylla YAML configuration read from: {node.public_ip_address} {yaml_file} is:") + node.remoter.run(f'sudo cat {yaml_file}') node.stop_scylla_server() node.start_scylla_server() @@ -142,10 +142,10 @@ def _pre_create_schema_scylla_bench(self): session.execute(create_table_query) def _run_scylla_bench_on_single_node(self, node, stress_cmd): - self.log.info('Stopping all other nodes before updating {}'.format(node.name)) + self.log.info(f'Stopping all other nodes before updating {node.name}') self.stop_all_nodes_except_for(node=node) - self.log.info('Updating cluster data only for {}'.format(node.name)) - self.log.info("Run stress command of: {}".format(stress_cmd)) + self.log.info(f'Updating cluster data only for {node.name}') + self.log.info(f"Run stress command of: {stress_cmd}") stress_queue = self.run_stress_thread_bench(stress_cmd=stress_cmd, stats_aggregate_cmds=False, round_robin=True) self.get_stress_results_bench(queue=stress_queue) @@ -166,36 +166,36 @@ def test_row_level_repair_single_node_diff(self): for node in self.db_cluster.nodes: used_capacity = self.get_used_capacity(node=node) - self.log.debug("Node {} initial used capacity is: {}".format(node.public_ip_address, used_capacity)) + self.log.debug(f"Node {node.public_ip_address} initial used capacity is: {used_capacity}") self._disable_hinted_handoff() - self.log.info('Stopping node-3 ({}) before updating cluster data'.format(node1.name)) + self.log.info(f'Stopping node-3 ({node1.name}) before updating cluster data') node1.stop_scylla_server() - self.log.info('Updating cluster data when node3 ({}) is down'.format(node1.name)) + self.log.info(f'Updating cluster data when node3 ({node1.name}) is down') self.log.info('Starting c-s/s-b write workload') self.preload_data() self.wait_no_compactions_running() - self.log.info('Starting node-3 ({}) after updated cluster data'.format(node1.name)) + self.log.info(f'Starting node-3 ({node1.name}) after updated cluster data') node1.start_scylla_server() for node in self.db_cluster.nodes: used_capacity = self.get_used_capacity(node=node) self.log.debug( - "Node {} used capacity after pre-load data is: {}".format(node.public_ip_address, used_capacity)) + f"Node {node.public_ip_address} used capacity after pre-load data is: {used_capacity}") - self.log.info('Run Repair on node: {} , 0% synced'.format(node1.name)) + self.log.info(f'Run Repair on node: {node1.name} , 0% synced') repair_time = self._run_repair(node=node1)[0] # pylint: disable=unsubscriptable-object - self.log.info('Repair (0% synced) time on node: {} is: {}'.format(node1.name, repair_time)) + self.log.info(f'Repair (0% synced) time on node: {node1.name} is: {repair_time}') stats['repair_runtime_all_diff'] = repair_time self.wait_no_compactions_running() - self.log.info('Run Repair on node: {} , 100% synced'.format(node1.name)) + self.log.info(f'Run Repair on node: {node1.name} , 100% synced') repair_time = self._run_repair(node=node1)[0] # pylint: disable=unsubscriptable-object - self.log.info('Repair (100% synced) time on node: {} is: {}'.format(node1.name, repair_time)) + self.log.info(f'Repair (100% synced) time on node: {node1.name} is: {repair_time}') stats['repair_runtime_no_diff'] = repair_time self.update_test_details(scylla_conf=True, extra_stats=stats) @@ -219,14 +219,14 @@ def test_row_level_repair_3_nodes_small_diff(self): self._disable_hinted_handoff() self.print_nodes_used_capacity() for node in [node1, node2, node3]: - self.log.info('Stopping all other nodes before updating {}'.format(node.name)) + self.log.info(f'Stopping all other nodes before updating {node.name}') self.stop_all_nodes_except_for(node=node) - self.log.info('Updating cluster data only for {}'.format(node.name)) + self.log.info(f'Updating cluster data only for {node.name}') distinct_write_cmd = "{} -pop seq={}..{} -node {}".format(base_distinct_write_cmd, sequence_current_index + 1, sequence_current_index + sequence_range, node.private_ip_address) - self.log.info("Run stress command of: {}".format(distinct_write_cmd)) + self.log.info(f"Run stress command of: {distinct_write_cmd}") stress_thread = self.run_stress_thread(stress_cmd=distinct_write_cmd, round_robin=True) self.verify_stress_thread(cs_thread_pool=stress_thread) self.start_all_nodes() @@ -243,13 +243,13 @@ def test_row_level_repair_3_nodes_small_diff(self): self.log.debug("Nodes total used capacity before starting repair is:") self.print_nodes_used_capacity() - self.log.info('Run Repair on node: {} , 99.8% synced'.format(node3.name)) + self.log.info(f'Run Repair on node: {node3.name} , 99.8% synced') repair_time = self._run_repair(node=node3)[0] # pylint: disable=unsubscriptable-object self.log.debug("Nodes total used capacity after repair end is:") self.print_nodes_used_capacity() - self.log.info('Repair (99.8% synced) time on node: {} is: {}'.format(node3.name, repair_time)) + self.log.info(f'Repair (99.8% synced) time on node: {node3.name} is: {repair_time}') stats = {'repair_runtime_small_diff': repair_time} @@ -267,10 +267,10 @@ def _populate_scylla_bench_data_in_parallel(self, base_cmd, partition_count, clu write_queue = [] offset = 0 for _ in range(n_loaders): - str_offset = "-partition-offset {}".format(offset) + str_offset = f"-partition-offset {offset}" stress_cmd = " ".join( [base_cmd, str_additional_args, str_offset]) - self.log.debug('Scylla-bench stress command to execute: {}'.format(stress_cmd)) + self.log.debug(f'Scylla-bench stress command to execute: {stress_cmd}') write_queue.append(self.run_stress_thread_bench(stress_cmd=stress_cmd, stats_aggregate_cmds=False, round_robin=True)) offset += partitions_per_loader @@ -312,7 +312,7 @@ def test_row_level_repair_large_partitions(self): partition_count_per_node, clustering_row_count_per_node) for node in self.db_cluster.nodes: - str_offset = "-partition-offset {}".format(offset) + str_offset = f"-partition-offset {offset}" stress_cmd = " ".join( [scylla_bench_base_cmd, str_additional_args, str_offset]) self._run_scylla_bench_on_single_node(node=node, stress_cmd=stress_cmd) @@ -322,13 +322,13 @@ def test_row_level_repair_large_partitions(self): self.log.debug("Nodes total used capacity before starting repair is:") self.print_nodes_used_capacity() - self.log.info('Run Repair on node: {}'.format(node3.name)) + self.log.info(f'Run Repair on node: {node3.name}') repair_time = self._run_repair(node=node3)[0] # pylint: disable=unsubscriptable-object self.log.debug("Nodes total used capacity after repair end is:") self.print_nodes_used_capacity() - self.log.info('Repair (with large partitions) time on node: {} is: {}'.format(node3.name, repair_time)) + self.log.info(f'Repair (with large partitions) time on node: {node3.name} is: {repair_time}') stats = {'repair_runtime_large_partitions': repair_time} @@ -370,16 +370,16 @@ def test_row_level_repair_during_load(self, preload_data=True): params.update({'stress_cmd': stress_cmd}) # Run stress command params.update(dict(stats_aggregate_cmds=False)) - self.log.debug('RUNNING stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'RUNNING stress cmd: {stress_cmd}') stress_queue.append(self.run_stress_thread(**params)) - self.log.info('Run Repair on node: {} , during r/w load'.format(node1.name)) + self.log.info(f'Run Repair on node: {node1.name} , during r/w load') repair_time = self._run_repair(node=node1)[0] # pylint: disable=unsubscriptable-object self.log.debug("Nodes total used capacity after repair end is:") self.print_nodes_used_capacity() - self.log.info('Repair (during r/w load) time on node: {} is: {}'.format(node1.name, repair_time)) + self.log.info(f'Repair (during r/w load) time on node: {node1.name} is: {repair_time}') stats = {'repair_runtime_during_load': repair_time} diff --git a/performance_regression_test.py b/performance_regression_test.py index ffbc550fbd2..1e973b775ed 100644 --- a/performance_regression_test.py +++ b/performance_regression_test.py @@ -16,19 +16,18 @@ import os import time - from enum import Enum import yaml -from upgrade_test import UpgradeTest -from sdcm.tester import ClusterTester, teardown_on_exception from sdcm.sct_events import Severity from sdcm.sct_events.filters import EventsSeverityChangerFilter from sdcm.sct_events.loaders import CassandraStressEvent from sdcm.sct_events.system import HWPerforanceEvent, InfoEvent -from sdcm.utils.decorators import log_run_info, latency_calculator_decorator +from sdcm.tester import ClusterTester, teardown_on_exception from sdcm.utils.csrangehistogram import CSHistogramTagTypes +from sdcm.utils.decorators import latency_calculator_decorator, log_run_info +from upgrade_test import UpgradeTest KB = 1024 @@ -106,35 +105,35 @@ def display_single_result(self, result): def get_test_xml(self, result, test_name=''): test_content = """ - - "%s test, ami_id: %s, scylla version: - %s", hardware: %s + + "{} test, ami_id: {}, scylla version: + {}", hardware: {} - target-ami_id-%s - target-version-%s + target-ami_id-{} + target-version-{} - %s + {} - + - - - - - - - - - - + + + + + + + + + + -""" % (test_name, result['loader_idx'], +""".format(test_name, result['loader_idx'], result['loader_idx'], result['cpu_idx'], result['keyspace_idx'], @@ -173,11 +172,11 @@ def display_results(self, results, test_name=''): test_xml += self.get_test_xml(single_result, test_name=test_name) with open(os.path.join(self.logdir, 'jenkins_perf_PerfPublisher.xml'), 'w', encoding="utf-8") as pref_file: - content = """%s""" % (test_name, test_xml) + content = f"""{test_xml}""" pref_file.write(content) except Exception as ex: # pylint: disable=broad-except # noqa: BLE001 - self.log.debug('Failed to display results: {0}'.format(results)) - self.log.debug('Exception: {0}'.format(ex)) + self.log.debug(f'Failed to display results: {results}') + self.log.debug(f'Exception: {ex}') def _workload(self, stress_cmd, stress_num, test_name, sub_type=None, keyspace_num=1, prefix='', debug_message='', # pylint: disable=too-many-arguments # noqa: PLR0913 save_stats=True): @@ -195,7 +194,7 @@ def _workload(self, stress_cmd, stress_num, test_name, sub_type=None, keyspace_n self.display_results(results, test_name=test_name) self.check_regression() total_ops = self._get_total_ops() - self.log.debug('Total ops: {}'.format(total_ops)) + self.log.debug(f'Total ops: {total_ops}') return total_ops return None @@ -242,7 +241,7 @@ def preload_data(self, compaction_strategy=None): params.update({'stress_cmd': stress_cmd}) # Run all stress commands params.update(dict(stats_aggregate_cmds=False)) - self.log.debug('RUNNING stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'RUNNING stress cmd: {stress_cmd}') stress_queue.append(self.run_stress_thread(**params)) # One stress cmd command else: @@ -346,12 +345,12 @@ def prepare_mv(self, on_populated=False): base_table_name = 'standard1' if not on_populated: # Truncate base table before materialized view creation - self.log.debug('Truncate base table: {0}.{1}'.format(ks_name, base_table_name)) + self.log.debug(f'Truncate base table: {ks_name}.{base_table_name}') self.truncate_cf(ks_name, base_table_name, session) # Create materialized view view_name = base_table_name + '_mv' - self.log.debug('Create materialized view: {0}.{1}'.format(ks_name, view_name)) + self.log.debug(f'Create materialized view: {ks_name}.{view_name}') self.create_materialized_view(ks_name, base_table_name, view_name, ['"C0"'], ['key'], session, mv_columns=['"C0"', 'key']) @@ -372,7 +371,7 @@ def _write_with_mv(self, on_populated): # Run a write workload without MV ops_without_mv = self._workload(stress_cmd=base_cmd_w, stress_num=2, sub_type='write_without_mv', test_name=test_name, keyspace_num=1, - debug_message='First write cassandra-stress command: {}'.format(base_cmd_w)) + debug_message=f'First write cassandra-stress command: {base_cmd_w}') # Create MV self.prepare_mv(on_populated=on_populated) @@ -380,7 +379,7 @@ def _write_with_mv(self, on_populated): # Start cassandra-stress writes again now with MV ops_with_mv = self._workload(stress_cmd=base_cmd_w, stress_num=2, sub_type='write_with_mv', test_name=test_name, keyspace_num=1, - debug_message='Second write cassandra-stress command: {}'.format(base_cmd_w)) + debug_message=f'Second write cassandra-stress command: {base_cmd_w}') self.assert_mv_performance(ops_without_mv, ops_with_mv, 'Throughput of run with materialized view is more than {} times lower then ' @@ -403,26 +402,26 @@ def _read_with_mv(self, on_populated): self.create_test_stats() # prepare schema and data before read self._workload(stress_cmd=base_cmd_p, stress_num=2, test_name=test_name, prefix='preload-', keyspace_num=1, - debug_message='Prepare the test, run cassandra-stress command: {}'.format(base_cmd_p), + debug_message=f'Prepare the test, run cassandra-stress command: {base_cmd_p}', save_stats=False) # run a read workload ops_without_mv = self._workload(stress_cmd=base_cmd_r, stress_num=2, sub_type='read_without_mv', test_name=test_name, keyspace_num=1, - debug_message='First read cassandra-stress command: {}'.format(base_cmd_r)) + debug_message=f'First read cassandra-stress command: {base_cmd_r}') self.prepare_mv(on_populated=on_populated) # If the MV was created on the empty base table, populate it before reads if not on_populated: self._workload(stress_cmd=base_cmd_w, stress_num=2, test_name=test_name, prefix='preload-', keyspace_num=1, - debug_message='Prepare test before second cassandra-stress command: {}'.format(base_cmd_w), + debug_message=f'Prepare test before second cassandra-stress command: {base_cmd_w}', save_stats=False) # run a read workload ops_with_mv = self._workload(stress_cmd=base_cmd_r, stress_num=2, sub_type='read_with_mv', test_name=test_name, keyspace_num=1, - debug_message='Second read cassandra-stress command: {}'.format(base_cmd_r)) + debug_message=f'Second read cassandra-stress command: {base_cmd_r}') self.assert_mv_performance(ops_without_mv, ops_with_mv, 'Throughput of run with materialized view is more than {} times lower then ' @@ -442,13 +441,13 @@ def _mixed_with_mv(self, on_populated): self.create_test_stats() # run a write workload as a preparation self._workload(stress_cmd=base_cmd_p, stress_num=2, test_name=test_name, keyspace_num=1, prefix='preload-', - debug_message='Prepare the test, run cassandra-stress command: {}'.format(base_cmd_p), + debug_message=f'Prepare the test, run cassandra-stress command: {base_cmd_p}', save_stats=False) # run a mixed workload without MV ops_without_mv = self._workload(stress_cmd=base_cmd_m, stress_num=2, sub_type='mixed_without_mv', test_name=test_name, keyspace_num=1, - debug_message='First mixed cassandra-stress command: {}'.format(base_cmd_m)) + debug_message=f'First mixed cassandra-stress command: {base_cmd_m}') self.prepare_mv(on_populated=on_populated) @@ -464,7 +463,7 @@ def _mixed_with_mv(self, on_populated): self.ops_threshold_prc / 100)) def assert_mv_performance(self, ops_without_mv, ops_with_mv, failure_message): - self.log.debug('Performance results. Ops without MV: {0}; Ops with MV: {1}'.format(ops_without_mv, ops_with_mv)) + self.log.debug(f'Performance results. Ops without MV: {ops_without_mv}; Ops with MV: {ops_with_mv}') self.assertLessEqual(ops_without_mv, (ops_with_mv * self.ops_threshold_prc) / 100, failure_message) def _scylla_bench_prepare_table(self): @@ -641,16 +640,16 @@ def test_mv_write(self): 5. Drop MV """ def run_workload(stress_cmd, user_profile): - self.log.debug('Run stress test with user profile {}'.format(user_profile)) - assert os.path.exists(user_profile), 'File not found: {}'.format(user_profile) - self.log.debug('Stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'Run stress test with user profile {user_profile}') + assert os.path.exists(user_profile), f'File not found: {user_profile}' + self.log.debug(f'Stress cmd: {stress_cmd}') stress_queue = self.run_stress_thread(stress_cmd=stress_cmd, stress_num=1, profile=user_profile, stats_aggregate_cmds=False) results = self.get_stress_results(queue=stress_queue) self.update_test_details(scylla_conf=True) self.display_results(results, test_name=test_name) self.check_regression() - self.log.debug('Finish stress test with user profile {}'.format(user_profile)) + self.log.debug(f'Finish stress test with user profile {user_profile}') def get_mv_name(user_profile): @@ -666,29 +665,29 @@ def get_mv_name(user_profile): break if not mv_name: - assert False, 'Failed to recognoze materialized view name from {0}: {1}'.format( + assert False, 'Failed to recognoze materialized view name from {}: {}'.format( user_profile, user_profile_yaml) return mv_name def drop_mv(mv_name): # drop MV - self.log.debug('Start dropping materialized view {}'.format(mv_name)) - query = 'drop materialized view {}'.format(mv_name) + self.log.debug(f'Start dropping materialized view {mv_name}') + query = f'drop materialized view {mv_name}' try: with self.db_cluster.cql_connection_patient_exclusive(self.db_cluster.nodes[0]) as session: - self.log.debug('Run query: {}'.format(query)) + self.log.debug(f'Run query: {query}') session.execute(query) except Exception as ex: - self.log.debug('Failed to drop materialized view using query {0}. Error: {1}'.format(query, str(ex))) + self.log.debug(f'Failed to drop materialized view using query {query}. Error: {str(ex)}') raise - self.log.debug('Finish dropping materialized view {}'.format(mv_name)) + self.log.debug(f'Finish dropping materialized view {mv_name}') test_name = 'test_mv_write' duration = self.params.get('test_duration') - self.log.debug('Start materialized views performance test. Test duration {} minutes'.format(duration)) + self.log.debug(f'Start materialized views performance test. Test duration {duration} minutes') self.create_test_stats() cmd_no_mv = self.params.get('stress_cmd_no_mv') cmd_no_mv_profile = self.params.get('stress_cmd_no_mv_profile') diff --git a/performance_regression_user_profiles_test.py b/performance_regression_user_profiles_test.py index f6693fe5fce..fa1765e6f21 100644 --- a/performance_regression_user_profiles_test.py +++ b/performance_regression_user_profiles_test.py @@ -31,10 +31,10 @@ def _clean_keyspace(self, cs_profile): # pylint: disable=invalid-name with open(cs_profile, encoding="utf-8") as fdr: key_space = [line.split(':')[-1].strip() for line in fdr.readlines() if line.startswith('keyspace:')] if key_space: - self.log.debug('Drop keyspace {}'.format(key_space[0])) + self.log.debug(f'Drop keyspace {key_space[0]}') with self.db_cluster.cql_connection_patient(self.db_cluster.nodes[0]) as session: # pylint: disable=no-member - session.execute('DROP KEYSPACE IF EXISTS {};'.format(key_space[0])) + session.execute(f'DROP KEYSPACE IF EXISTS {key_space[0]};') def test_user_profiles(self): """ @@ -44,14 +44,14 @@ def test_user_profiles(self): user_profiles = self.params.get('cs_user_profiles') assert user_profiles is not None, 'No user profiles defined!' for cs_profile in user_profiles: - assert os.path.exists(cs_profile), 'File not found: {}'.format(cs_profile) - self.log.debug('Run stress test with user profile {}, duration {}'.format(cs_profile, duration)) + assert os.path.exists(cs_profile), f'File not found: {cs_profile}' + self.log.debug(f'Run stress test with user profile {cs_profile}, duration {duration}') profile_dst = os.path.join('/tmp', os.path.basename(cs_profile)) with open(cs_profile, encoding="utf-8") as pconf: cont = pconf.readlines() for cmd in [line.lstrip('#').strip() for line in cont if line.find('cassandra-stress') > 0]: stress_cmd = (cmd.format(profile_dst, duration)) - self.log.debug('Stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'Stress cmd: {stress_cmd}') self.create_test_stats() stress_queue = self.run_stress_thread(stress_cmd=stress_cmd, stress_num=2, profile=cs_profile) self.get_stress_results(queue=stress_queue) diff --git a/performance_search_max_throughput_test.py b/performance_search_max_throughput_test.py index ddd3488c35e..5af331e4ba6 100644 --- a/performance_search_max_throughput_test.py +++ b/performance_search_max_throughput_test.py @@ -1,6 +1,5 @@ -import re import json - +import re from collections import defaultdict from typing import Any diff --git a/pyproject.toml b/pyproject.toml index e9fe87306ef..c74e6c808d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.ruff] -select = ["PL", "YTT", "BLE"] +select = ["PL", "YTT", "BLE", "I", "UP"] ignore = ["E501"] diff --git a/query_limits_test.py b/query_limits_test.py index 9a4da7baecc..6163f2a0679 100644 --- a/query_limits_test.py +++ b/query_limits_test.py @@ -15,8 +15,7 @@ import logging -from sdcm.tester import ClusterTester -from sdcm.tester import teardown_on_exception +from sdcm.tester import ClusterTester, teardown_on_exception class QueryLimitsTest(ClusterTester): @@ -81,7 +80,7 @@ def setUp(self): def test_connection_limits(self): ips = self.db_cluster.get_node_private_ips() params = " --servers %s --duration 600 --queries 1000000" % (ips[0]) - cmd = '%s %s' % (self.payload, params) + cmd = f'{self.payload} {params}' result = self.loaders.nodes[0].remoter.run(cmd, ignore_status=True) if result.exit_status != 0: self.fail('Payload failed:\n%s' % result) diff --git a/sct.py b/sct.py index 7eaa0551160..09c77bec35b 100755 --- a/sct.py +++ b/sct.py @@ -14,102 +14,130 @@ # Copyright (c) 2021 ScyllaDB # pylint: disable=too-many-lines +import glob +import logging import os +import pprint import re +import subprocess import sys -import unittest -import logging -import glob import time -import subprocess import traceback +import unittest import uuid -import pprint from concurrent.futures import ProcessPoolExecutor -from pathlib import Path from functools import partial -from typing import List +from pathlib import Path from uuid import UUID -import pytest import click +import pytest import yaml -from prettytable import PrettyTable -from argus.client.sct.types import LogLink -from argus.client.base import ArgusClientError from argus.backend.util.enums import TestStatus +from argus.client.base import ArgusClientError +from argus.client.sct.types import LogLink +from prettytable import PrettyTable import sct_ssh +import sdcm.provision.azure.utils as azure_utils +from sdcm.cluster_k8s import mini_k8s from sdcm.localhost import LocalHost +from sdcm.monitorstack import ( + get_monitoring_stack_services, + kill_running_monitoring_stack_services, + restore_monitoring_stack, +) +from sdcm.parallel_timeline_report.generate_pt_report import ( + ParallelTimelinesReportGenerator, +) from sdcm.provision import AzureProvisioner from sdcm.provision.provisioner import VmInstance from sdcm.remote import LOCALRUNNER -from sdcm.results_analyze import PerformanceResultsAnalyzer, BaseResultsAnalyzer +from sdcm.results_analyze import BaseResultsAnalyzer, PerformanceResultsAnalyzer from sdcm.sct_config import SCTConfiguration -from sdcm.sct_provision.common.layout import SCTProvisionLayout, create_sct_configuration +from sdcm.sct_provision.common.layout import ( + SCTProvisionLayout, + create_sct_configuration, +) from sdcm.sct_provision.instances_provider import provision_sct_resources -from sdcm.sct_runner import AwsSctRunner, GceSctRunner, AzureSctRunner, get_sct_runner, clean_sct_runners, \ - update_sct_runner_tags -from sdcm.utils.ci_tools import get_job_name, get_job_url -from sdcm.utils.git import get_git_commit_id, get_git_status_info +from sdcm.sct_runner import ( + AwsSctRunner, + AzureSctRunner, + GceSctRunner, + clean_sct_runners, + get_sct_runner, + update_sct_runner_tags, +) +from sdcm.send_email import ( + build_reporter, + get_running_instances_for_email_report, + read_email_data_from_file, + send_perf_email, +) from sdcm.utils.argus import get_argus_client +from sdcm.utils.aws_builder import AwsBuilder, AwsCiBuilder +from sdcm.utils.aws_peering import AwsVpcPeering +from sdcm.utils.aws_region import AwsRegion +from sdcm.utils.aws_utils import AwsArchType from sdcm.utils.azure_region import AzureRegion -from sdcm.utils.cloud_monitor import cloud_report, cloud_qa_report +from sdcm.utils.ci_tools import get_job_name, get_job_url +from sdcm.utils.cloud_monitor import cloud_qa_report, cloud_report from sdcm.utils.cloud_monitor.cloud_monitor import cloud_non_qa_report from sdcm.utils.common import ( aws_tags_to_dict, - create_pretty_table, clean_cloud_resources, clean_resources_according_post_behavior, + create_pretty_table, format_timestamp, + gce_meta_to_dict, get_ami_images, get_ami_images_versioned, + get_builder_by_test_id, get_gce_images, get_gce_images_versioned, - gce_meta_to_dict, - get_builder_by_test_id, get_testrun_dir, + list_cloudformation_stacks_aws, list_clusters_eks, list_clusters_gke, list_elastic_ips_aws, - list_test_security_groups, - list_load_balancers_aws, - list_cloudformation_stacks_aws, list_instances_aws, - list_placement_groups_aws, list_instances_gce, + list_load_balancers_aws, list_logs_by_test_id, - list_resources_docker, list_parallel_timelines_report_urls, - search_test_id_in_latest + list_placement_groups_aws, + list_resources_docker, + list_test_security_groups, + search_test_id_in_latest, ) -from sdcm.utils.nemesis import NemesisJobGenerator -from sdcm.utils.net import get_sct_runner_ip -from sdcm.utils.jepsen import JepsenResults +from sdcm.utils.context_managers import environment from sdcm.utils.docker_utils import docker_hub_login -from sdcm.monitorstack import (restore_monitoring_stack, get_monitoring_stack_services, - kill_running_monitoring_stack_services) -from sdcm.utils.log import setup_stdout_logger -from sdcm.utils.aws_region import AwsRegion -from sdcm.utils.aws_builder import AwsCiBuilder, AwsBuilder -from sdcm.utils.gce_region import GceRegion +from sdcm.utils.es_index import create_index, get_mapping from sdcm.utils.gce_builder import GceBuilder -from sdcm.utils.aws_peering import AwsVpcPeering -from sdcm.utils.get_username import get_username -from sdcm.utils.sct_cmd_helpers import add_file_logger, CloudRegion, get_test_config, get_all_regions -from sdcm.send_email import get_running_instances_for_email_report, read_email_data_from_file, build_reporter, \ - send_perf_email -from sdcm.parallel_timeline_report.generate_pt_report import ParallelTimelinesReportGenerator -from sdcm.utils.aws_utils import AwsArchType +from sdcm.utils.gce_region import GceRegion from sdcm.utils.gce_utils import SUPPORTED_PROJECTS, gce_public_addresses -from sdcm.utils.context_managers import environment -from sdcm.cluster_k8s import mini_k8s -from sdcm.utils.es_index import create_index, get_mapping +from sdcm.utils.get_username import get_username +from sdcm.utils.git import get_git_commit_id, get_git_status_info +from sdcm.utils.jepsen import JepsenResults +from sdcm.utils.log import setup_stdout_logger +from sdcm.utils.nemesis import NemesisJobGenerator +from sdcm.utils.net import get_sct_runner_ip +from sdcm.utils.sct_cmd_helpers import ( + CloudRegion, + add_file_logger, + get_all_regions, + get_test_config, +) from sdcm.utils.version_utils import get_s3_scylla_repos_mapping -import sdcm.provision.azure.utils as azure_utils -from utils.build_system.create_test_release_jobs import JenkinsPipelines # pylint: disable=no-name-in-module,import-error -from utils.get_supported_scylla_base_versions import UpgradeBaseVersion # pylint: disable=no-name-in-module,import-error -from utils.mocks.aws_mock import AwsMock # pylint: disable=no-name-in-module,import-error +from utils.build_system.create_test_release_jobs import ( + JenkinsPipelines, # pylint: disable=no-name-in-module,import-error +) +from utils.get_supported_scylla_base_versions import ( + UpgradeBaseVersion, # pylint: disable=no-name-in-module,import-error +) +from utils.mocks.aws_mock import ( + AwsMock, # pylint: disable=no-name-in-module,import-error +) SUPPORTED_CLOUDS = ("aws", "gce", "azure",) DEFAULT_CLOUD = SUPPORTED_CLOUDS[0] @@ -131,7 +159,7 @@ def install_callback(ctx, _, value): return value shell, path = "bash", Path.home() / '.bash_completion' path.write_text((Path(__file__).parent / 'utils' / '.bash_completion').read_text()) - click.echo('%s completion installed in %s' % (shell, path)) + click.echo(f'{shell} completion installed in {path}') return sys.exit(0) @@ -554,7 +582,7 @@ def list_resources(ctx, user, test_id, get_all, get_all_running, verbose): # no click.secho("Nothing found for selected filters in Docker!", fg="yellow") click.secho("Checking Azure instances...", fg='green') - instances: List[VmInstance] = [] + instances: list[VmInstance] = [] for provisioner in AzureProvisioner.discover_regions(params.get("TestId", "")): instances += provisioner.list_instances() if user: @@ -824,9 +852,9 @@ def lint_yamls(backend, exclude: str, include: str): # pylint: disable=too-many for root, _, files in os.walk('./test-cases'): for file in files: full_path = os.path.join(root, file) - if not any((flt.search(file) or flt.search(full_path) for flt in include_filters)): + if not any(flt.search(file) or flt.search(full_path) for flt in include_filters): continue - if any((flt.search(file) or flt.search(full_path) for flt in exclude_filters)): + if any(flt.search(file) or flt.search(full_path) for flt in exclude_filters): continue features.append(process_pool.submit(_run_yaml_test, backend, full_path, original_env)) @@ -936,7 +964,7 @@ def show_log(test_id, output_format): table.align = "l" for log in files: table.add_row([log["date"].strftime("%Y%m%d_%H%M%S"), log["type"], log["link"]]) - click.echo(table.get_string(title="Log links for testrun with test id {}".format(test_id))) + click.echo(table.get_string(title=f"Log links for testrun with test id {test_id}")) elif output_format == 'markdown': click.echo("\n## Logs\n") for log in files: @@ -951,7 +979,7 @@ def show_log(test_id, output_format): def show_monitor(test_id, date_time, kill, cluster_name): add_file_logger() - click.echo('Search monitoring stack archive files for test id {} and restoring...'.format(test_id)) + click.echo(f'Search monitoring stack archive files for test id {test_id} and restoring...') containers = {} try: containers = restore_monitoring_stack(test_id, date_time) @@ -1007,7 +1035,7 @@ def search_builder(test_id): for result in results: tbl.add_row([result['builder']['name'], result['builder']['public_ip'], result['path']]) - click.echo(tbl.get_string(title='Found builders for Test-id: {}'.format(test_id))) + click.echo(tbl.get_string(title=f'Found builders for Test-id: {test_id}')) @investigate.command('show-events', help='Return content of file events_log/events for running job by test-id') @@ -1051,7 +1079,7 @@ def show_events(test_id: str, follow: bool = False, last_n: int = None, save_to: @click.option("-t", "--test", required=False, default="", help="Run specific test file from unit-tests directory") def unit_tests(test): - sys.exit(pytest.main(['-v', '-p', 'no:warnings', '-m', 'not integration', 'unit_tests/{}'.format(test)])) + sys.exit(pytest.main(['-v', '-p', 'no:warnings', '-m', 'not integration', f'unit_tests/{test}'])) @cli.command('integration-tests', help="Run all the SCT internal integration-tests") @@ -1071,7 +1099,7 @@ def integration_tests(test): ) local_cluster.setup_prerequisites() - sys.exit(pytest.main(['-v', '-p', 'no:warnings', '-m', 'integration', 'unit_tests/{}'.format(test)])) + sys.exit(pytest.main(['-v', '-p', 'no:warnings', '-m', 'integration', f'unit_tests/{test}'])) @cli.command('pre-commit', help="Run pre-commit checkers") @@ -1088,7 +1116,7 @@ def pre_commit(): sys.exit(result) -class OutputLogger(): +class OutputLogger: def __init__(self, filename, terminal): self.terminal = terminal self.log = open(filename, "a", encoding="utf-8") # pylint: disable=consider-using-with @@ -1209,7 +1237,7 @@ def collect_logs(test_id=None, logdir=None, backend=None, config_file=None): current_cluster_type = link.split("/")[-1].split("-")[0] table.add_row([current_cluster_type, link]) - click.echo(table.get_string(title="Collected logs by test-id: {}".format(collector.test_id))) + click.echo(table.get_string(title=f"Collected logs by test-id: {collector.test_id}")) update_sct_runner_tags(test_id=collector.test_id, tags={"logs_collected": True}) if collector.test_id: diff --git a/sct_ssh.py b/sct_ssh.py index 9dc708421c8..216e9abdec3 100644 --- a/sct_ssh.py +++ b/sct_ssh.py @@ -18,22 +18,22 @@ import click import questionary -from questionary import Choice from google.cloud import compute_v1 +from questionary import Choice +from sdcm.utils.aws_region import AwsRegion from sdcm.utils.common import ( - list_instances_aws, + gce_meta_to_dict, get_free_port, + list_instances_aws, list_instances_gce, - gce_meta_to_dict, ) from sdcm.utils.gce_utils import ( - gce_public_addresses, gce_private_addresses, - get_gce_compute_instances_client, + gce_public_addresses, gce_set_tags, + get_gce_compute_instances_client, ) -from sdcm.utils.aws_region import AwsRegion def get_region(instance: dict) -> str: diff --git a/sdcm/__init__.py b/sdcm/__init__.py index aca95dbcd99..c536786a9d1 100644 --- a/sdcm/__init__.py +++ b/sdcm/__init__.py @@ -13,7 +13,6 @@ import os - SCT_ROOT = os.path.dirname(os.path.dirname(__file__)) diff --git a/sdcm/audit.py b/sdcm/audit.py index 6f1c696aca3..e05b3bbbef7 100644 --- a/sdcm/audit.py +++ b/sdcm/audit.py @@ -12,10 +12,13 @@ # Copyright (c) 2023 ScyllaDB import logging from dataclasses import dataclass -from datetime import datetime, date -from typing import Literal, Optional, List +from datetime import date, datetime +from typing import Literal -from cassandra.util import uuid_from_time, datetime_from_uuid1 # pylint: disable=no-name-in-module +from cassandra.util import ( # pylint: disable=no-name-in-module + datetime_from_uuid1, + uuid_from_time, +) from sdcm.sct_events import Severity from sdcm.sct_events.system import InfoEvent @@ -30,9 +33,9 @@ class AuditConfiguration: """https://enterprise.docs.scylladb.com/stable/operating-scylla/security/auditing.html""" store: AuditStore # if store is none, then audit is disabled - categories: List[AuditCategory] - tables: List[str] - keyspaces: List[str] + categories: list[AuditCategory] + tables: list[str] + keyspaces: list[str] @classmethod def from_scylla_yaml(cls, scylla_yaml): @@ -63,21 +66,21 @@ class AuditLogReader: # pylint: disable=too-few-public-methods def __init__(self, cluster): self._cluster = cluster - def read(self, from_datetime: Optional[datetime] = None, - category: Optional[AuditCategory] = None, - operation: Optional[str] = None, + def read(self, from_datetime: datetime | None = None, + category: AuditCategory | None = None, + operation: str | None = None, limit_rows: int = 1000 - ) -> List[AuditLogRow]: + ) -> list[AuditLogRow]: raise NotImplementedError() class TableAuditLogReader(AuditLogReader): # pylint: disable=too-few-public-methods - def read(self, from_datetime: Optional[datetime] = None, - category: Optional[AuditCategory] = None, - operation: Optional[str] = None, + def read(self, from_datetime: datetime | None = None, + category: AuditCategory | None = None, + operation: str | None = None, limit_rows: int = 1000 - ) -> List[AuditLogRow]: + ) -> list[AuditLogRow]: """Return audit log rows based on the given filters.""" where_list = [] if from_datetime: @@ -109,11 +112,11 @@ def read(self, from_datetime: Optional[datetime] = None, def get_audit_log_rows(node, # pylint: disable=too-many-locals - from_datetime: Optional[datetime] = None, - category: Optional[AuditCategory] = None, - operation: Optional[str] = None, + from_datetime: datetime | None = None, + category: AuditCategory | None = None, + operation: str | None = None, limit_rows: int = 1000 - ) -> List[AuditLogRow]: + ) -> list[AuditLogRow]: with node.open_system_log(on_datetime=from_datetime) as log_file: found_rows = 0 for line in log_file: @@ -154,9 +157,9 @@ def get_audit_log_rows(node, # pylint: disable=too-many-locals class SyslogAuditLogReader(AuditLogReader): # pylint: disable=too-few-public-methods - def read(self, from_datetime: Optional[datetime] = None, category: Optional[AuditCategory] = None, - operation: Optional[str] = None, - limit_rows: int = 1000) -> List: + def read(self, from_datetime: datetime | None = None, category: AuditCategory | None = None, + operation: str | None = None, + limit_rows: int = 1000) -> list: """Return audit log rows from syslog based on the given filters.""" rows = [] for node in self._cluster.nodes: @@ -203,9 +206,9 @@ def configure(self, audit_configuration: AuditConfiguration): LOGGER.debug("Audit configuration completed") self._configuration = audit_configuration - def get_audit_log(self, from_datetime: Optional[datetime] = None, category: Optional[AuditCategory] = None, - operation: Optional[str] = None, - limit_rows: int = 1000) -> List: + def get_audit_log(self, from_datetime: datetime | None = None, category: AuditCategory | None = None, + operation: str | None = None, + limit_rows: int = 1000) -> list: """Return audit log rows based on the given filters.""" reader: AuditLogReader if not self.is_enabled(): diff --git a/sdcm/cassandra_harry_thread.py b/sdcm/cassandra_harry_thread.py index 34e30fe3365..1a3f5d28a8e 100644 --- a/sdcm/cassandra_harry_thread.py +++ b/sdcm/cassandra_harry_thread.py @@ -11,19 +11,21 @@ # # Copyright (c) 2021 ScyllaDB +import logging import os -import uuid import time -import logging +import uuid from sdcm.loader import CassandraHarryStressExporter from sdcm.prometheus import nemesis_metrics_obj -from sdcm.sct_events.loaders import CassandraHarryEvent, CASSANDRA_HARRY_ERROR_EVENTS_PATTERNS -from sdcm.utils.docker_remote import RemoteDocker -from sdcm.stress_thread import DockerBasedStressThread +from sdcm.sct_events.loaders import ( + CASSANDRA_HARRY_ERROR_EVENTS_PATTERNS, + CassandraHarryEvent, +) from sdcm.stress.base import format_stress_cmd_error +from sdcm.stress_thread import DockerBasedStressThread from sdcm.utils.common import FileFollowerThread - +from sdcm.utils.docker_remote import RemoteDocker LOGGER = logging.getLogger(__name__) diff --git a/sdcm/cdclog_reader_thread.py b/sdcm/cdclog_reader_thread.py index 2e0bc7df4ff..a39ad50cfea 100644 --- a/sdcm/cdclog_reader_thread.py +++ b/sdcm/cdclog_reader_thread.py @@ -12,16 +12,14 @@ # Copyright (c) 2020 ScyllaDB import logging -import uuid import pprint +import uuid from pathlib import Path -from typing import List, Dict from sdcm.sct_events.loaders import CDCReaderStressEvent -from sdcm.utils.docker_remote import RemoteDocker -from sdcm.stress.base import format_stress_cmd_error, DockerBasedStressThread +from sdcm.stress.base import DockerBasedStressThread, format_stress_cmd_error from sdcm.utils.cdc.options import CDC_LOGTABLE_SUFFIX - +from sdcm.utils.docker_remote import RemoteDocker LOGGER = logging.getLogger(__name__) @@ -88,7 +86,7 @@ def _run_stress(self, loader, loader_idx, cpu_idx): # pylint: disable=unused-ar return None @staticmethod - def _parse_cdcreaderstressor_results(lines: List[str]) -> Dict: + def _parse_cdcreaderstressor_results(lines: list[str]) -> dict: """parse result of cdcreader results lines: Results: @@ -148,7 +146,7 @@ def _parse_cdcreaderstressor_results(lines: List[str]) -> Dict: LOGGER.debug(result) return result - def get_results(self) -> List[Dict]: + def get_results(self) -> list[dict]: """Return results of cdclog readers return list of dicts: diff --git a/sdcm/cluster.py b/sdcm/cluster.py index 80e3e9c1acc..e7d62619203 100644 --- a/sdcm/cluster.py +++ b/sdcm/cluster.py @@ -12,140 +12,194 @@ # Copyright (c) 2016 ScyllaDB # pylint: disable=too-many-lines -import contextlib -import queue +import ipaddress +import itertools +import json import logging import os -import shutil -import sys +import queue import random import re +import shutil +import sys import tempfile import threading import time import traceback -import itertools -import json -import ipaddress - -from typing import List, Optional, Dict, Union, Set, Iterable, ContextManager, Any, IO, AnyStr -from datetime import datetime -from textwrap import dedent -from functools import cached_property, wraps from collections import defaultdict -from collections.abc import Iterator +from collections.abc import Iterable, Iterator +from contextlib import AbstractContextManager, ExitStack, contextmanager from dataclasses import dataclass +from datetime import datetime +from functools import cached_property, wraps from pathlib import Path -from contextlib import ExitStack, contextmanager -import packaging.version +from textwrap import dedent +from typing import ( + IO, + Any, + AnyStr, +) -import yaml +import packaging.version import requests -from paramiko import SSHException -from tenacity import RetryError -from invoke.exceptions import UnexpectedExit, Failure +import yaml +from argus.backend.util.enums import ResourceState from cassandra import ConsistencyLevel from cassandra.auth import PlainTextAuthProvider -from cassandra.cluster import Cluster as ClusterDriver # pylint: disable=no-name-in-module +from cassandra.cluster import ( + Cluster as ClusterDriver, # pylint: disable=no-name-in-module +) from cassandra.cluster import NoHostAvailable # pylint: disable=no-name-in-module -from cassandra.policies import RetryPolicy -from cassandra.policies import WhiteListRoundRobinPolicy, HostFilterPolicy, RoundRobinPolicy +from cassandra.policies import ( + HostFilterPolicy, + RetryPolicy, + RoundRobinPolicy, + WhiteListRoundRobinPolicy, +) from cassandra.query import SimpleStatement # pylint: disable=no-name-in-module +from invoke.exceptions import Failure, UnexpectedExit +from paramiko import SSHException +from tenacity import RetryError -from argus.backend.util.enums import ResourceState -from sdcm.node_exporter_setup import NodeExporterSetup +from sdcm import mgmt, wait +from sdcm.coredump import CoredumpExportSystemdThread from sdcm.db_log_reader import DbLogReader +from sdcm.exceptions import ( + KillNemesis, + NodeNotReady, + SstablesNotFound, +) +from sdcm.keystore import KeyStore +from sdcm.log import SDCMAdapter +from sdcm.logcollector import ( + GrafanaScreenShot, + GrafanaSnapshot, + PrometheusSnapshots, + collect_diagnostic_data, + save_kallsyms_map, + upload_archive_to_s3, +) from sdcm.mgmt import AnyManagerCluster, ScyllaManagerError from sdcm.mgmt.common import get_manager_repo_from_defaults, get_manager_scylla_backend -from sdcm.prometheus import start_metrics_server, PrometheusAlertManagerListener, AlertSilencer -from sdcm.log import SDCMAdapter +from sdcm.monitorstack.ui import AlternatorDashboard +from sdcm.node_exporter_setup import NodeExporterSetup +from sdcm.paths import ( + SCYLLA_MANAGER_AGENT_YAML_PATH, + SCYLLA_MANAGER_TLS_CERT_FILE, + SCYLLA_MANAGER_TLS_KEY_FILE, + SCYLLA_MANAGER_YAML_PATH, + SCYLLA_PROPERTIES_PATH, + SCYLLA_YAML_PATH, +) +from sdcm.prometheus import ( + AlertSilencer, + PrometheusAlertManagerListener, + start_metrics_server, +) from sdcm.provision.common.configuration_script import ConfigurationScriptBuilder +from sdcm.provision.helpers.certificate import ( + install_client_certificate, + install_encryption_at_rest_files, +) from sdcm.provision.scylla_yaml import ScyllaYamlNodeAttrBuilder -from sdcm.provision.scylla_yaml.certificate_builder import ScyllaYamlCertificateAttrBuilder - +from sdcm.provision.scylla_yaml.certificate_builder import ( + ScyllaYamlCertificateAttrBuilder, +) from sdcm.provision.scylla_yaml.cluster_builder import ScyllaYamlClusterAttrBuilder from sdcm.provision.scylla_yaml.scylla_yaml import ScyllaYaml -from sdcm.provision.helpers.certificate import install_client_certificate, install_encryption_at_rest_files -from sdcm.remote import RemoteCmdRunnerBase, LOCALRUNNER, NETWORK_EXCEPTIONS, shell_script_cmd, RetryableNetworkException +from sdcm.remote import ( + LOCALRUNNER, + NETWORK_EXCEPTIONS, + RemoteCmdRunnerBase, + RetryableNetworkException, + shell_script_cmd, +) from sdcm.remote.libssh2_client import UnexpectedExit as Libssh2_UnexpectedExit -from sdcm.remote.remote_file import remote_file, yaml_file_to_dict, dict_to_yaml_file -from sdcm import wait, mgmt +from sdcm.remote.remote_file import dict_to_yaml_file, remote_file, yaml_file_to_dict from sdcm.sct_config import SCTConfiguration +from sdcm.sct_events import Severity +from sdcm.sct_events.base import LogEvent, add_severity_limit_rules, max_severity from sdcm.sct_events.continuous_event import ContinuousEventsRegistry -from sdcm.sct_events.system import AwsKmsEvent +from sdcm.sct_events.database import ( + SYSTEM_ERROR_EVENTS, + SYSTEM_ERROR_EVENTS_PATTERNS, + ScyllaHelpErrorEvent, + ScyllaYamlUpdateEvent, +) +from sdcm.sct_events.decorators import raise_event_on_failure +from sdcm.sct_events.filters import EventsSeverityChangerFilter +from sdcm.sct_events.grafana import set_grafana_url +from sdcm.sct_events.health import ClusterHealthValidatorEvent +from sdcm.sct_events.nodetool import NodetoolEvent +from sdcm.sct_events.system import ( + INSTANCE_STATUS_EVENTS_PATTERNS, + AwsKmsEvent, + InfoEvent, + TestFrameworkEvent, +) +from sdcm.sct_provision.aws.user_data import ScyllaUserDataBuilder from sdcm.snitch_configuration import SnitchConfig -from sdcm.utils import properties +from sdcm.test_config import TestConfig +from sdcm.utils import cdc, properties from sdcm.utils.adaptive_timeouts import Operations, adaptive_timeout +from sdcm.utils.auto_ssh import AutoSshContainerMixin from sdcm.utils.aws_kms import AwsKms from sdcm.utils.benchmarks import ScyllaClusterBenchmarkManager +from sdcm.utils.ci_tools import get_test_name from sdcm.utils.common import ( + PageFetcher, S3Storage, ScyllaCQLSession, - PageFetcher, deprecation, - get_data_dir_path, - verify_scylla_repo_file, - normalize_ipv6_url, download_dir_from_cloud, generate_random_string, + get_data_dir_path, + get_sct_root_path, + normalize_ipv6_url, prepare_and_start_saslauthd_service, raise_exception_in_thread, - get_sct_root_path, + verify_scylla_repo_file, +) +from sdcm.utils.decorators import ( + NoValue, + log_run_info, + optional_cached_property, + retrying, ) -from sdcm.utils.ci_tools import get_test_name from sdcm.utils.distro import Distro -from sdcm.utils.install import InstallMode from sdcm.utils.docker_utils import ContainerManager, NotFound, docker_hub_login -from sdcm.utils.health_checker import check_nodes_status, check_node_status_in_gossip_and_nodetool_status, \ - check_schema_version, check_nulls_in_peers, check_schema_agreement_in_gossip_and_peers, \ - check_group0_tokenring_consistency, CHECK_NODE_HEALTH_RETRIES, CHECK_NODE_HEALTH_RETRY_DELAY -from sdcm.utils.decorators import NoValue, retrying, log_run_info, optional_cached_property +from sdcm.utils.file import File +from sdcm.utils.health_checker import ( + CHECK_NODE_HEALTH_RETRIES, + CHECK_NODE_HEALTH_RETRY_DELAY, + check_group0_tokenring_consistency, + check_node_status_in_gossip_and_nodetool_status, + check_nodes_status, + check_nulls_in_peers, + check_schema_agreement_in_gossip_and_peers, + check_schema_version, +) +from sdcm.utils.install import InstallMode +from sdcm.utils.ldap import ( + DEFAULT_PWD_SUFFIX, + LDAP_BASE_OBJECT, + LDAP_PASSWORD, + LDAP_PORT, + LDAP_SSH_TUNNEL_LOCAL_PORT, + LDAP_USERS, +) +from sdcm.utils.node import build_node_api_command +from sdcm.utils.raft import get_raft_mode +from sdcm.utils.remote_logger import get_system_logging_thread from sdcm.utils.remotewebbrowser import WebDriverContainerMixin -from sdcm.test_config import TestConfig +from sdcm.utils.scylla_args import ScyllaArgParser from sdcm.utils.sstable.sstable_utils import SstableUtils from sdcm.utils.version_utils import ( + SCYLLA_VERSION_RE, + ComparableScyllaVersion, assume_version, get_gemini_version, get_systemd_version, - ComparableScyllaVersion, - SCYLLA_VERSION_RE, -) -from sdcm.utils.node import build_node_api_command -from sdcm.sct_events import Severity -from sdcm.sct_events.base import LogEvent, add_severity_limit_rules, max_severity -from sdcm.sct_events.health import ClusterHealthValidatorEvent -from sdcm.sct_events.system import TestFrameworkEvent, INSTANCE_STATUS_EVENTS_PATTERNS, InfoEvent -from sdcm.sct_events.grafana import set_grafana_url -from sdcm.sct_events.database import SYSTEM_ERROR_EVENTS_PATTERNS, ScyllaHelpErrorEvent, ScyllaYamlUpdateEvent, SYSTEM_ERROR_EVENTS -from sdcm.sct_events.nodetool import NodetoolEvent -from sdcm.sct_events.decorators import raise_event_on_failure -from sdcm.sct_events.filters import EventsSeverityChangerFilter -from sdcm.utils.auto_ssh import AutoSshContainerMixin -from sdcm.monitorstack.ui import AlternatorDashboard -from sdcm.logcollector import GrafanaSnapshot, GrafanaScreenShot, PrometheusSnapshots, upload_archive_to_s3, \ - save_kallsyms_map, collect_diagnostic_data -from sdcm.utils.ldap import LDAP_SSH_TUNNEL_LOCAL_PORT, LDAP_BASE_OBJECT, LDAP_PASSWORD, LDAP_USERS, \ - LDAP_PORT, DEFAULT_PWD_SUFFIX -from sdcm.utils.remote_logger import get_system_logging_thread -from sdcm.utils.scylla_args import ScyllaArgParser -from sdcm.utils.file import File -from sdcm.utils import cdc -from sdcm.utils.raft import get_raft_mode -from sdcm.coredump import CoredumpExportSystemdThread -from sdcm.keystore import KeyStore -from sdcm.paths import ( - SCYLLA_YAML_PATH, - SCYLLA_PROPERTIES_PATH, - SCYLLA_MANAGER_YAML_PATH, - SCYLLA_MANAGER_AGENT_YAML_PATH, - SCYLLA_MANAGER_TLS_CERT_FILE, - SCYLLA_MANAGER_TLS_KEY_FILE, -) -from sdcm.sct_provision.aws.user_data import ScyllaUserDataBuilder -from sdcm.exceptions import ( - KillNemesis, - NodeNotReady, - SstablesNotFound, ) # Test duration (min). Parameter used to keep instances produced by tests that @@ -203,10 +257,10 @@ class NodeCleanedAfterDecommissionAborted(Exception): def prepend_user_prefix(user_prefix: str, base_name: str): - return '%s-%s' % (user_prefix, base_name) + return f'{user_prefix}-{base_name}' -class UserRemoteCredentials(): +class UserRemoteCredentials: def __init__(self, key_file): self.type = 'user' @@ -215,7 +269,7 @@ def __init__(self, key_file): self.key_pair_name = self.name def __str__(self): - return "Key Pair %s -> %s" % (self.name, self.key_file) + return f"Key Pair {self.name} -> {self.key_file}" def write_key_file(self): pass @@ -255,7 +309,7 @@ def __init__(self, name, parent_cluster, ssh_login_info=None, base_logdir=None, self._containers = {} self.is_seed = False - self.remoter: Optional[RemoteCmdRunnerBase] = None + self.remoter: RemoteCmdRunnerBase | None = None self._use_dns_names: bool = parent_cluster.params.get('use_dns_names') if parent_cluster else False self._spot_monitoring_thread = None @@ -269,7 +323,7 @@ def __init__(self, name, parent_cluster, ssh_login_info=None, base_logdir=None, self.last_line_no = 1 self.last_log_position = 0 self._continuous_events_registry = ContinuousEventsRegistry() - self._coredump_thread: Optional[CoredumpExportSystemdThread] = None + self._coredump_thread: CoredumpExportSystemdThread | None = None self._db_log_reader_thread = None self._scylla_manager_journal_thread = None self._decoding_backtraces_thread = None @@ -277,7 +331,7 @@ def __init__(self, name, parent_cluster, ssh_login_info=None, base_logdir=None, self.db_init_finished = False self._short_hostname = None - self._alert_manager: Optional[PrometheusAlertManagerListener] = None + self._alert_manager: PrometheusAlertManagerListener | None = None self.termination_event = threading.Event() self.stop_wait_db_up_event = threading.Event() @@ -392,7 +446,7 @@ def host_id(self): return self.parent_cluster.get_nodetool_info(self, publish_event=False).get('ID') @property - def db_node_instance_type(self) -> Optional[str]: + def db_node_instance_type(self) -> str | None: backend = self.parent_cluster.cluster_backend if backend in ("aws", "aws-siren"): return self.parent_cluster.params.get("instance_type_db") @@ -453,7 +507,7 @@ def refresh_ip_address(self): self._init_port_mapping() @cached_property - def tags(self) -> Dict[str, str]: + def tags(self) -> dict[str, str]: return {**self.parent_cluster.tags, "Name": str(self.name), "UserName": str(self.ssh_login_info.get('user'))} @@ -534,7 +588,7 @@ def is_client_encrypt(self): return 'true' in result.stdout.lower() @property - def cpu_cores(self) -> Optional[int]: + def cpu_cores(self) -> int | None: try: result = self.remoter.run("nproc", ignore_status=True) return int(result.stdout) @@ -691,7 +745,7 @@ def is_gce() -> bool: def scylla_pkg(self): return 'scylla-enterprise' if self.is_enterprise else 'scylla' - def file_exists(self, file_path: str) -> Optional[bool]: + def file_exists(self, file_path: str) -> bool | None: try: return self.remoter.sudo(f"test -e '{file_path}'", ignore_status=True).ok except Exception as details: # pylint: disable=broad-except # noqa: BLE001 @@ -728,7 +782,7 @@ def is_enterprise(self): return "scylla-enterprise" in self.remoter.sudo("apt-cache search scylla-enterprise", ignore_status=True).stdout @property - def public_ip_address(self) -> Optional[str]: + def public_ip_address(self) -> str | None: if self._public_ip_address_cached is None: self._public_ip_address_cached = self._get_public_ip_address() return self._public_ip_address_cached @@ -741,7 +795,7 @@ def public_dns_name(self) -> str: def private_dns_name(self) -> str: return self.name - def _get_public_ip_address(self) -> Optional[str]: + def _get_public_ip_address(self) -> str | None: public_ips, _ = self._refresh_instance_state() if public_ips: return public_ips[0] @@ -749,12 +803,12 @@ def _get_public_ip_address(self) -> Optional[str]: return None @property - def private_ip_address(self) -> Optional[str]: + def private_ip_address(self) -> str | None: if self._private_ip_address_cached is None: self._private_ip_address_cached = self._get_private_ip_address() return self._private_ip_address_cached - def _get_private_ip_address(self) -> Optional[str]: + def _get_private_ip_address(self) -> str | None: _, private_ips = self._refresh_instance_state() if private_ips: return private_ips[0] @@ -762,12 +816,12 @@ def _get_private_ip_address(self) -> Optional[str]: return None @property - def ipv6_ip_address(self) -> Optional[str]: + def ipv6_ip_address(self) -> str | None: if self._ipv6_ip_address_cached is None: self._ipv6_ip_address_cached = self._get_ipv6_ip_address() return self._ipv6_ip_address_cached - def _get_ipv6_ip_address(self) -> Optional[str]: + def _get_ipv6_ip_address(self) -> str | None: raise NotImplementedError() def get_all_ip_addresses(self): @@ -924,7 +978,7 @@ def silence_alert(self, alert_name, duration=None, start=None, end=None): return AlertSilencer(self._alert_manager, alert_name, duration, start, end) def __str__(self): - return 'Node %s [%s | %s%s] (seed: %s)' % ( + return 'Node {} [{} | {}{}] (seed: {})'.format( self.name, self.public_ip_address, self.private_ip_address, @@ -1210,7 +1264,7 @@ def keyspace_available(): return res.exit_status == 0 wait.wait_for(keyspace_available, timeout=600, step=60, - text='Waiting until keyspace {} is available'.format(keyspace), throw_exc=False) + text=f'Waiting until keyspace {keyspace} is available', throw_exc=False) # Don't need NodetoolEvent when waiting for space_node_threshold before start the nemesis, not publish it result = self.run_nodetool(sub_cmd='cfstats', args=keyspace, timeout=300, warning_event_on_exception=(Failure, UnexpectedExit, Libssh2_UnexpectedExit,), publish_event=False) @@ -1315,7 +1369,7 @@ def set_web_listen_address(self): if find_web_param.exit_status == 1: cmd = """sudo sh -c "sed -i 's|ExecStart=/usr/bin/node_exporter --collector.interrupts|""" \ """ExecStart=/usr/bin/node_exporter --collector.interrupts """ \ - """--web.listen-address="[%s]:9100"|g' %s" """ % (self.ip_address, node_exporter_file) + f"""--web.listen-address="[{self.ip_address}]:9100"|g' {node_exporter_file}" """ self.remoter.run(cmd) self.remoter.run('sudo systemctl restart node-exporter.service') @@ -1361,7 +1415,7 @@ def mark_log(self): def follow_system_log( self, - patterns: Optional[List[Union[str, re.Pattern, LogEvent]]] = None, + patterns: list[str | re.Pattern | LogEvent] | None = None, start_from_beginning: bool = False ) -> Iterable[str]: stream = File(self.system_log) @@ -1379,10 +1433,10 @@ def follow_system_log( regexps.append(re.compile(pattern.regex, flags=re.IGNORECASE)) return stream.read_lines_filtered(*regexps) - @contextlib.contextmanager - def open_system_log(self, on_datetime: Optional[datetime] = None) -> IO[AnyStr]: + @contextmanager + def open_system_log(self, on_datetime: datetime | None = None) -> IO[AnyStr]: """Opens system log file and seeks to the given datetime.""" - with open(self.system_log, 'r', encoding="utf-8") as log_file: + with open(self.system_log, encoding="utf-8") as log_file: if not on_datetime: yield log_file return @@ -1470,7 +1524,7 @@ def copy_scylla_debug_info(self, node_name: str, debug_file: str): if not os.path.exists(transit_scylla_debug_file): db_node.remoter.receive_files(debug_file, transit_scylla_debug_file) res = self.remoter.run( - "test -f {}".format(final_scylla_debug_file), ignore_status=True, verbose=False) + f"test -f {final_scylla_debug_file}", ignore_status=True, verbose=False) if res.exited != 0: self.remoter.send_files(transit_scylla_debug_file, # pylint: disable=not-callable final_scylla_debug_file) @@ -1490,9 +1544,9 @@ def decode_raw_backtrace(self, scylla_debug_file, raw_backtrace): :returns: result of bactrace :rtype: {str} """ - return self.remoter.run('addr2line -Cpife {0} {1}'.format(scylla_debug_file, raw_backtrace), verbose=True) + return self.remoter.run(f'addr2line -Cpife {scylla_debug_file} {raw_backtrace}', verbose=True) - def get_scylla_build_id(self) -> Optional[str]: + def get_scylla_build_id(self) -> str | None: for scylla_executable in ("/usr/bin/scylla", "/opt/scylladb/libexec/scylla", ): build_id_result = self.remoter.run(f"{scylla_executable} --build-id", ignore_status=True) if build_id_result.ok: @@ -1518,8 +1572,8 @@ def _remote_properties(self, path): def remote_cassandra_rackdc_properties(self): return self._remote_properties(path=self.add_install_prefix(abs_path=SCYLLA_PROPERTIES_PATH)) - @contextlib.contextmanager - def remote_scylla_yaml(self) -> ContextManager[ScyllaYaml]: + @contextmanager + def remote_scylla_yaml(self) -> AbstractContextManager[ScyllaYaml]: with self._remote_yaml(path=self.add_install_prefix(abs_path=SCYLLA_YAML_PATH)) as scylla_yaml: new_scylla_yaml = ScyllaYaml(**scylla_yaml) old_scylla_yaml = new_scylla_yaml.copy() @@ -1651,21 +1705,21 @@ def download_scylla_repo(self, scylla_repo): return if self.distro.is_rhel_like: repo_path = '/etc/yum.repos.d/scylla.repo' - self.remoter.sudo('curl --retry 5 --retry-max-time 300 -o %s -L %s' % (repo_path, scylla_repo)) + self.remoter.sudo(f'curl --retry 5 --retry-max-time 300 -o {repo_path} -L {scylla_repo}') self.remoter.sudo('chown root:root %s' % repo_path) self.remoter.sudo('chmod 644 %s' % repo_path) result = self.remoter.run('cat %s' % repo_path, verbose=True) verify_scylla_repo_file(result.stdout, is_rhel_like=True) elif self.distro.is_sles: repo_path = '/etc/zypp/repos.d/scylla.repo' - self.remoter.sudo('curl --retry 5 --retry-max-time 300 -o %s -L %s' % (repo_path, scylla_repo)) + self.remoter.sudo(f'curl --retry 5 --retry-max-time 300 -o {repo_path} -L {scylla_repo}') self.remoter.sudo('chown root:root %s' % repo_path) self.remoter.sudo('chmod 644 %s' % repo_path) result = self.remoter.run('cat %s' % repo_path, verbose=True) verify_scylla_repo_file(result.stdout, is_rhel_like=True) else: repo_path = '/etc/apt/sources.list.d/scylla.list' - self.remoter.sudo('curl --retry 5 --retry-max-time 300 -o %s -L %s' % (repo_path, scylla_repo)) + self.remoter.sudo(f'curl --retry 5 --retry-max-time 300 -o {repo_path} -L {scylla_repo}') result = self.remoter.run('cat %s' % repo_path, verbose=True) verify_scylla_repo_file(result.stdout, is_rhel_like=False) self.install_package('gnupg2') @@ -1736,7 +1790,7 @@ def is_apt_lock_free(self) -> bool: result = self.remoter.sudo("lsof /var/lib/dpkg/lock", ignore_status=True) return result.exit_status == 1 - def install_manager_agent(self, package_path: Optional[str] = None) -> None: + def install_manager_agent(self, package_path: str | None = None) -> None: package_name = "scylla-manager-agent" if package_path: package_name = f"{package_path}scylla-manager-agent*" @@ -1768,7 +1822,7 @@ def install_manager_agent(self, package_path: Optional[str] = None) -> None: manager_agent_version = self.remoter.run("scylla-manager-agent --version").stdout self.log.info("node %s has scylla-manager-agent version %s", self.name, manager_agent_version) - def update_manager_agent_config(self, region: Optional[str] = None) -> None: + def update_manager_agent_config(self, region: str | None = None) -> None: backup_backend = self.parent_cluster.params.get("backup_bucket_backend") backup_backend_config = {} if backup_backend == "s3": @@ -1892,12 +1946,12 @@ def install_scylla(self, scylla_repo, scylla_version=None): # noqa: PLR0915 self.install_epel() self.remoter.run("sudo yum remove -y abrt") # https://docs.scylladb.com/operating-scylla/admin/#core-dumps version = f"-{scylla_version}*" if scylla_version else "" - self.remoter.run('sudo yum install -y {}{}'.format(self.scylla_pkg(), version)) + self.remoter.run(f'sudo yum install -y {self.scylla_pkg()}{version}') elif self.distro.is_sles15: self.remoter.sudo('SUSEConnect --product sle-module-legacy/15.3/x86_64') self.remoter.sudo('SUSEConnect --product sle-module-python2/15.3/x86_64') version = f"-{scylla_version}" if scylla_version else "" - self.remoter.sudo('zypper install -y {}{}'.format(self.scylla_pkg(), version)) + self.remoter.sudo(f'zypper install -y {self.scylla_pkg()}{version}') else: self.install_package(package_name="software-properties-common") if self.distro.is_debian: @@ -1985,7 +2039,7 @@ def offline_install_scylla(self, unified_package, nonroot): # noqa: PLR0912 """) self.remoter.run('sudo bash -cxe "%s"' % install_cmds) - def web_install_scylla(self, scylla_version: Optional[str] = None) -> None: + def web_install_scylla(self, scylla_version: str | None = None) -> None: """ Install scylla by Scylla Web Installer Script. Try to use install the latest unstable build for the test branch, generally it should install same version @@ -2022,7 +2076,7 @@ def is_scylla_installed(self, raise_if_not_installed=False): raise Exception(f"There is no pre-installed ScyllaDB on {self}") return False - def get_scylla_binary_version(self) -> Optional[str]: + def get_scylla_binary_version(self) -> str | None: result = self.remoter.run(f"{self.add_install_prefix('/usr/bin/scylla')} --version", ignore_status=True) if result.ok: return result.stdout.strip() @@ -2030,7 +2084,7 @@ def get_scylla_binary_version(self) -> Optional[str]: result.command, result.stdout, result.stderr) return None - def get_scylla_package_version(self) -> Optional[str]: + def get_scylla_package_version(self) -> str | None: if self.distro.is_rhel_like: cmd = f"rpm --query --queryformat '%{{VERSION}}' {self.scylla_pkg()}" else: @@ -2043,7 +2097,7 @@ def get_scylla_package_version(self) -> Optional[str]: return None @optional_cached_property - def scylla_version_detailed(self) -> Optional[str]: + def scylla_version_detailed(self) -> str | None: scylla_version = self.get_scylla_binary_version() if scylla_version is None: if scylla_version := self.get_scylla_package_version(): @@ -2057,7 +2111,7 @@ def scylla_version_detailed(self) -> Optional[str]: return scylla_version @optional_cached_property - def scylla_build_mode(self) -> Optional[str]: + def scylla_build_mode(self) -> str | None: result = self.remoter.run(f"{self.add_install_prefix('/usr/bin/scylla')} --build-mode", ignore_status=True) if result.ok: return result.stdout.strip() @@ -2066,7 +2120,7 @@ def scylla_build_mode(self) -> Optional[str]: return None @optional_cached_property - def scylla_version(self) -> Optional[str]: + def scylla_version(self) -> str | None: if scylla_version := self.scylla_version_detailed: if match := SCYLLA_VERSION_RE.match(scylla_version): scylla_version = match.group() @@ -2088,7 +2142,7 @@ def detect_disks(self, nvme=True): """ patt = (r'nvme*n*', r'nvme\d+n\d+') if nvme else (r'sd[b-z]', r'sd\w+') result = self.remoter.run(f"ls /dev/{patt[0]}", ignore_status=True) - disks = re.findall(r'/dev/{}'.format(patt[1]), result.stdout) + disks = re.findall(rf'/dev/{patt[1]}', result.stdout) # filter out the used disk, the free disk doesn't have partition. disks = [i for i in disks if disks.count(i) == 1] assert disks, 'Failed to find disks!' @@ -2103,7 +2157,7 @@ def kernel_version(self): self._kernel_version = "unknown" else: self._kernel_version = res.stdout.strip() - self.log.info("Found kernel version: {}".format(self._kernel_version)) + self.log.info(f"Found kernel version: {self._kernel_version}") return self._kernel_version def increase_jmx_heap_memory(self, jmx_memory): @@ -2165,7 +2219,7 @@ def upgrade_mgmt(self, scylla_mgmt_address, start_manager_after_upgrade=True): time.sleep(5) # pylint: disable=too-many-branches - def install_mgmt(self, package_url: Optional[str] = None) -> None: # noqa: PLR0912 + def install_mgmt(self, package_url: str | None = None) -> None: # noqa: PLR0912 self.log.debug("Install scylla-manager") if self.is_docker(): @@ -2411,7 +2465,7 @@ def prepare_files_for_archive(self, fileslist): old_full_path = os.path.dirname(f)[1:] new_full_path = os.path.join(root_dir, old_full_path) self.remoter.run('mkdir -p %s' % new_full_path, ignore_status=True) - self.remoter.run('cp -r %s %s' % (f, new_full_path), ignore_status=True) + self.remoter.run(f'cp -r {f} {new_full_path}', ignore_status=True) return root_dir def generate_coredump_file(self, restart_scylla=True, node_name: str = None, message: str = None): @@ -2513,7 +2567,7 @@ def run_nodetool(self, sub_cmd, args="", options="", timeout=None, # noqa: PLR0 try: result = \ self.remoter.run(cmd, timeout=timeout, ignore_status=ignore_status, verbose=verbose, retry=retry) - self.log.debug("Command '%s' duration -> %s s" % (result.command, result.duration)) + self.log.debug(f"Command '{result.command}' duration -> {result.duration} s") nodetool_event.duration = result.duration return result @@ -2704,7 +2758,7 @@ def _gen_cqlsh_cmd(self, command, keyspace, timeout, host, port, connect_timeout """cqlsh [options] [host [port]]""" credentials = self.parent_cluster.get_db_auth() auth_params = "-u {} -p '{}'".format(*credentials) if credentials else '' - use_keyspace = "--keyspace {}".format(keyspace) if keyspace else "" + use_keyspace = f"--keyspace {keyspace}" if keyspace else "" ssl_params = '--ssl' if self.parent_cluster.params.get("client_encrypt") else '' options = "--no-color {auth_params} {use_keyspace} --request-timeout={timeout} " \ "--connect-timeout={connect_timeout} {ssl_params}".format( @@ -2757,10 +2811,10 @@ def run_startup_script(self): tmp_file.flush() self.remoter.send_files(src=tmp_file.name, dst=startup_script_remote_path) # pylint: disable=not-callable - cmds = dedent(""" - chmod +x {0} - {0} - """.format(startup_script_remote_path)) + cmds = dedent(f""" + chmod +x {startup_script_remote_path} + {startup_script_remote_path} + """) result = self.remoter.run("sudo bash -ce '%s'" % cmds) LOGGER.debug(result.stdout) @@ -2799,7 +2853,7 @@ def configure_remote_logging(self): self.remoter.sudo(shell_script_cmd(script, quote="'")) @property - def scylla_packages_installed(self) -> List[str]: + def scylla_packages_installed(self) -> list[str]: if self.distro.is_rhel_like or self.distro.is_sles: cmd = "rpm -qa 'scylla*'" else: @@ -2858,7 +2912,7 @@ def wait_for_machine_image_configured(self): self.log.info("Waiting for Scylla Machine Image setup to finish...") wait.wait_for(self.is_machine_image_configured, step=10, timeout=300, throw_exc=False) - def get_sysctl_properties(self) -> Dict[str, str]: + def get_sysctl_properties(self) -> dict[str, str]: sysctl_properties = {} result = self.remoter.sudo("sysctl -a", ignore_status=True) @@ -2899,7 +2953,7 @@ def disable_daily_triggered_services(self): self.remoter.sudo('apt-get remove -y unattended-upgrades', ignore_status=True) self.remoter.sudo('apt-get remove -y update-manager', ignore_status=True) - def get_nic_devices(self) -> List: + def get_nic_devices(self) -> list: """Returns list of ethernet network interfaces""" result = self.remoter.run('/sbin/ip -o link show |grep ether |awk -F": " \'{print $2}\'', verbose=True) return result.stdout.strip().split() @@ -2963,7 +3017,7 @@ def get_perftune_yaml(self) -> str: else: return cmd.stderr - def get_list_of_sstables(self, keyspace_name: str, table_name: str, suffix: str = "-Statistics.db") -> List[str]: + def get_list_of_sstables(self, keyspace_name: str, table_name: str, suffix: str = "-Statistics.db") -> list[str]: statistics_files = [] find_cmd = f"find /var/lib/scylla/data/{keyspace_name}/{table_name}-* -name *{suffix}" @@ -3087,8 +3141,8 @@ def __init__(self, cluster_uuid=None, cluster_prefix='cluster', node_prefix='nod self.uuid = cluster_uuid self.node_type = node_type self.shortid = str(self.uuid)[:8] - self.name = '%s-%s' % (cluster_prefix, self.shortid) - self.node_prefix = '%s-%s' % (node_prefix, self.shortid) + self.name = f'{cluster_prefix}-{self.shortid}' + self.node_prefix = f'{node_prefix}-{self.shortid}' self._node_index = 0 # I wanted to avoid some parameter passing # from the tester class to the cluster test. @@ -3113,7 +3167,7 @@ def __init__(self, cluster_uuid=None, cluster_prefix='cluster', node_prefix='nod # get_node_ips_param should be defined in child self._node_public_ips = self.params.get(self.get_node_ips_param(public_ip=True)) or [] self._node_private_ips = self.params.get(self.get_node_ips_param(public_ip=False)) or [] - self.log.debug('Node public IPs: {}, private IPs: {}'.format(self._node_public_ips, self._node_private_ips)) + self.log.debug(f'Node public IPs: {self._node_public_ips}, private IPs: {self._node_private_ips}') # NOTE: following is needed in case of K8S where we init multiple DB clusters first # and only then we add nodes to it calling code in parallel. @@ -3139,7 +3193,7 @@ def __init__(self, cluster_uuid=None, cluster_prefix='cluster', node_prefix='nod rack = None if self.params.get('simulated_racks') else az_index self.add_nodes(nodes_per_az[az_index], rack=rack, enable_auto_bootstrap=self.auto_bootstrap) else: - raise ValueError('Unsupported type: {}'.format(type(n_nodes))) + raise ValueError(f'Unsupported type: {type(n_nodes)}') self.run_node_benchmarks() self.coredumps = {} super().__init__() @@ -3156,7 +3210,7 @@ def auto_bootstrap(self): return not self.params.get('use_legacy_cluster_init') @cached_property - def tags(self) -> Dict[str, str]: + def tags(self) -> dict[str, str]: key = self.node_type if "db" not in self.node_type else "db" action = self.params.get(f"post_behavior_{key}_nodes") return {**self.test_config.common_tags(), @@ -3226,7 +3280,7 @@ def get_backtraces(self): if node.n_coredumps > 0: self.coredumps[node.name] = node.n_coredumps except Exception as ex: # pylint: disable=broad-except - self.log.exception("Unable to get coredump status from node {node}: {ex}".format(node=node, ex=ex)) + self.log.exception(f"Unable to get coredump status from node {node}: {ex}") def node_setup(self, node, verbose=False, timeout=3600): raise NotImplementedError("Derived class must implement 'node_setup' method!") @@ -3462,7 +3516,7 @@ def cql_connection_patient_exclusive(self, node, keyspace=None, # noqa: PLR0913 def get_non_system_ks_cf_list(self, db_node, # pylint: disable=too-many-arguments # noqa: PLR0913 filter_out_table_with_counter=False, filter_out_mv=False, filter_empty_tables=True, - filter_by_keyspace: list = None) -> List[str]: + filter_by_keyspace: list = None) -> list[str]: return self.get_any_ks_cf_list(db_node, filter_out_table_with_counter=filter_out_table_with_counter, filter_out_mv=filter_out_mv, filter_empty_tables=filter_empty_tables, filter_out_system=True, filter_out_cdc_log_tables=True, filter_by_keyspace=filter_by_keyspace) @@ -3470,7 +3524,7 @@ def get_non_system_ks_cf_list(self, db_node, # pylint: disable=too-many-argumen def get_any_ks_cf_list(self, db_node, # pylint: disable=too-many-arguments # noqa: PLR0913 filter_out_table_with_counter=False, filter_out_mv=False, filter_empty_tables=True, filter_out_system=False, filter_out_cdc_log_tables=False, - filter_by_keyspace: list = None) -> List[str]: + filter_by_keyspace: list = None) -> list[str]: regular_column_names = ["keyspace_name", "table_name"] materialized_view_column_names = ["keyspace_name", "view_name"] regular_table_names, materialized_view_table_names = set(), set() @@ -3538,7 +3592,7 @@ def execute_cmd(cql_session, entity_type): return list(regular_table_names - materialized_view_table_names) - def is_table_has_data(self, session, table_name: str) -> (bool, Optional[Exception]): + def is_table_has_data(self, session, table_name: str) -> (bool, Exception | None): """ Return True if `table_name` has data in it. Checking if a result page with any data is received. @@ -3551,7 +3605,7 @@ def is_table_has_data(self, session, table_name: str) -> (bool, Optional[Excepti self.log.warning(f'Failed to get rows from {table_name} table. Error: {exc}') return False, exc - def get_all_tables_with_cdc(self, db_node: BaseNode) -> List[str]: + def get_all_tables_with_cdc(self, db_node: BaseNode) -> list[str]: """Return list of all tables with enabled cdc feature Get all non system keyspaces and tables and return @@ -3571,7 +3625,7 @@ def get_all_tables_with_cdc(self, db_node: BaseNode) -> List[str]: return ks_tables_with_cdc - def get_all_tables_with_twcs(self, db_node: BaseNode) -> List[Dict[str, Union[Dict[str, Any], int, str]]]: + def get_all_tables_with_twcs(self, db_node: BaseNode) -> list[dict[str, dict[str, Any] | int | str]]: """ Get list of keyspace.table with TWCS example of returning dict: { "name": ks.test, @@ -3766,7 +3820,7 @@ class ClusterNodesNotReady(Exception): class BaseScyllaCluster: # pylint: disable=too-many-public-methods, too-many-instance-attributes, too-many-statements node_setup_requires_scylla_restart = True name: str - nodes: List[BaseNode] + nodes: list[BaseNode] log: logging.Logger def __init__(self, *args, **kwargs): @@ -3812,7 +3866,7 @@ def connection_bundle_file(self) -> Path | None: return Path(bundle_file) if bundle_file else None @property - def racks(self) -> Set[int]: + def racks(self) -> set[int]: return {node.rack for node in self.nodes} def set_seeds(self, wait_for_timeout=300, first_only=False): @@ -3871,12 +3925,10 @@ def validate_seeds_on_all_nodes(self): yaml_seeds_ips = node.extract_seeds_from_scylla_yaml() for ip in yaml_seeds_ips: assert ip in self.seed_nodes_addresses, \ - 'Wrong seed IP {act_ip} in the scylla.yaml on the {node_name} node. ' \ - 'Expected {exp_ips}'.format(node_name=node.name, - exp_ips=self.seed_nodes_addresses, - act_ip=ip) + f'Wrong seed IP {ip} in the scylla.yaml on the {node.name} node. ' \ + f'Expected {self.seed_nodes_addresses}' - @contextlib.contextmanager + @contextmanager def patch_params(self) -> SCTConfiguration: new_params, old_params = self.params.copy(), self.params yield new_params @@ -4247,11 +4299,9 @@ def cfstat_reached_threshold(self, key, threshold, keyspaces=None): # Calculate space on the disk of all test keyspaces on the one node. # It's decided to check the threshold on one node only for keyspace_name in keyspaces: - self.log.debug("Get cfstats on the node %s for %s keyspace" % - (node.name, keyspace_name)) + self.log.debug(f"Get cfstats on the node {node.name} for {keyspace_name} keyspace") node_space += node.get_cfstats(keyspace_name)[key] - self.log.debug("Current cfstats on the node %s for %s keyspaces: %s" % - (node.name, keyspaces, node_space)) + self.log.debug(f"Current cfstats on the node {node.name} for {keyspaces} keyspaces: {node_space}") reached_threshold = True if node_space < threshold: reached_threshold = False @@ -4267,7 +4317,7 @@ def wait_total_space_used_per_node(self, size=None, keyspace='keyspace1'): keyspace = [keyspace] key = 'Space used (total)' wait.wait_for(func=self.cfstat_reached_threshold, step=10, timeout=600, - text="Waiting until cfstat '%s' reaches value '%s'" % (key, size), + text=f"Waiting until cfstat '{key}' reaches value '{size}'", key=key, threshold=size, keyspaces=keyspace, throw_exc=False) def add_nemesis(self, nemesis, tester_obj): @@ -4564,7 +4614,7 @@ def verify_logging_from_nodes(nodes_list): absent.append(node) if absent: raise Exception( - "No db log from the following nodes:\n%s\nfiles expected to be find:\n%s" % ( + "No db log from the following nodes:\n{}\nfiles expected to be find:\n{}".format( '\n'.join(map(lambda node: node.name, absent)), '\n'.join(map(lambda node: node.system_log, absent)) )) @@ -4689,9 +4739,8 @@ def get_node_ip_list(verification_node): if target_node_ip in node_ip_list and not missing_host_ids and not decommission_done: # Decommission was interrupted during streaming data. cluster_status = self.get_nodetool_status(verification_node) - error_msg = ('Node that was decommissioned %s still in the cluster. ' - 'Cluster status info: %s' % (node, - cluster_status)) + error_msg = (f'Node that was decommissioned {node} still in the cluster. ' + f'Cluster status info: {cluster_status}') LOGGER.error('Decommission %s FAIL', node) LOGGER.error(error_msg) @@ -4741,11 +4790,11 @@ def get_cluster_manager(self, create_cluster_if_not_exists: bool = True) -> AnyM if not self.params.get('use_mgmt'): raise ScyllaManagerError('Scylla-manager configuration is not defined!') manager_tool = mgmt.get_scylla_manager_tool(manager_node=self.scylla_manager_node, scylla_cluster=self) - LOGGER.debug("sctool version is : {}".format(manager_tool.sctool.version)) + LOGGER.debug(f"sctool version is : {manager_tool.sctool.version}") cluster_name = self.scylla_manager_cluster_name # pylint: disable=no-member mgr_cluster = manager_tool.get_cluster(cluster_name) if not mgr_cluster and create_cluster_if_not_exists: - self.log.debug("Could not find cluster : {} on Manager. Adding it to Manager".format(cluster_name)) + self.log.debug(f"Could not find cluster : {cluster_name} on Manager. Adding it to Manager") return self.create_cluster_manager(cluster_name, manager_tool=manager_tool) return mgr_cluster @@ -4791,7 +4840,7 @@ def get_db_nodes_cpu_mode(self): return results -class BaseLoaderSet(): +class BaseLoaderSet: def __init__(self, params): self._loader_cycle = None @@ -5175,10 +5224,10 @@ def configure_ngrok(self): "bind_tls": False } res = requests.post('http://localhost:4040/api/tunnels', json=tunnel) - assert res.ok, "failed to add a ngrok tunnel [{}, {}]".format(res, res.text) + assert res.ok, f"failed to add a ngrok tunnel [{res}, {res.text}]" ngrok_address = res.json()['public_url'].replace('http://', '') - return "{}:80".format(ngrok_address) + return f"{ngrok_address}:80" def set_local_sct_ip(self): @@ -5233,7 +5282,7 @@ def install_scylla_monitoring_prereqs(node): # pylint: disable=invalid-name cat /etc/debian_version """) else: - raise ValueError('Unsupported Distribution type: {}'.format(str(node.distro))) + raise ValueError(f'Unsupported Distribution type: {str(node.distro)}') node_exporter_setup = NodeExporterSetup() node_exporter_setup.install(node) @@ -5361,7 +5410,7 @@ def configure_alert_manager(self, node): alert_cont_tmp_file.write(conf) alert_cont_tmp_file.flush() node.remoter.send_files(src=alert_cont_tmp_file.name, dst=alert_cont_tmp_file.name) - node.remoter.run("bash -ce 'cat %s >> %s'" % (alert_cont_tmp_file.name, alertmanager_conf_file)) + node.remoter.run(f"bash -ce 'cat {alert_cont_tmp_file.name} >> {alertmanager_conf_file}'") @retrying(n=5, sleep_time=10, allowed_exceptions=(Failure, UnexpectedExit), message="Waiting for restarting scylla monitoring") @@ -5443,16 +5492,15 @@ def start_scylla_monitoring(self, node): def save_monitoring_version(self, node): node.remoter.run( - 'echo "{0.monitor_branch}:{0.monitoring_version}" > \ - {0.monitor_install_path}/monitor_version'.format(self), ignore_status=True) + f'echo "{self.monitor_branch}:{self.monitoring_version}" > \ + {self.monitor_install_path}/monitor_version', ignore_status=True) def add_sct_dashboards_to_grafana(self, node): def _register_grafana_json(json_filename): - url = "'http://{0}:{1.grafana_port}/api/dashboards/db'".format(normalize_ipv6_url(node.external_address), - self) - result = LOCALRUNNER.run('curl -g -XPOST -i %s --data-binary @%s -H "Content-Type: application/json"' % - (url, json_filename)) + url = f"'http://{normalize_ipv6_url(node.external_address)}:{self.grafana_port}/api/dashboards/db'" + result = LOCALRUNNER.run( + f'curl -g -XPOST -i {url} --data-binary @{json_filename} -H "Content-Type: application/json"') return result.exited == 0 wait.wait_for(_register_grafana_json, step=10, @@ -5462,7 +5510,7 @@ def _register_grafana_json(json_filename): def save_sct_dashboards_config(self, node): sct_monitoring_addons_dir = os.path.join(self.monitor_install_path, 'sct_monitoring_addons') - node.remoter.run('mkdir -p {}'.format(sct_monitoring_addons_dir), ignore_status=True) + node.remoter.run(f'mkdir -p {sct_monitoring_addons_dir}', ignore_status=True) node.remoter.send_files(src=self.sct_dashboard_json_file, dst=sct_monitoring_addons_dir) @log_run_info @@ -5495,7 +5543,7 @@ def stop_scylla_monitoring(self, node): """) node.remoter.run("bash -ce '%s'" % kill_script) - def get_grafana_screenshot_and_snapshot(self, test_start_time: Optional[int] = None) -> dict[str, list[str]]: + def get_grafana_screenshot_and_snapshot(self, test_start_time: int | None = None) -> dict[str, list[str]]: """ Take screenshot of the Grafana per-server-metrics dashboard and upload to S3 """ @@ -5523,7 +5571,7 @@ def get_grafana_screenshots(self, node: BaseNode, test_start_time: float) -> lis extra_entities=grafana_extra_dashboards) screenshot_files = screenshot_collector.collect(node, self.logdir) for screenshot in screenshot_files: - s3_path = "{test_id}/{date}".format(test_id=self.test_config.test_id(), date=date_time) + s3_path = f"{self.test_config.test_id()}/{date_time}" screenshot_links.append(S3Storage().upload_file(screenshot, s3_path)) return screenshot_links @@ -5550,8 +5598,7 @@ def upload_annotations_to_s3(self): annotations = self.get_grafana_annotations(self.nodes[0]) if annotations: annotations_url = S3Storage().generate_url('annotations.json', self.monitor_id) - self.log.info("Uploading 'annotations.json' to {s3_url}".format( - s3_url=annotations_url)) + self.log.info(f"Uploading 'annotations.json' to {annotations_url}") response = requests.put(annotations_url, data=annotations, headers={ 'Content-type': 'application/json; charset=utf-8'}) response.raise_for_status() @@ -5573,7 +5620,7 @@ def download_monitor_data(self) -> str: return "" -class NoMonitorSet(): +class NoMonitorSet: def __init__(self): diff --git a/sdcm/cluster_aws.py b/sdcm/cluster_aws.py index 9c42506f916..98a614cfcf1 100644 --- a/sdcm/cluster_aws.py +++ b/sdcm/cluster_aws.py @@ -26,7 +26,7 @@ from datetime import datetime from functools import cached_property from textwrap import dedent -from typing import Dict, Optional, ParamSpec, TypeVar +from typing import ParamSpec, TypeVar import boto3 import botocore.exceptions @@ -35,18 +35,17 @@ from mypy_boto3_ec2 import EC2Client from mypy_boto3_ec2.service_resource import EC2ServiceResource -from sdcm import ec2_client, cluster, wait +from sdcm import cluster, ec2_client, wait from sdcm.ec2_client import CreateSpotInstancesError from sdcm.provision.aws.utils import configure_set_preserve_hostname_script from sdcm.provision.common.utils import configure_hosts_set_hostname_script from sdcm.provision.scylla_yaml import SeedProvider -from sdcm.sct_provision.aws.cluster import PlacementGroup - -from sdcm.remote import LocalCmdRunner, shell_script_cmd, NETWORK_EXCEPTIONS +from sdcm.remote import NETWORK_EXCEPTIONS, LocalCmdRunner, shell_script_cmd from sdcm.sct_events.database import DatabaseLogEvent from sdcm.sct_events.filters import DbEventsFilter from sdcm.sct_events.system import SpotTerminationEvent -from sdcm.utils.aws_utils import tags_as_ec2_tags, ec2_instance_wait_public_ip +from sdcm.sct_provision.aws.cluster import PlacementGroup +from sdcm.utils.aws_utils import ec2_instance_wait_public_ip, tags_as_ec2_tags from sdcm.utils.common import list_instances_aws from sdcm.utils.decorators import retrying from sdcm.wait import exponential_retry @@ -116,9 +115,7 @@ def __init__(self, ec2_ami_id, ec2_subnet_id, ec2_security_group_ids, # pylint: ) def __str__(self): - return 'Cluster %s (AMI: %s Type: %s)' % (self.name, - self._ec2_ami_id, - self._ec2_instance_type) + return f'Cluster {self.name} (AMI: {self._ec2_ami_id} Type: {self._ec2_instance_type})' @property def instance_profile_name(self) -> str | None: @@ -276,7 +273,7 @@ def check_spot_error(self, cl_ex, instance_provision): def _create_mixed_instances(self, count, interfaces, ec2_user_data, dc_idx, instance_type=None): # pylint: disable=too-many-arguments # noqa: PLR0913 instances = [] max_num_on_demand = 2 - if isinstance(self, (ScyllaAWSCluster, CassandraAWSCluster)): + if isinstance(self, ScyllaAWSCluster | CassandraAWSCluster): if count > 2: # noqa: PLR2004 count_on_demand = max_num_on_demand elif count == 2: # noqa: PLR2004 @@ -429,7 +426,7 @@ def __init__(self, ec2_instance, ec2_service, credentials, parent_cluster, # py dc_idx=dc_idx, rack=rack) def init(self): - LOGGER.debug("Waiting until instance {0._instance} starts running...".format(self)) + LOGGER.debug(f"Waiting until instance {self._instance} starts running...") self._instance_wait_safe(self._instance.wait_until_running) if not self.test_config.REUSE_CLUSTER: @@ -451,7 +448,7 @@ def short_hostname(self): return self.name @cached_property - def tags(self) -> Dict[str, str]: + def tags(self) -> dict[str, str]: return {**super().tags, "NodeIndex": str(self.node_index), } @@ -541,15 +538,15 @@ def private_dns_name(self) -> str: 'curl http://169.254.169.254/latest/meta-data/local-hostname', verbose=False) return result.stdout.strip() - def _get_public_ip_address(self) -> Optional[str]: + def _get_public_ip_address(self) -> str | None: return self._instance.public_ip_address - def _get_private_ip_address(self) -> Optional[str]: + def _get_private_ip_address(self) -> str | None: if self._eth1_private_ip_address: return self._eth1_private_ip_address return self._instance.private_ip_address - def _get_ipv6_ip_address(self) -> Optional[str]: + def _get_ipv6_ip_address(self) -> str | None: return self._instance.network_interfaces[0].ipv6_addresses[0]["Ipv6Address"] def _refresh_instance_state(self): @@ -686,7 +683,7 @@ def restart(self): self.remoter.sudo(create_devices_file) break else: - raise IOError("scylla_create_devices file isn't found") + raise OSError("scylla_create_devices file isn't found") self.start_scylla_server(verify_up=False) @@ -751,7 +748,7 @@ def traffic_control(self, tcconfig_params=None): tc_command = LOCAL_CMD_RUNNER.run("tcdel eth1 --tc-command", ignore_status=True).stdout self.remoter.run('sudo bash -cxe "%s"' % tc_command, ignore_status=True) else: - tc_command = LOCAL_CMD_RUNNER.run("tcset eth1 {} --tc-command".format(tcconfig_params)).stdout + tc_command = LOCAL_CMD_RUNNER.run(f"tcset eth1 {tcconfig_params} --tc-command").stdout self.remoter.run('sudo bash -cxe "%s"' % tc_command) def install_traffic_control(self): @@ -810,11 +807,11 @@ def add_nodes(self, count, ec2_user_data='', dc_idx=0, rack=0, enable_auto_boots if not ec2_user_data: if self._ec2_user_data and isinstance(self._ec2_user_data, str): ec2_user_data = re.sub(r'(--totalnodes\s)(\d*)(\s)', - r'\g<1>{}\g<3>'.format(len(self.nodes) + count), self._ec2_user_data) + rf'\g<1>{len(self.nodes) + count}\g<3>', self._ec2_user_data) elif self._ec2_user_data and isinstance(self._ec2_user_data, dict): ec2_user_data = self._ec2_user_data else: - ec2_user_data = ('--clustername %s --totalnodes %s ' % (self.name, count)) + ec2_user_data = (f'--clustername {self.name} --totalnodes {count} ') if self.nodes and isinstance(ec2_user_data, str): node_ips = [node.ip_address for node in self.nodes if node.is_seed] seeds = ",".join(node_ips) @@ -871,10 +868,10 @@ def __init__(self, ec2_ami_id, ec2_subnet_id, ec2_security_group_ids, # pylint: node_prefix = cluster.prepend_user_prefix(user_prefix, 'cs-db-node') node_type = 'cs-db' shortid = str(cluster_uuid)[:8] - name = '%s-%s' % (cluster_prefix, shortid) - user_data = ('--clustername %s ' - '--totalnodes %s --version community ' - '--release 2.1.15' % (name, sum(n_nodes))) + name = f'{cluster_prefix}-{shortid}' + user_data = (f'--clustername {name} ' + f'--totalnodes {sum(n_nodes)} --version community ' + '--release 2.1.15') # pylint: disable=unexpected-keyword-arg super().__init__( ec2_ami_id=ec2_ami_id, @@ -910,12 +907,10 @@ def add_nodes(self, count, ec2_user_data='', dc_idx=0, rack=0, enable_auto_boots if not ec2_user_data: if self.nodes: seeds = ",".join(self.get_seed_nodes()) - ec2_user_data = ('--clustername %s ' - '--totalnodes %s --seeds %s ' + ec2_user_data = (f'--clustername {self.name} ' + f'--totalnodes {count} --seeds {seeds} ' '--version community ' - '--release 2.1.15' % (self.name, - count, - seeds)) + '--release 2.1.15') ec2_user_data = self.update_bootstrap(ec2_user_data, enable_auto_bootstrap) added_nodes = super().add_nodes( count=count, @@ -955,8 +950,7 @@ def __init__(self, ec2_ami_id, ec2_subnet_id, ec2_security_group_ids, # pylint: node_prefix = cluster.prepend_user_prefix(user_prefix, 'loader-node') node_type = 'loader' cluster_prefix = cluster.prepend_user_prefix(user_prefix, 'loader-set') - user_data = ('--clustername %s --totalnodes %s --bootstrap false --stop-services' % - (cluster_prefix, n_nodes)) + user_data = (f'--clustername {cluster_prefix} --totalnodes {n_nodes} --bootstrap false --stop-services') cluster.BaseLoaderSet.__init__(self, params=params) diff --git a/sdcm/cluster_azure.py b/sdcm/cluster_azure.py index ed9c3a831c4..84829a0deee 100644 --- a/sdcm/cluster_azure.py +++ b/sdcm/cluster_azure.py @@ -13,7 +13,6 @@ import json import logging from functools import cached_property -from typing import Dict, List from sdcm import cluster from sdcm.provision.azure.provisioner import AzureProvisioner @@ -68,7 +67,7 @@ def init(self) -> None: self.remoter.sudo("systemctl daemon-reload", ignore_status=True) @cached_property - def tags(self) -> Dict[str, str]: + def tags(self) -> dict[str, str]: return {**super().tags, "NodeIndex": str(self.node_index), } @@ -163,11 +162,11 @@ def configure_remote_logging(self) -> None: class AzureCluster(cluster.BaseCluster): # pylint: disable=too-many-instance-attributes def __init__(self, image_id, root_disk_size, # pylint: disable=too-many-arguments, too-many-locals # noqa: PLR0913 - provisioners: List[AzureProvisioner], credentials, + provisioners: list[AzureProvisioner], credentials, cluster_uuid=None, instance_type='Standard_L8s_v3', region_names=None, user_name='root', cluster_prefix='cluster', node_prefix='node', n_nodes=3, params=None, node_type=None): - self.provisioners: List[AzureProvisioner] = provisioners + self.provisioners: list[AzureProvisioner] = provisioners self._image_id = image_id self._root_disk_size = root_disk_size self._credentials = credentials @@ -217,7 +216,7 @@ def _create_node(self, instance, node_index, dc_idx): except Exception as ex: # noqa: BLE001 raise CreateAzureNodeError('Failed to create node: %s' % ex) from ex - def _create_instances(self, count, dc_idx=0, instance_type=None) -> List[VmInstance]: + def _create_instances(self, count, dc_idx=0, instance_type=None) -> list[VmInstance]: region = self._definition_builder.regions[dc_idx] assert region, "no region provided, please add `azure_region_name` param" pricing_model = PricingModel.SPOT if 'spot' in self.instance_provision else PricingModel.ON_DEMAND @@ -246,7 +245,7 @@ def wait_for_init(self): class ScyllaAzureCluster(cluster.BaseScyllaCluster, AzureCluster): def __init__(self, image_id, root_disk_size, # pylint: disable=too-many-arguments # noqa: PLR0913 - provisioners: List[AzureProvisioner], credentials, + provisioners: list[AzureProvisioner], credentials, instance_type='Standard_L8s_v3', user_name='ubuntu', user_prefix=None, n_nodes=3, params=None, region_names=None): diff --git a/sdcm/cluster_baremetal.py b/sdcm/cluster_baremetal.py index 1a4d15dac6a..556d1b4c0bc 100644 --- a/sdcm/cluster_baremetal.py +++ b/sdcm/cluster_baremetal.py @@ -1,5 +1,5 @@ import logging -from typing import Optional, TypedDict +from typing import TypedDict from invoke import UnexpectedExit @@ -54,7 +54,7 @@ def init(self): self.set_hostname() self.run_startup_script() - def _get_public_ip_address(self) -> Optional[str]: + def _get_public_ip_address(self) -> str | None: return self._public_ip @property @@ -72,7 +72,7 @@ def scylla_setup(self, disks, devname: str): return raise exc - def _get_private_ip_address(self) -> Optional[str]: + def _get_private_ip_address(self) -> str | None: return self._private_ip def set_hostname(self): @@ -125,7 +125,7 @@ def _create_node(self, name, public_ip, private_ip): def add_nodes(self, count, ec2_user_data='', dc_idx=0, rack=0, enable_auto_bootstrap=False, instance_type=None): # noqa: PLR0913 assert instance_type is None, "baremetal can provision diffrent types" for node_index in range(count): - node_name = '%s-%s' % (self.node_prefix, node_index) + node_name = f'{self.node_prefix}-{node_index}' self.nodes.append(self._create_node(node_name, self._node_public_ips[node_index], self._node_private_ips[node_index])) diff --git a/sdcm/cluster_docker.py b/sdcm/cluster_docker.py index ff23f7fb05f..a27f4249ac3 100644 --- a/sdcm/cluster_docker.py +++ b/sdcm/cluster_docker.py @@ -13,15 +13,19 @@ # pylint: disable=too-many-arguments; looks like we need to increase DESIGN.max_args to 10 in our pylintrc # pylint: disable=invalid-overridden-method; pylint doesn't know that cached_property is property +import logging import os import re -import logging -from typing import Optional, Union, Dict from functools import cached_property from sdcm import cluster from sdcm.remote import LOCALRUNNER -from sdcm.utils.docker_utils import get_docker_bridge_gateway, Container, ContainerManager, DockerException +from sdcm.utils.docker_utils import ( + Container, + ContainerManager, + DockerException, + get_docker_bridge_gateway, +) from sdcm.utils.health_checker import check_nodes_status from sdcm.utils.net import get_my_public_ip @@ -63,10 +67,10 @@ def node_container_run_args(self, seed_ip): class DockerNode(cluster.BaseNode, NodeContainerMixin): # pylint: disable=abstract-method def __init__(self, # pylint: disable=too-many-arguments # noqa: PLR0913 parent_cluster: "DockerCluster", - container: Optional[Container] = None, + container: Container | None = None, node_prefix: str = "node", - base_logdir: Optional[str] = None, - ssh_login_info: Optional[dict] = None, + base_logdir: str | None = None, + ssh_login_info: dict | None = None, node_index: int = 1) -> None: super().__init__(name=f"{node_prefix}-{node_index}", parent_cluster=parent_cluster, @@ -83,14 +87,14 @@ def is_docker(self): return True @cached_property - def tags(self) -> Dict[str, str]: + def tags(self) -> dict[str, str]: return {**super().tags, "NodeIndex": str(self.node_index), } - def _get_public_ip_address(self) -> Optional[str]: + def _get_public_ip_address(self) -> str | None: return ContainerManager.get_ip_address(self, "node") - def _get_private_ip_address(self) -> Optional[str]: + def _get_private_ip_address(self) -> str | None: return self.public_ip_address def _refresh_instance_state(self): @@ -193,11 +197,11 @@ class DockerCluster(cluster.BaseCluster): # pylint: disable=abstract-method def __init__(self, # noqa: PLR0913 docker_image: str = DEFAULT_SCYLLA_DB_IMAGE, docker_image_tag: str = DEFAULT_SCYLLA_DB_IMAGE_TAG, - node_key_file: Optional[str] = None, + node_key_file: str | None = None, cluster_prefix: str = "cluster", node_prefix: str = "node", - node_type: Optional[str] = None, - n_nodes: Union[list, int] = 3, + node_type: str | None = None, + n_nodes: list | int = 3, params: dict = None) -> None: self.source_image = f"{docker_image}:{docker_image_tag}" self.node_container_image_tag = f"scylla-sct:{node_type}-{str(self.test_config.test_id())[:8]}" @@ -268,9 +272,9 @@ class ScyllaDockerCluster(cluster.BaseScyllaCluster, DockerCluster): # pylint: def __init__(self, # noqa: PLR0913 docker_image: str = DEFAULT_SCYLLA_DB_IMAGE, docker_image_tag: str = DEFAULT_SCYLLA_DB_IMAGE_TAG, - node_key_file: Optional[str] = None, - user_prefix: Optional[str] = None, - n_nodes: Union[list, str] = 3, + node_key_file: str | None = None, + user_prefix: str | None = None, + n_nodes: list | str = 3, params: dict = None) -> None: cluster_prefix = cluster.prepend_user_prefix(user_prefix, 'db-cluster') node_prefix = cluster.prepend_user_prefix(user_prefix, 'db-node') @@ -334,9 +338,9 @@ class LoaderSetDocker(cluster.BaseLoaderSet, DockerCluster): def __init__(self, # noqa: PLR0913 docker_image: str = DEFAULT_SCYLLA_DB_IMAGE, docker_image_tag: str = DEFAULT_SCYLLA_DB_IMAGE_TAG, - node_key_file: Optional[str] = None, - user_prefix: Optional[str] = None, - n_nodes: Union[list, str] = 3, + node_key_file: str | None = None, + user_prefix: str | None = None, + n_nodes: list | str = 3, params: dict = None) -> None: node_prefix = cluster.prepend_user_prefix(user_prefix, 'loader-node') cluster_prefix = cluster.prepend_user_prefix(user_prefix, 'loader-set') @@ -370,9 +374,9 @@ class DockerMonitoringNode(cluster.BaseNode): # pylint: disable=abstract-method def __init__(self, # noqa: PLR0913 parent_cluster: "MonitorSetDocker", node_prefix: str = "monitor-node", - base_logdir: Optional[str] = None, + base_logdir: str | None = None, node_index: int = 1, - ssh_login_info: Optional[dict] = None) -> None: + ssh_login_info: dict | None = None) -> None: super().__init__(name=f"{node_prefix}-{node_index}", parent_cluster=parent_cluster, base_logdir=base_logdir, @@ -421,8 +425,8 @@ def disable_daily_triggered_services(self): class MonitorSetDocker(cluster.BaseMonitorSet, DockerCluster): # pylint: disable=abstract-method def __init__(self, targets: dict, - user_prefix: Optional[str] = None, - n_nodes: Union[list, int] = 3, + user_prefix: str | None = None, + n_nodes: list | int = 3, params: dict = None) -> None: node_prefix = cluster.prepend_user_prefix(user_prefix, 'monitor-node') cluster_prefix = cluster.prepend_user_prefix(user_prefix, 'monitor-set') diff --git a/sdcm/cluster_gce.py b/sdcm/cluster_gce.py index 7ed5fbfaa41..e5e018aa6c9 100644 --- a/sdcm/cluster_gce.py +++ b/sdcm/cluster_gce.py @@ -11,39 +11,38 @@ # # Copyright (c) 2020 ScyllaDB +import logging import os import re import time -import logging -from typing import Dict, Any, ParamSpec, TypeVar -from textwrap import dedent -from functools import cached_property, cache from collections.abc import Callable +from functools import cache, cached_property +from textwrap import dedent +from typing import Any, ParamSpec, TypeVar -import tenacity import google.api_core.exceptions +import tenacity from google.cloud import compute_v1 from sdcm import cluster +from sdcm.keystore import pub_key_from_private_key_file from sdcm.sct_events import Severity from sdcm.sct_events.gce_events import GceInstanceEvent +from sdcm.sct_events.system import SpotTerminationEvent +from sdcm.utils.common import gce_meta_to_dict, list_instances_gce +from sdcm.utils.decorators import retrying from sdcm.utils.gce_utils import ( GceLoggingClient, create_instance, disk_from_image, - get_gce_compute_disks_client, - wait_for_extended_operation, gce_private_addresses, gce_public_addresses, - random_zone, gce_set_labels, + get_gce_compute_disks_client, + random_zone, + wait_for_extended_operation, ) from sdcm.wait import exponential_retry -from sdcm.keystore import pub_key_from_private_key_file -from sdcm.sct_events.system import SpotTerminationEvent -from sdcm.utils.common import list_instances_gce, gce_meta_to_dict -from sdcm.utils.decorators import retrying - SPOT_TERMINATION_CHECK_DELAY = 5 * 60 @@ -106,7 +105,7 @@ def init(self): super().init() @cached_property - def tags(self) -> Dict[str, str]: + def tags(self) -> dict[str, str]: return {**super().tags, "NodeIndex": str(self.node_index), } @@ -268,21 +267,21 @@ def __init__(self, gce_image, gce_image_type, gce_image_size, gce_network, gce_s def __str__(self): identifier = 'GCE Cluster %s | ' % self.name identifier += 'Image: %s | ' % os.path.basename(self._gce_image) - identifier += 'Root Disk: %s %s GB | ' % (self._gce_image_type, self._gce_image_size) + identifier += f'Root Disk: {self._gce_image_type} {self._gce_image_size} GB | ' if self._gce_n_local_ssd: identifier += 'Local SSD: %s | ' % self._gce_n_local_ssd if self._add_disks: for disk_type, disk_size in self._add_disks.items(): if int(disk_size): - identifier += '%s: %s | ' % (disk_type, disk_size) + identifier += f'{disk_type}: {disk_size} | ' identifier += 'Type: %s' % self._gce_instance_type return identifier def _get_disk_url(self, disk_type='pd-standard', dc_idx=0): - return "projects/%s/zones/%s/diskTypes/%s" % (self.project, self._gce_zone_names[dc_idx], disk_type) + return f"projects/{self.project}/zones/{self._gce_zone_names[dc_idx]}/diskTypes/{disk_type}" def _get_root_disk_struct(self, name, disk_type='pd-standard', dc_idx=0): - device_name = '%s-root-%s' % (name, disk_type) + device_name = f'{name}-root-{disk_type}' return disk_from_image(disk_type=self._get_disk_url(disk_type, dc_idx=dc_idx), disk_size_gb=self._gce_image_size, boot=True, @@ -293,7 +292,7 @@ def _get_root_disk_struct(self, name, disk_type='pd-standard', dc_idx=0): ) def _get_local_ssd_disk_struct(self, name, index, interface='NVME', dc_idx=0): - device_name = '%s-data-local-ssd-%s' % (name, index) + device_name = f'{name}-data-local-ssd-{index}' return disk_from_image(disk_type=self._get_disk_url('local-ssd', dc_idx=dc_idx), boot=False, auto_delete=True, @@ -303,7 +302,7 @@ def _get_local_ssd_disk_struct(self, name, index, interface='NVME', dc_idx=0): ) def _get_persistent_disk_struct(self, name, disk_size, disk_type='pd-ssd', dc_idx=0): - device_name = '%s-data-%s' % (name, disk_type) + device_name = f'{name}-data-{disk_type}' return disk_from_image(disk_type=self._get_disk_url(disk_type, dc_idx=dc_idx), disk_size_gb=disk_size, boot=False, @@ -501,7 +500,7 @@ def add_nodes(self, count, ec2_user_data='', dc_idx=0, rack=0, enable_auto_boots if self.test_config.REUSE_CLUSTER: instances = self._get_instances(dc_idx) if not instances: - raise RuntimeError("No nodes found for testId %s " % (self.test_config.test_id(),)) + raise RuntimeError(f"No nodes found for testId {self.test_config.test_id()} ") else: instances = self._create_instances(count, dc_idx, enable_auto_bootstrap, instance_type=instance_type) diff --git a/sdcm/cluster_k8s/__init__.py b/sdcm/cluster_k8s/__init__.py index 71f4ffd79bc..6398f2cd951 100644 --- a/sdcm/cluster_k8s/__init__.py +++ b/sdcm/cluster_k8s/__init__.py @@ -13,64 +13,75 @@ # pylint: disable=too-many-arguments # pylint: disable=too-many-lines - -from __future__ import annotations - -import os -import re import abc +import base64 +import contextlib import json +import logging import math +import os +import random +import re import shutil import tempfile import time -import base64 -import random -import socket -import logging import traceback -import contextlib -from pathlib import Path +from collections.abc import Callable +from contextlib import AbstractContextManager from copy import deepcopy from datetime import datetime from difflib import unified_diff -from functools import cached_property, partialmethod, partial +from functools import cached_property, partial, partialmethod +from pathlib import Path from tempfile import NamedTemporaryFile, TemporaryDirectory from textwrap import dedent from threading import Lock, RLock -from typing import Optional, Union, List, Dict, Any, ContextManager, Type, Tuple, Callable +from typing import Any -import yaml -import kubernetes as k8s -from kubernetes.client import exceptions as k8s_exceptions -from kubernetes.client import V1ConfigMap -from kubernetes.dynamic.resource import Resource, ResourceField, ResourceInstance, ResourceList, Subresource import invoke +import kubernetes as k8s +import yaml from invoke.exceptions import CommandTimedOut +from kubernetes.client import V1ConfigMap +from kubernetes.client import exceptions as k8s_exceptions +from kubernetes.dynamic.resource import ( + Resource, + ResourceField, + ResourceInstance, + ResourceList, + Subresource, +) -from sdcm import sct_abs_path, cluster -from sdcm.cluster import DeadNode, ClusterNodesNotReady -from sdcm.provision.scylla_yaml.scylla_yaml import ScyllaYaml -from sdcm.test_config import TestConfig +import sdcm.utils.sstable.load_inventory as datasets +from sdcm import cluster, sct_abs_path +from sdcm.cluster import ClusterNodesNotReady, DeadNode +from sdcm.cluster_k8s.operator_monitoring import ScyllaOperatorLogMonitoring +from sdcm.coredump import CoredumpExportFileThread from sdcm.db_stats import PrometheusDBStats +from sdcm.log import SDCMAdapter +from sdcm.mgmt import AnyManagerCluster +from sdcm.provision.scylla_yaml.scylla_yaml import ScyllaYaml from sdcm.remote import LOCALRUNNER, NETWORK_EXCEPTIONS from sdcm.remote.kubernetes_cmd_runner import ( KubernetesCmdRunner, KubernetesPodRunner, ) -from sdcm.coredump import CoredumpExportFileThread -from sdcm.log import SDCMAdapter -from sdcm.mgmt import AnyManagerCluster -from sdcm.sct_events.database import DatabaseLogEvent -from sdcm.sct_events.filters import DbEventsFilter from sdcm.sct_events.health import ClusterHealthValidatorEvent from sdcm.sct_events.system import TestFrameworkEvent -from sdcm.utils import properties -import sdcm.utils.sstable.load_inventory as datasets -from sdcm.utils.adaptive_timeouts import adaptive_timeout, Operations +from sdcm.test_config import TestConfig +from sdcm.utils.adaptive_timeouts import Operations, adaptive_timeout from sdcm.utils.ci_tools import get_test_name from sdcm.utils.common import download_from_github, shorten_cluster_name, walk_thru_data +from sdcm.utils.decorators import log_run_info, retrying +from sdcm.utils.decorators import timeout as timeout_wrapper from sdcm.utils.k8s import ( + JSON_PATCH_TYPE, + KUBECTL_TIMEOUT, + ApiCallRateLimiter, + HelmValues, + KubernetesOps, + ScyllaPodsIPChangeTrackerThread, + TokenUpdateThread, add_pool_node_affinity, convert_cpu_units_to_k8s_value, convert_cpu_value_from_k8s_to_units, @@ -79,28 +90,22 @@ get_helm_pool_affinity_values, get_pool_affinity_modifiers, get_preferred_pod_anti_affinity_values, - ApiCallRateLimiter, - JSON_PATCH_TYPE, - KubernetesOps, - KUBECTL_TIMEOUT, - HelmValues, - ScyllaPodsIPChangeTrackerThread, - TokenUpdateThread, ) -from sdcm.utils.decorators import log_run_info, retrying -from sdcm.utils.decorators import timeout as timeout_wrapper from sdcm.utils.k8s.chaos_mesh import ChaosMesh -from sdcm.utils.remote_logger import get_system_logging_thread, CertManagerLogger, ScyllaOperatorLogger, \ - KubectlClusterEventsLogger, ScyllaManagerLogger, KubernetesWrongSchedulingLogger, HaproxyIngressLogger +from sdcm.utils.remote_logger import ( + CertManagerLogger, + HaproxyIngressLogger, + KubectlClusterEventsLogger, + KubernetesWrongSchedulingLogger, + ScyllaManagerLogger, + ScyllaOperatorLogger, + get_system_logging_thread, +) from sdcm.utils.sstable.load_utils import SstableLoadUtils from sdcm.utils.version_utils import ComparableScyllaOperatorVersion from sdcm.wait import wait_for -from sdcm.cluster_k8s.operator_monitoring import ScyllaOperatorLogMonitoring - -ANY_KUBERNETES_RESOURCE = Union[ # pylint: disable=invalid-name - Resource, ResourceField, ResourceInstance, ResourceList, Subresource, -] +ANY_KUBERNETES_RESOURCE = Resource | ResourceField | ResourceInstance | ResourceList | Subresource NAMESPACE_CREATION_LOCK = Lock() NODE_INIT_LOCK = Lock() @@ -215,7 +220,7 @@ def __str__(self): return f"<{self.__class__.__name__}:{', '.join(data)}>" @cached_property - def affinity_modifiers(self) -> List[Callable]: + def affinity_modifiers(self) -> list[Callable]: return get_pool_affinity_modifiers(self.pool_label_name, self.name) @cached_property @@ -227,7 +232,7 @@ def pool_label_name(self) -> str: return self.k8s_cluster.POOL_LABEL_NAME @cached_property - def cpu_and_memory_capacity(self) -> Tuple[float, float]: + def cpu_and_memory_capacity(self) -> tuple[float, float]: for item in self.k8s_cluster.k8s_core_v1_api.list_node().items: if item.metadata.labels.get(self.pool_label_name, '') == self.name: capacity = item.status.allocatable @@ -287,19 +292,19 @@ class KubernetesCluster(metaclass=abc.ABCMeta): # pylint: disable=too-many-publ NODE_CONFIG_CRD_FILE = None TOKEN_UPDATE_NEEDED = True - api_call_rate_limiter: Optional[ApiCallRateLimiter] = None + api_call_rate_limiter: ApiCallRateLimiter | None = None - _cert_manager_journal_thread: Optional[CertManagerLogger] = None - _scylla_manager_journal_thread: Optional[ScyllaManagerLogger] = None - _scylla_operator_journal_thread: Optional[ScyllaOperatorLogger] = None - _scylla_operator_scheduling_thread: Optional[KubernetesWrongSchedulingLogger] = None - _haproxy_ingress_log_thread: Optional[HaproxyIngressLogger] = None - _scylla_cluster_events_threads: Dict[str, KubectlClusterEventsLogger] = {} - _scylla_operator_log_monitor_thread: Optional[ScyllaOperatorLogMonitoring] = None - _token_update_thread: Optional[TokenUpdateThread] = None - scylla_pods_ip_change_tracker_thread: Optional[ScyllaPodsIPChangeTrackerThread] = None + _cert_manager_journal_thread: CertManagerLogger | None = None + _scylla_manager_journal_thread: ScyllaManagerLogger | None = None + _scylla_operator_journal_thread: ScyllaOperatorLogger | None = None + _scylla_operator_scheduling_thread: KubernetesWrongSchedulingLogger | None = None + _haproxy_ingress_log_thread: HaproxyIngressLogger | None = None + _scylla_cluster_events_threads: dict[str, KubectlClusterEventsLogger] = {} + _scylla_operator_log_monitor_thread: ScyllaOperatorLogMonitoring | None = None + _token_update_thread: TokenUpdateThread | None = None + scylla_pods_ip_change_tracker_thread: ScyllaPodsIPChangeTrackerThread | None = None - pools: Dict[str, CloudK8sNodePool] + pools: dict[str, CloudK8sNodePool] scylla_pods_ip_mapping = {} def __init__(self, params: dict, user_prefix: str = '', region_name: str = None, cluster_uuid: str = None): @@ -356,7 +361,7 @@ def cluster_backend(self): return self.params.get("cluster_backend") @property - def k8s_server_url(self) -> Optional[str]: + def k8s_server_url(self) -> str | None: return None @cached_property @@ -1494,7 +1499,7 @@ def s3_provider_endpoint(self) -> str: return f"http://{self.minio_ip_address}:9000" @cached_property - def tags(self) -> Dict[str, str]: + def tags(self) -> dict[str, str]: return get_tags_from_params(self.params) @cached_property @@ -1573,7 +1578,7 @@ def update_secret_from_data(self, secret_name: str, namespace: str, data: dict, self.k8s_core_v1_api.patch_namespaced_secret(secret_name, namespace, existing) def create_secret_from_directory(self, secret_name: str, path: str, namespace: str, secret_type: str = 'generic', # noqa: PLR0913 - only_files: List[str] = None): + only_files: list[str] = None): files = [fname for fname in os.listdir(path) if os.path.isfile(os.path.join(path, fname)) and (not only_files or fname in only_files)] cmd = f'create secret {secret_type} {secret_name} ' + \ @@ -1704,7 +1709,7 @@ def scylla_config_map(self, namespace: str = SCYLLA_NAMESPACE) -> dict: ) @contextlib.contextmanager - def manage_file_in_scylla_config_map(self, filename: str, namespace: str = SCYLLA_NAMESPACE) -> ContextManager: + def manage_file_in_scylla_config_map(self, filename: str, namespace: str = SCYLLA_NAMESPACE) -> AbstractContextManager: """Update scylla.yaml or cassandra-rackdc.properties, k8s way Scylla Operator handles file updates using ConfigMap resource @@ -1731,11 +1736,11 @@ def manage_file_in_scylla_config_map(self, filename: str, namespace: str = SCYLL scylla_config_map[filename] = new_data_as_str self.scylla_restart_required = True - def remote_scylla_yaml(self, namespace: str = SCYLLA_NAMESPACE) -> ContextManager: + def remote_scylla_yaml(self, namespace: str = SCYLLA_NAMESPACE) -> AbstractContextManager: return self.manage_file_in_scylla_config_map( filename='scylla.yaml', namespace=namespace) - def remote_cassandra_rackdc_properties(self, namespace: str = SCYLLA_NAMESPACE) -> ContextManager: + def remote_cassandra_rackdc_properties(self, namespace: str = SCYLLA_NAMESPACE) -> AbstractContextManager: return self.manage_file_in_scylla_config_map( filename='cassandra-rackdc.properties', namespace=namespace) @@ -1771,7 +1776,7 @@ class BasePodContainer(cluster.BaseNode): # pylint: disable=too-many-public-met pod_terminate_timeout = 5 # minutes def __init__(self, name: str, parent_cluster: PodCluster, node_prefix: str = "node", node_index: int = 1, # noqa: PLR0913 - base_logdir: Optional[str] = None, dc_idx: int = 0, rack=0): + base_logdir: str | None = None, dc_idx: int = 0, rack=0): self.node_index = node_index cluster.BaseNode.__init__( self, name=name, @@ -1794,7 +1799,7 @@ def is_docker() -> bool: return True @cached_property - def tags(self) -> Dict[str, str]: + def tags(self) -> dict[str, str]: return {**super().tags, "NodeIndex": str(self.node_index), } @@ -1882,7 +1887,7 @@ def cql_address(self): return self.ip_address @property - def private_ip_address(self) -> Optional[str]: + def private_ip_address(self) -> str | None: if ip := self.k8s_cluster.scylla_pods_ip_mapping.get( self.parent_cluster.namespace, {}).get(self.name, {}).get('current_ip'): return ip @@ -2043,20 +2048,20 @@ def proposed_scylla_yaml(self) -> ScyllaYaml: parent_cluster: ScyllaPodCluster - def actual_scylla_yaml(self) -> ContextManager[ScyllaYaml]: + def actual_scylla_yaml(self) -> AbstractContextManager[ScyllaYaml]: return super().remote_scylla_yaml() - def actual_cassandra_rackdc_properties(self) -> ContextManager: + def actual_cassandra_rackdc_properties(self) -> AbstractContextManager: return super().remote_cassandra_rackdc_properties() - def remote_scylla_yaml(self) -> ContextManager: + def remote_scylla_yaml(self) -> AbstractContextManager: """ Scylla Operator handles 'scylla.yaml' file updates using ConfigMap resource and we don't need to update it on each node separately. """ return self.k8s_cluster.remote_scylla_yaml() - def remote_cassandra_rackdc_properties(self) -> ContextManager: + def remote_cassandra_rackdc_properties(self) -> AbstractContextManager: """ Scylla Operator handles 'cassandra-rackdc.properties' file updates using ConfigMap resource and we don't need to update it on each node separately. @@ -2238,7 +2243,7 @@ def node_type(self) -> str: def __init__(self, name: str, parent_cluster: PodCluster, # noqa: PLR0913 node_prefix: str = "node", node_index: int = 1, - base_logdir: Optional[str] = None, dc_idx: int = 0, rack=0): + base_logdir: str | None = None, dc_idx: int = 0, rack=0): self.loader_cluster_name = parent_cluster.loader_cluster_name self.loader_name = name self.loader_pod_name_template = f"{self.loader_name}-pod" @@ -2290,20 +2295,20 @@ def restart(self): class PodCluster(cluster.BaseCluster): - PodContainerClass: Type[BasePodContainer] = BasePodContainer + PodContainerClass: type[BasePodContainer] = BasePodContainer def __init__(self, # noqa: PLR0913 - k8s_clusters: List[KubernetesCluster], + k8s_clusters: list[KubernetesCluster], namespace: str = "default", - container: Optional[str] = None, - cluster_uuid: Optional[str] = None, + container: str | None = None, + cluster_uuid: str | None = None, cluster_prefix: str = "cluster", node_prefix: str = "node", - node_type: Optional[str] = None, - n_nodes: Union[list, int] = 3, - params: Optional[dict] = None, - node_pool_name: Optional[str] = '', - add_nodes: Optional[bool] = True, + node_type: str | None = None, + n_nodes: list | int = 3, + params: dict | None = None, + node_pool_name: str | None = '', + add_nodes: bool | None = True, ) -> None: self.k8s_clusters = k8s_clusters self.namespace = namespace @@ -2346,7 +2351,7 @@ def add_nodes(self, # noqa: PLR0913 rack: int = 0, enable_auto_bootstrap: bool = False, instance_type=None, - ) -> List[BasePodContainer]: + ) -> list[BasePodContainer]: # TODO: make it work when we have decommissioned (by nodetool) nodes. # Now it will fail because pod which hosts decommissioned Scylla member is reported @@ -2364,7 +2369,7 @@ def add_nodes(self, # noqa: PLR0913 k8s_pods = KubernetesOps.list_pods(self.k8s_clusters[dc_idx], namespace=self.namespace) nodes = [] for pod in k8s_pods: - if not any((x for x in pod.status.container_statuses if x.name == self.container)): + if not any(x for x in pod.status.container_statuses if x.name == self.container): continue is_already_registered = False for node in current_dc_nodes: @@ -2403,7 +2408,7 @@ def wait_sts_rollout_restart(self, pods_to_wait: int, dc_idx: int = 0): namespace=self.namespace, timeout=timeout * 60 + 10) - def get_nodes_reboot_timeout(self, count) -> Union[float, int]: + def get_nodes_reboot_timeout(self, count) -> float | int: """ Return readiness timeout (in minutes) for case when nodes are restarted sums out readiness and terminate timeouts for given nodes @@ -2415,7 +2420,7 @@ def pod_selector(self): return '' @cached_property - def get_nodes_readiness_delay(self) -> Union[float, int]: + def get_nodes_readiness_delay(self) -> float | int: return self.PodContainerClass.pod_readiness_delay def wait_for_pods_readiness(self, pods_to_wait: int, total_pods: int, readiness_timeout: int = None, @@ -2460,11 +2465,11 @@ class ScyllaPodCluster(cluster.BaseScyllaCluster, PodCluster): # pylint: disabl node_setup_requires_scylla_restart = False def __init__(self, # noqa: PLR0913 - k8s_clusters: List[KubernetesCluster], - scylla_cluster_name: Optional[str] = None, - user_prefix: Optional[str] = None, - n_nodes: Union[list, int] = 3, - params: Optional[dict] = None, + k8s_clusters: list[KubernetesCluster], + scylla_cluster_name: str | None = None, + user_prefix: str | None = None, + n_nodes: list | int = 3, + params: dict | None = None, node_pool_name: str = '', add_nodes: bool = True, ) -> None: @@ -2565,7 +2570,7 @@ def _k8s_scylla_cluster_api(self, dc_idx: int = 0) -> Resource: @retrying(n=20, sleep_time=3, allowed_exceptions=(k8s_exceptions.ApiException, ), message="Failed to update ScyllaCluster's spec...") def replace_scylla_cluster_value(self, path: str, value: Any, - dc_idx: int = 0) -> Optional[ANY_KUBERNETES_RESOURCE]: + dc_idx: int = 0) -> ANY_KUBERNETES_RESOURCE | None: self.k8s_clusters[dc_idx].log.debug( "Replace `%s' with `%s' in %s's spec", path, value, self.scylla_cluster_name) return self._k8s_scylla_cluster_api(dc_idx=dc_idx).patch( @@ -2574,7 +2579,7 @@ def replace_scylla_cluster_value(self, path: str, value: Any, namespace=self.namespace, content_type=JSON_PATCH_TYPE) - def get_scylla_cluster_value(self, path: str, dc_idx: int = 0) -> Optional[ANY_KUBERNETES_RESOURCE]: + def get_scylla_cluster_value(self, path: str, dc_idx: int = 0) -> ANY_KUBERNETES_RESOURCE | None: """ Get scylla cluster value from kubernetes API. """ @@ -2582,7 +2587,7 @@ def get_scylla_cluster_value(self, path: str, dc_idx: int = 0) -> Optional[ANY_K namespace=self.namespace, name=self.scylla_cluster_name) return walk_thru_data(cluster_data, path) - def get_scylla_cluster_plain_value(self, path: str, dc_idx: int = 0) -> Union[Dict, List, str, None]: + def get_scylla_cluster_plain_value(self, path: str, dc_idx: int = 0) -> dict | list | str | None: """ Get scylla cluster value from kubernetes API and converts result to basic python data types. Use it if you are going to modify the data. @@ -2626,13 +2631,13 @@ def remove_scylla_cluster_value(self, path: str, element_name: str, dc_idx: int namespace=self.namespace, content_type=JSON_PATCH_TYPE) - def scylla_config_map(self, dc_idx: int = 0) -> ContextManager: + def scylla_config_map(self, dc_idx: int = 0) -> AbstractContextManager: return self.k8s_clusters[dc_idx].scylla_config_map() - def remote_scylla_yaml(self, dc_idx: int = 0) -> ContextManager: + def remote_scylla_yaml(self, dc_idx: int = 0) -> AbstractContextManager: return self.k8s_clusters[dc_idx].remote_scylla_yaml() - def remote_cassandra_rackdc_properties(self, dc_idx: int = 0) -> ContextManager: + def remote_cassandra_rackdc_properties(self, dc_idx: int = 0) -> AbstractContextManager: return self.k8s_clusters[dc_idx].remote_cassandra_rackdc_properties() def update_seed_provider(self): @@ -2757,7 +2762,7 @@ def add_nodes(self, # pylint: disable=too-many-locals,too-many-branches noqa: dc_idx: int = None, rack: int = 0, enable_auto_bootstrap: bool = False, - instance_type=None) -> List[BasePodContainer]: + instance_type=None) -> list[BasePodContainer]: if dc_idx is None: dc_idx = list(range(len(self.k8s_clusters))) elif isinstance(dc_idx, int): @@ -3011,11 +3016,11 @@ def pod_selector(self): class LoaderPodCluster(cluster.BaseLoaderSet, PodCluster): def __init__(self, # noqa: PLR0913 - k8s_clusters: List[KubernetesCluster], - loader_cluster_name: Optional[str] = None, - user_prefix: Optional[str] = None, - n_nodes: Union[list, int] = 3, - params: Optional[dict] = None, + k8s_clusters: list[KubernetesCluster], + loader_cluster_name: str | None = None, + user_prefix: str | None = None, + n_nodes: list | int = 3, + params: dict | None = None, node_pool_name: str = '', add_nodes: bool = True, ) -> None: @@ -3054,7 +3059,7 @@ def pod_selector(self): def node_setup(self, node: BasePodContainer, verbose: bool = False, - db_node_address: Optional[str] = None, + db_node_address: str | None = None, **kwargs) -> None: if self.params.get('client_encrypt'): @@ -3076,7 +3081,7 @@ def add_nodes(self, # noqa: PLR0913 rack: int = 0, enable_auto_bootstrap: bool = False, instance_type=None - ) -> List[BasePodContainer]: + ) -> list[BasePodContainer]: if self.loader_cluster_created: raise NotImplementedError( "Changing number of nodes in LoaderPodCluster is not supported.") @@ -3144,7 +3149,7 @@ def add_nodes(self, # noqa: PLR0913 return new_nodes -def get_tags_from_params(params: dict) -> Dict[str, str]: +def get_tags_from_params(params: dict) -> dict[str, str]: tags = TestConfig().common_tags() if params.get("post_behavior_k8s_cluster").startswith("keep"): # NOTE: case when 'post_behavior_k8s_cluster' is 'keep' or 'keep-on-failure': diff --git a/sdcm/cluster_k8s/eks.py b/sdcm/cluster_k8s/eks.py index 3903b19a09b..c60cf6145f9 100644 --- a/sdcm/cluster_k8s/eks.py +++ b/sdcm/cluster_k8s/eks.py @@ -12,27 +12,25 @@ # Copyright (c) 2020 ScyllaDB import base64 import os -import time import pprint +import time +from collections.abc import Callable +from functools import cached_property from textwrap import dedent from threading import Lock -from typing import List, Dict, Literal, ParamSpec, TypeVar -from functools import cached_property -from collections.abc import Callable +from typing import Literal, ParamSpec, TypeVar import boto3 import tenacity -from mypy_boto3_ec2.type_defs import LaunchTemplateBlockDeviceMappingRequestTypeDef, \ - LaunchTemplateEbsBlockDeviceRequestTypeDef, RequestLaunchTemplateDataTypeDef, \ - LaunchTemplateTagSpecificationRequestTypeDef +from mypy_boto3_ec2.type_defs import ( + LaunchTemplateBlockDeviceMappingRequestTypeDef, + LaunchTemplateEbsBlockDeviceRequestTypeDef, + LaunchTemplateTagSpecificationRequestTypeDef, + RequestLaunchTemplateDataTypeDef, +) -from sdcm import sct_abs_path, cluster +from sdcm import cluster, ec2_client, sct_abs_path from sdcm.cluster_aws import MonitorSetAWS -from sdcm import ec2_client -from sdcm.utils.ci_tools import get_test_name -from sdcm.utils.common import list_instances_aws -from sdcm.utils.k8s import TokenUpdateThread -from sdcm.wait import wait_for, exponential_retry from sdcm.cluster_k8s import ( BaseScyllaPodContainer, CloudK8sNodePool, @@ -41,11 +39,14 @@ ) from sdcm.remote import LOCALRUNNER from sdcm.utils.aws_utils import ( + EksClusterCleanupMixin, get_ec2_network_configuration, tags_as_ec2_tags, - EksClusterCleanupMixin, ) - +from sdcm.utils.ci_tools import get_test_name +from sdcm.utils.common import list_instances_aws +from sdcm.utils.k8s import TokenUpdateThread +from sdcm.wait import exponential_retry, wait_for P = ParamSpec("P") # pylint: disable=invalid-name R = TypeVar("R") # pylint: disable=invalid-name @@ -57,7 +58,7 @@ def init_k8s_eks_cluster(region_name: str, availability_zone: str, params: dict, - credentials: List[cluster.UserRemoteCredentials], + credentials: list[cluster.UserRemoteCredentials], cluster_uuid: str = None): """Dedicated for the usage by the 'Tester' class which orchestrates all the resources creation. @@ -166,8 +167,8 @@ def __init__( # noqa: PLR0913 instance_type: str, role_arn: str, labels: dict = None, - security_group_ids: List[str] = None, - ec2_subnet_ids: List[str] = None, + security_group_ids: list[str] = None, + ec2_subnet_ids: list[str] = None, ssh_key_pair_name: str = None, provision_type: Literal['ON_DEMAND', 'SPOT'] = 'ON_DEMAND', launch_template: str = None, @@ -325,7 +326,7 @@ class EksCluster(KubernetesCluster, EksClusterCleanupMixin): # pylint: disable= NODE_PREPARE_FILE = sct_abs_path("sdcm/k8s_configs/eks/scylla-node-prepare.yaml") NODE_CONFIG_CRD_FILE = sct_abs_path("sdcm/k8s_configs/eks/node-config-crd.yaml") STORAGE_CLASS_FILE = sct_abs_path("sdcm/k8s_configs/eks/storageclass.yaml") - pools: Dict[str, EksNodePool] + pools: dict[str, EksNodePool] short_cluster_name: str # pylint: disable=too-many-arguments @@ -751,7 +752,7 @@ def terminate_k8s_node(self): class EksScyllaPodCluster(ScyllaPodCluster): PodContainerClass = EksScyllaPodContainer - nodes: List[EksScyllaPodContainer] + nodes: list[EksScyllaPodContainer] # pylint: disable=too-many-arguments def add_nodes(self, # noqa: PLR0913 @@ -761,7 +762,7 @@ def add_nodes(self, # noqa: PLR0913 rack: int = 0, enable_auto_bootstrap: bool = False, instance_type=None - ) -> List[EksScyllaPodContainer]: + ) -> list[EksScyllaPodContainer]: new_nodes = super().add_nodes(count=count, ec2_user_data=ec2_user_data, dc_idx=dc_idx, diff --git a/sdcm/cluster_k8s/gke.py b/sdcm/cluster_k8s/gke.py index 97ba3e679ef..8c9a14fe003 100644 --- a/sdcm/cluster_k8s/gke.py +++ b/sdcm/cluster_k8s/gke.py @@ -11,35 +11,39 @@ # # Copyright (c) 2020 ScyllaDB -from textwrap import dedent -from typing import List, Dict, ParamSpec, TypeVar -from functools import cached_property +import json +import tempfile from collections.abc import Callable +from functools import cached_property +from textwrap import dedent +from typing import ParamSpec, TypeVar -import tempfile -import json -import yaml import tenacity +import yaml from google.cloud import compute_v1 -from sdcm import sct_abs_path, cluster -from sdcm.wait import exponential_retry -from sdcm.utils.common import list_instances_gce, gce_meta_to_dict -from sdcm.utils.k8s import ApiCallRateLimiter, TokenUpdateThread +from sdcm import cluster, sct_abs_path +from sdcm.cluster_gce import MonitorSetGCE +from sdcm.cluster_k8s import ( + BaseScyllaPodContainer, + CloudK8sNodePool, + KubernetesCluster, + ScyllaPodCluster, +) +from sdcm.keystore import KeyStore +from sdcm.remote import LOCALRUNNER +from sdcm.utils.ci_tools import get_test_name +from sdcm.utils.common import gce_meta_to_dict, list_instances_gce from sdcm.utils.gce_utils import ( GcloudContainerMixin, GcloudContextManager, + gce_private_addresses, + gce_public_addresses, get_gce_compute_instances_client, wait_for_extended_operation, - gce_public_addresses, - gce_private_addresses, ) -from sdcm.utils.ci_tools import get_test_name -from sdcm.cluster_k8s import KubernetesCluster, ScyllaPodCluster, BaseScyllaPodContainer, CloudK8sNodePool -from sdcm.cluster_gce import MonitorSetGCE -from sdcm.keystore import KeyStore -from sdcm.remote import LOCALRUNNER - +from sdcm.utils.k8s import ApiCallRateLimiter, TokenUpdateThread +from sdcm.wait import exponential_retry GKE_API_CALL_RATE_LIMIT = 5 # ops/s GKE_API_CALL_QUEUE_SIZE = 1000 # ops @@ -264,7 +268,7 @@ class GkeCluster(KubernetesCluster): NODE_PREPARE_FILE = sct_abs_path("sdcm/k8s_configs/gke/scylla-node-prepare.yaml") NODE_CONFIG_CRD_FILE = sct_abs_path("sdcm/k8s_configs/gke/node-config-crd.yaml") TOKEN_UPDATE_NEEDED = False - pools: Dict[str, GkeNodePool] + pools: dict[str, GkeNodePool] # pylint: disable=too-many-arguments,too-many-locals def __init__(self, # noqa: PLR0913 @@ -595,7 +599,7 @@ def add_nodes(self, # noqa: PLR0913 dc_idx: int = None, rack: int = 0, enable_auto_bootstrap: bool = False, - instance_type=None) -> List[GkeScyllaPodContainer]: + instance_type=None) -> list[GkeScyllaPodContainer]: new_nodes = super().add_nodes(count=count, ec2_user_data=ec2_user_data, dc_idx=dc_idx, diff --git a/sdcm/cluster_k8s/mini_k8s.py b/sdcm/cluster_k8s/mini_k8s.py index a5803cb6eea..f3af044af5c 100644 --- a/sdcm/cluster_k8s/mini_k8s.py +++ b/sdcm/cluster_k8s/mini_k8s.py @@ -15,11 +15,11 @@ import logging import os import re -from typing import Tuple, Optional, Callable -from textwrap import dedent +from collections.abc import Callable from functools import cached_property -import yaml +from textwrap import dedent +import yaml from invoke.exceptions import UnexpectedExit from sdcm import ( @@ -27,13 +27,7 @@ sct_abs_path, ) from sdcm.cluster import LocalK8SHostNode -from sdcm.remote import LOCALRUNNER -from sdcm.remote.base import CommandRunner from sdcm.cluster_k8s import ( - CloudK8sNodePool, - KubernetesCluster, - BaseScyllaPodContainer, - ScyllaPodCluster, COMMON_CONTAINERS_RESOURCES, INGRESS_CONTROLLER_CONFIG_PATH, K8S_LOCAL_VOLUME_PROVISIONER_VERSION, @@ -43,13 +37,18 @@ SCYLLA_MANAGER_AGENT_RESOURCES, SCYLLA_MANAGER_AGENT_VERSION_IN_SCYLLA_MANAGER, SCYLLA_VERSION_IN_SCYLLA_MANAGER, + BaseScyllaPodContainer, + CloudK8sNodePool, + KubernetesCluster, + ScyllaPodCluster, ) -from sdcm.utils.k8s import TokenUpdateThread, HelmValues -from sdcm.utils.k8s.chaos_mesh import ChaosMesh +from sdcm.remote import LOCALRUNNER +from sdcm.remote.base import CommandRunner +from sdcm.utils import version_utils from sdcm.utils.decorators import retrying from sdcm.utils.docker_utils import docker_hub_login -from sdcm.utils import version_utils - +from sdcm.utils.k8s import HelmValues, TokenUpdateThread +from sdcm.utils.k8s.chaos_mesh import ChaosMesh SRC_APISERVER_AUDIT_POLICY = sct_abs_path("sdcm/k8s_configs/local-kind/audit-policy.yaml") DST_APISERVER_AUDIT_POLICY = "/etc/kubernetes/policies/audit-policy.yaml" @@ -75,7 +74,7 @@ def resize(self, num_nodes: int): pass @cached_property - def cpu_and_memory_capacity(self) -> Tuple[float, float]: + def cpu_and_memory_capacity(self) -> tuple[float, float]: cpu_per_member = 1 # NOTE: Setting '1' to 'memory_for_cpu_multiplier' we will get failure incresing CPUs # Setting '2' to 'memory_for_cpu_multiplier' we will be able to add 1 CPU per member @@ -282,7 +281,7 @@ def _create_kubectl_config_cmd(self): @property @abc.abstractmethod - def _target_user(self) -> Optional[str]: + def _target_user(self) -> str | None: pass @property @@ -331,7 +330,7 @@ def on_deploy_completed(self): @cached_property def minio_images(self): - with open(LOCAL_MINIO_DIR + '/values.yaml', mode='r', encoding='utf8') as minio_config_stream: + with open(LOCAL_MINIO_DIR + '/values.yaml', encoding='utf8') as minio_config_stream: minio_config = yaml.safe_load(minio_config_stream) return [ f"{minio_config['image']['repository']}:{minio_config['image']['tag']}", @@ -340,7 +339,7 @@ def minio_images(self): @cached_property def static_local_volume_provisioner_image(self): - with open(LOCAL_PROVISIONER_FILE, mode='r', encoding='utf8') as provisioner_config_stream: + with open(LOCAL_PROVISIONER_FILE, encoding='utf8') as provisioner_config_stream: for doc in yaml.safe_load_all(provisioner_config_stream): if doc["kind"] != "DaemonSet": continue @@ -371,7 +370,7 @@ def ingress_controller_images(self): for subfile in subfiles: if not subfile.endswith('yaml'): continue - with open(os.path.join(root, subfile), mode='r', encoding='utf8') as file_stream: + with open(os.path.join(root, subfile), encoding='utf8') as file_stream: for doc in yaml.safe_load_all(file_stream): if doc["kind"] != "Deployment": continue @@ -459,7 +458,7 @@ class LocalKindCluster(LocalMinimalClusterBase): docker_pull: Callable docker_tag: Callable host_node: 'BaseNode' - scylla_image: Optional[str] + scylla_image: str | None software_version: str _target_user: str _create_kubectl_config_cmd: str = '/var/tmp/kind export kubeconfig' diff --git a/sdcm/cluster_k8s/operator_monitoring.py b/sdcm/cluster_k8s/operator_monitoring.py index 7ac4b2b856e..5c16e01985c 100644 --- a/sdcm/cluster_k8s/operator_monitoring.py +++ b/sdcm/cluster_k8s/operator_monitoring.py @@ -11,15 +11,15 @@ # # Copyright (c) 2020 ScyllaDB -import threading import logging -from typing import Iterable +import threading +from collections.abc import Iterable -from sdcm.utils.file import File -from sdcm.utils.decorators import timeout from sdcm.sct_events.operator import ( SCYLLA_OPERATOR_EVENT_PATTERNS, ) +from sdcm.utils.decorators import timeout +from sdcm.utils.file import File class ScyllaOperatorLogMonitoring(threading.Thread): diff --git a/sdcm/coredump.py b/sdcm/coredump.py index 8b4ef615684..0a1b6ed7cf0 100644 --- a/sdcm/coredump.py +++ b/sdcm/coredump.py @@ -11,23 +11,22 @@ # # Copyright (c) 2020 ScyllaDB +import json import os import re import time -import json from abc import abstractmethod -from typing import List, Optional, Dict +from dataclasses import dataclass from datetime import datetime from functools import cached_property -from threading import Thread, Event -from dataclasses import dataclass +from threading import Event, Thread from sdcm.log import SDCMAdapter from sdcm.remote import NETWORK_EXCEPTIONS +from sdcm.sct_events.decorators import raise_event_on_failure +from sdcm.sct_events.system import CoreDumpEvent from sdcm.utils.decorators import timeout from sdcm.utils.version_utils import get_systemd_version -from sdcm.sct_events.system import CoreDumpEvent -from sdcm.sct_events.decorators import raise_event_on_failure # pylint: disable=too-many-instance-attributes @@ -36,7 +35,7 @@ class CoreDumpInfo: pid: str node: 'BaseNode' = None corefile: str = '' - source_timestamp: Optional[float] = None + source_timestamp: float | None = None coredump_info: str = '' download_instructions: str = '' download_url: str = '' @@ -62,7 +61,7 @@ def __str__(self): def update(self, # noqa: PLR0913 node: 'BaseNode' = None, corefile: str = None, - source_timestamp: Optional[float] = None, + source_timestamp: float | None = None, coredump_info: str = None, download_instructions: str = None, download_url: str = None, @@ -93,10 +92,10 @@ def __init__(self, node: 'BaseNode', max_core_upload_limit: int): self.node = node self.log = SDCMAdapter(node.log, extra={"prefix": self.__class__.__name__}) self.max_core_upload_limit = max_core_upload_limit - self.found: List[CoreDumpInfo] = [] - self.in_progress: List[CoreDumpInfo] = [] - self.completed: List[CoreDumpInfo] = [] - self.uploaded: List[CoreDumpInfo] = [] + self.found: list[CoreDumpInfo] = [] + self.in_progress: list[CoreDumpInfo] = [] + self.completed: list[CoreDumpInfo] = [] + self.uploaded: list[CoreDumpInfo] = [] self.termination_event = Event() self.exception = None super().__init__(daemon=True) @@ -128,7 +127,7 @@ def main_cycle_body(self): new_cores = self.extract_info_from_core_pids(self.get_list_of_cores(), exclude_cores=self.found) self.push_new_cores_to_process(new_cores) - def push_new_cores_to_process(self, new_cores: List[CoreDumpInfo]): + def push_new_cores_to_process(self, new_cores: list[CoreDumpInfo]): self.found.extend(new_cores) for core_dump in new_cores: if 'bash' in core_dump.executable: @@ -145,9 +144,9 @@ def process_coredumps(self): def _process_coredumps( self, - in_progress: List[CoreDumpInfo], - completed: List[CoreDumpInfo], - uploaded: List[CoreDumpInfo] + in_progress: list[CoreDumpInfo], + completed: list[CoreDumpInfo], + uploaded: list[CoreDumpInfo] ): """ Get core files from node and report them @@ -176,7 +175,7 @@ def _process_coredumps( pass @abstractmethod - def get_list_of_cores(self) -> Optional[List[CoreDumpInfo]]: + def get_list_of_cores(self) -> list[CoreDumpInfo] | None: ... def publish_event(self, core_info: CoreDumpInfo): @@ -186,7 +185,7 @@ def publish_event(self, core_info: CoreDumpInfo): self.log.error(f"Failed to publish coredump event due to the: {str(exc)}") def extract_info_from_core_pids( - self, new_cores: Optional[List[CoreDumpInfo]], exclude_cores: List[CoreDumpInfo]) -> List[CoreDumpInfo]: + self, new_cores: list[CoreDumpInfo] | None, exclude_cores: list[CoreDumpInfo]) -> list[CoreDumpInfo]: output = [] for new_core_info in new_cores: found = False @@ -208,10 +207,10 @@ def _upload_coredump(self, core_info: CoreDumpInfo): upload_url = f'upload.scylladb.com/{coredump_id}/{os.path.basename(coredump)}' self.log.info('Uploading coredump %s to %s', coredump, upload_url) self.node.remoter.run("sudo curl --request PUT --fail --show-error --upload-file " - "'%s' 'https://%s'" % (coredump, upload_url)) - download_url = 'https://storage.cloud.google.com/%s' % upload_url + f"'{coredump}' 'https://{upload_url}'") + download_url = f'https://storage.cloud.google.com/{upload_url}' self.log.info("You can download it by %s (available for ScyllaDB employee)", download_url) - download_instructions = 'gsutil cp gs://%s .\ngunzip %s' % (upload_url, coredump) + download_instructions = f'gsutil cp gs://{upload_url} .\ngunzip {coredump}' core_info.download_url, core_info.download_instructions = download_url, download_instructions def upload_coredump(self, core_info: CoreDumpInfo): @@ -299,7 +298,7 @@ def systemd_version(self): self.log.warning("failed to get systemd version:", exc_info=True) return systemd_version - def get_list_of_cores_json(self) -> Optional[List[CoreDumpInfo]]: + def get_list_of_cores_json(self) -> list[CoreDumpInfo] | None: result = self.node.remoter.run( 'sudo coredumpctl -q --json=short', verbose=False, ignore_status=True) if not result.ok: @@ -322,7 +321,7 @@ def get_list_of_cores_json(self) -> Optional[List[CoreDumpInfo]]: pid_list.append(CoreDumpInfo(pid=str(dump['pid']), node=self.node)) return pid_list - def get_list_of_cores(self) -> Optional[List[CoreDumpInfo]]: + def get_list_of_cores(self) -> list[CoreDumpInfo] | None: if self.systemd_version >= 248: # noqa: PLR2004 # since systemd/systemd@0689cfd we have option to get # the coredump information in json format @@ -441,7 +440,7 @@ def _get_coredumpctl_info(self, core_info: CoreDumpInfo): """ Get coredump backtraces. - :param pid: PID of the core. + :param core_info: coredump info. :return: fabric.Result output """ output = self.node.remoter.run( @@ -457,7 +456,7 @@ class CoredumpExportFileThread(CoredumpThreadBase): """ checkup_time_core_to_complete = 1 - def __init__(self, node: 'BaseNode', max_core_upload_limit: int, coredump_directories: List[str]): + def __init__(self, node: 'BaseNode', max_core_upload_limit: int, coredump_directories: list[str]): self.coredumps_directories = coredump_directories super().__init__(node=node, max_core_upload_limit=max_core_upload_limit) @@ -474,7 +473,7 @@ def _install_file(self): raise RuntimeError("Distro is not supported") @staticmethod - def _extract_core_info_from_file_name(corefile: str) -> Dict[str, str]: + def _extract_core_info_from_file_name(corefile: str) -> dict[str, str]: data = os.path.splitext(corefile)[0].split('-') return { 'host': data[0], @@ -485,7 +484,7 @@ def _extract_core_info_from_file_name(corefile: str) -> Dict[str, str]: 'source_timestamp': data[5], } - def get_list_of_cores(self) -> Optional[List[CoreDumpInfo]]: + def get_list_of_cores(self) -> list[CoreDumpInfo] | None: output = [] for directory in self.coredumps_directories: for corefile in self.node.remoter.sudo(f'ls {directory}', verbose=False, ignore_status=True).stdout.split(): diff --git a/sdcm/db_log_reader.py b/sdcm/db_log_reader.py index 119a606b664..340f4a7af58 100644 --- a/sdcm/db_log_reader.py +++ b/sdcm/db_log_reader.py @@ -18,12 +18,11 @@ import os import re from functools import cached_property -from multiprocessing import Process, Event, Queue -from typing import Optional +from multiprocessing import Event, Process, Queue from sdcm.remote.base import CommandRunner from sdcm.sct_events.base import LogEvent -from sdcm.sct_events.database import get_pattern_to_event_to_func_mapping, BACKTRACE_RE +from sdcm.sct_events.database import BACKTRACE_RE, get_pattern_to_event_to_func_mapping from sdcm.sct_events.decorators import raise_event_on_failure from sdcm.utils.common import make_threads_be_daemonic_by_default @@ -55,7 +54,7 @@ def __init__(self, # noqa: PLR0913 remoter: CommandRunner, node_name: str, system_event_patterns: list, - decoding_queue: Optional[Queue], + decoding_queue: Queue | None, log_lines: bool, ): self._system_log = system_log @@ -236,7 +235,7 @@ def get_scylla_debuginfo_file(self): """ # first try default location scylla_debug_info = '/usr/lib/debug/bin/scylla.debug' - results = self._remoter.run('[[ -f {} ]]'.format(scylla_debug_info), ignore_status=True) + results = self._remoter.run(f'[[ -f {scylla_debug_info} ]]', ignore_status=True) if results.ok: return scylla_debug_info @@ -247,8 +246,8 @@ def get_scylla_debuginfo_file(self): # then look it up base on the build id if build_id := self.get_scylla_build_id(): - scylla_debug_info = "/usr/lib/debug/.build-id/{0}/{1}.debug".format(build_id[:2], build_id[2:]) - results = self._remoter.run('[[ -f {} ]]'.format(scylla_debug_info), ignore_status=True) + scylla_debug_info = f"/usr/lib/debug/.build-id/{build_id[:2]}/{build_id[2:]}.debug" + results = self._remoter.run(f'[[ -f {scylla_debug_info} ]]', ignore_status=True) if results.ok: return scylla_debug_info diff --git a/sdcm/db_stats.py b/sdcm/db_stats.py index 44f7355440a..1cac0d052e3 100644 --- a/sdcm/db_stats.py +++ b/sdcm/db_stats.py @@ -11,31 +11,29 @@ # # Copyright (c) 2020 ScyllaDB -import re import datetime -import time +import json +import logging import os import platform -import logging -import json +import re +import time import urllib.parse - -from textwrap import dedent -from math import sqrt -from typing import Optional -from functools import cached_property from collections import defaultdict +from functools import cached_property +from math import sqrt +from textwrap import dedent -import yaml import requests +import yaml from sdcm.es import ES -from sdcm.test_config import TestConfig -from sdcm.utils.common import normalize_ipv6_url, get_ami_tags -from sdcm.utils.git import get_git_commit_id -from sdcm.utils.decorators import retrying from sdcm.sct_events.system import ElasticsearchEvent +from sdcm.test_config import TestConfig from sdcm.utils.ci_tools import get_job_name, get_job_url +from sdcm.utils.common import get_ami_tags, normalize_ipv6_url +from sdcm.utils.decorators import retrying +from sdcm.utils.git import get_git_commit_id LOGGER = logging.getLogger(__name__) @@ -56,9 +54,9 @@ def __init__(self, cmd, ex): self.exception = repr(ex) def __str__(self): - return dedent(""" - Stress command: '{0.command}' - Error: {0.exception}""".format(self)) + return dedent(f""" + Stress command: '{self.command}' + Error: {self.exception}""") def __repr__(self): return self.__str__() @@ -255,11 +253,10 @@ def query(self, query, start, end, scrap_metrics_step=None): values: [[linux_timestamp1, value1], [linux_timestamp2, value2]...[linux_timestampN, valueN]] } """ - url = "{}://{}:{}/api/v1/query_range?query=".format(self.protocol, normalize_ipv6_url(self.host), self.port) + url = f"{self.protocol}://{normalize_ipv6_url(self.host)}:{self.port}/api/v1/query_range?query=" if not scrap_metrics_step: scrap_metrics_step = self.scylla_scrape_interval - _query = "{url}{query}&start={start}&end={end}&step={scrap_metrics_step}".format( - url=url, query=query, start=start, end=end, scrap_metrics_step=scrap_metrics_step) + _query = f"{url}{query}&start={start}&end={end}&step={scrap_metrics_step}" LOGGER.debug("Query to PrometheusDB: %s", _query) result = self.request(url=_query) if result: @@ -325,8 +322,8 @@ def get_scylla_scheduler_runtime_ms(self, start_time, end_time, node_ip, irate_s if not self._check_start_end_time(start_time, end_time): return {} # the query is taken from the Grafana Dashborad definition - query = 'avg(irate(scylla_scheduler_runtime_ms{group=~"sl:.*", instance="%s"} [%s] )) ' \ - 'by (group, instance)' % (node_ip, irate_sample_sec) + query = 'avg(irate(scylla_scheduler_runtime_ms{{group=~"sl:.*", instance="{}"}} [{}] )) ' \ + 'by (group, instance)'.format(node_ip, irate_sample_sec) results = self.query(query=query, start=start_time, end=end_time) res = defaultdict(dict) for item in results: @@ -382,9 +379,9 @@ def get_latency_write_99(self, start_time, end_time, scrap_metrics_step=None): scrap_metrics_step=scrap_metrics_step) def create_snapshot(self): - url = "http://{}:{}/api/v1/admin/tsdb/snapshot".format(normalize_ipv6_url(self.host), self.port) + url = f"http://{normalize_ipv6_url(self.host)}:{self.port}/api/v1/admin/tsdb/snapshot" result = self.request(url, True) - LOGGER.debug('Request result: {}'.format(result)) + LOGGER.debug(f'Request result: {result}') return result @staticmethod @@ -460,7 +457,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @cached_property - def elasticsearch(self) -> Optional[ES]: + def elasticsearch(self) -> ES | None: try: return ES() except Exception as exc: # pylint: disable=broad-except @@ -498,7 +495,7 @@ def update(self, data: dict) -> None: LOGGER.exception("Failed to update test stats (doc_id=%s)", self._test_id) ElasticsearchEvent(doc_id=self._test_id, error=str(exc)).publish() - def exists(self) -> Optional[bool]: + def exists(self) -> bool | None: if not self.elasticsearch: LOGGER.error("Failed to check for test stats existence: ES connection is not created (doc_id=%s)", self._test_id) @@ -674,18 +671,18 @@ def create_test_stats(self, sub_type=None, specific_tested_stats=None, # pylint if sub_type: self._stats['test_details']['sub_type'] = sub_type if sub_type and append_sub_test_to_name: - self._stats['test_details']['test_name'] = '{}_{}'.format(test_name, sub_type) + self._stats['test_details']['test_name'] = f'{test_name}_{sub_type}' else: self._stats['test_details']['test_name'] = test_name for stat in self.PROMETHEUS_STATS: self._stats['results'][stat] = {} if specific_tested_stats: self._stats['results'].update(specific_tested_stats) - self.log.info("Creating specific tested stats of: {}".format(specific_tested_stats)) + self.log.info(f"Creating specific tested stats of: {specific_tested_stats}") self.create() def update_stress_cmd_details(self, cmd, prefix='', stresser="cassandra-stress", aggregate=True): - section = '{prefix}{stresser}'.format(prefix=prefix, stresser=stresser) + section = f'{prefix}{stresser}' if section not in self._stats['test_details']: self._stats['test_details'][section] = [] if aggregate else {} if stresser == "cassandra-stress": @@ -774,8 +771,8 @@ def _convert_stat(self, stat, stress_result): try: return float(stress_result[stat]) except Exception as details: # pylint: disable=broad-except # noqa: BLE001 - self.log.warning("Error in conversion of '%s' for stat '%s': '%s'" - "Discarding stat." % (stress_result[stat], stat, details)) + self.log.warning("Error in conversion of '{}' for stat '{}': '{}'" + "Discarding stat.".format(stress_result[stat], stat, details)) return 0 def _calc_stat_total(self, stat): @@ -885,7 +882,7 @@ def output(cmd): self.update(update_data) - def get_doc_data(self, key) -> Optional[dict]: + def get_doc_data(self, key) -> dict | None: if self.create_stats and self._test_index and self._test_id: if not self.elasticsearch: LOGGER.error("Failed to get test stats: ES connection is not created (doc_id=%s)", self._test_id) diff --git a/sdcm/ec2_client.py b/sdcm/ec2_client.py index dd625470065..fe6e7a1cf82 100644 --- a/sdcm/ec2_client.py +++ b/sdcm/ec2_client.py @@ -1,17 +1,18 @@ -import logging +import base64 import datetime +import logging import time -import base64 import boto3 +from botocore.exceptions import ClientError, NoRegionError from mypy_boto3_ec2 import EC2Client, EC2ServiceResource from mypy_boto3_ec2.service_resource import Instance -from botocore.exceptions import ClientError, NoRegionError -from sdcm.utils.decorators import retrying -from sdcm.utils.aws_utils import tags_as_ec2_tags from sdcm.test_config import TestConfig +from sdcm.utils.aws_utils import tags_as_ec2_tags from sdcm.utils.common import list_placement_groups_aws +from sdcm.utils.decorators import retrying + LOGGER = logging.getLogger(__name__) STATUS_FULFILLED = 'fulfilled' @@ -47,7 +48,7 @@ class CreateSpotFleetError(ClientError): pass -class EC2ClientWrapper(): +class EC2ClientWrapper: def __init__(self, timeout=REQUEST_TIMEOUT, region_name=None, spot_max_price_percentage=None): self._client = self._get_ec2_client(region_name) @@ -146,7 +147,9 @@ def _get_spot_price(self, instance_type): :return: spot bid price """ LOGGER.info('Calculating spot price based on OnDemand price') - from sdcm.utils.pricing import AWSPricing # pylint: disable=import-outside-toplevel + from sdcm.utils.pricing import ( + AWSPricing, # pylint: disable=import-outside-toplevel + ) aws_pricing = AWSPricing() on_demand_price = float(aws_pricing.get_on_demand_instance_price(self.region_name, instance_type)) @@ -293,7 +296,7 @@ def create_spot_instances(self, instance_type, image_id, region_name, network_if LOGGER.info('Spot instances: %s', instance_ids) for ind, instance_id in enumerate(instance_ids): - self.add_tags(instance_id, {'Name': 'spot_{}_{}'.format(instance_id, ind)}) + self.add_tags(instance_id, {'Name': f'spot_{instance_id}_{ind}'}) self._client.cancel_spot_instance_requests(SpotInstanceRequestIds=request_ids) @@ -332,7 +335,7 @@ def create_spot_fleet(self, instance_type, image_id, region_name, network_if, ke LOGGER.info('Spot instances: %s', instance_ids) for ind, instance_id in enumerate(instance_ids): - self.add_tags(instance_id, {'Name': 'spot_fleet_{}_{}'.format(instance_id, ind)}) + self.add_tags(instance_id, {'Name': f'spot_fleet_{instance_id}_{ind}'}) self._client.cancel_spot_fleet_requests(SpotFleetRequestIds=[request_id], TerminateInstances=False) diff --git a/sdcm/fill_db_data.py b/sdcm/fill_db_data.py index 3f2763183ce..648827a3b41 100644 --- a/sdcm/fill_db_data.py +++ b/sdcm/fill_db_data.py @@ -19,24 +19,21 @@ import contextlib import logging import random -import time import re - +import time from collections import OrderedDict from uuid import UUID -from cassandra import InvalidRequest -from cassandra.util import sortedset, SortedSet # pylint: disable=no-name-in-module -from cassandra import ConsistencyLevel +from cassandra import ConsistencyLevel, InvalidRequest from cassandra.protocol import ProtocolException # pylint: disable=no-name-in-module +from cassandra.util import SortedSet, sortedset # pylint: disable=no-name-in-module from sdcm.tester import ClusterTester +from sdcm.utils.cdc.options import CDC_LOGTABLE_SUFFIX from sdcm.utils.database_query_utils import fetch_all_rows from sdcm.utils.decorators import retrying -from sdcm.utils.cdc.options import CDC_LOGTABLE_SUFFIX from sdcm.utils.version_utils import ComparableScyllaVersion - LOGGER = logging.getLogger(__name__) @@ -323,7 +320,7 @@ class FillDatabaseData(ClusterTester): ) """], 'truncates': ['TRUNCATE limit_ranges_test'], 'inserts': [ - "INSERT INTO limit_ranges_test (userid, url, time) VALUES ({}, 'http://foo.{}', 42)".format(_id, tld) + f"INSERT INTO limit_ranges_test (userid, url, time) VALUES ({_id}, 'http://foo.{tld}', 42)" for _id in range(0, 4) for tld in ['com', 'org', 'net']], 'queries': [ "SELECT * FROM limit_ranges_test WHERE token(userid) >= token(2) LIMIT 1", @@ -344,8 +341,7 @@ class FillDatabaseData(ClusterTester): ) """], 'truncates': ['TRUNCATE limit_multiget_test'], 'inserts': [ - "INSERT INTO limit_multiget_test (userid, url, time) VALUES ({}, 'http://foo.{}', 42)".format(_id, - tld) + f"INSERT INTO limit_multiget_test (userid, url, time) VALUES ({_id}, 'http://foo.{tld}', 42)" for _id in range(0, 100) for tld in ['com', 'org', 'net']], 'queries': [ "SELECT * FROM limit_multiget_test WHERE userid IN (48, 2) LIMIT 1"], @@ -391,8 +387,7 @@ class FillDatabaseData(ClusterTester): )"""], 'truncates': ['TRUNCATE limit_sparse_test'], 'inserts': [ - "INSERT INTO limit_sparse_test (userid, url, day, month, year) VALUES ({}, 'http://foo.{}', 1, 'jan', 2012)".format( - _id, tld) for _id in range(0, 100) for tld in ['com', 'org', 'net']], + f"INSERT INTO limit_sparse_test (userid, url, day, month, year) VALUES ({_id}, 'http://foo.{tld}', 1, 'jan', 2012)" for _id in range(0, 100) for tld in ['com', 'org', 'net']], 'queries': [ "SELECT * FROM limit_sparse_test LIMIT 4"], 'results': [ @@ -772,9 +767,9 @@ class FillDatabaseData(ClusterTester): 'results': [ [[x, x] for x in range(0, 10)], [[x, x] for x in range(9, -1, -1)], - [[x, y, '{}{}'.format(x, y)] for x in range(0, 10) for y in range(9, -1, -1)], - [[x, y, '{}{}'.format(x, y)] for x in range(0, 10) for y in range(9, -1, -1)], - [[x, y, '{}{}'.format(x, y)] for x in range(9, -1, -1) for y in range(0, 10)]], + [[x, y, f'{x}{y}'] for x in range(0, 10) for y in range(9, -1, -1)], + [[x, y, f'{x}{y}'] for x in range(0, 10) for y in range(9, -1, -1)], + [[x, y, f'{x}{y}'] for x in range(9, -1, -1) for y in range(0, 10)]], 'invalid_queries': [ "SELECT c1, c2, v FROM reversed_comparator_test2 WHERE k = 0 ORDER BY c1 ASC, c2 ASC", "SELECT c1, c2, v FROM reversed_comparator_test2 WHERE k = 0 ORDER BY c1 DESC, c2 DESC", @@ -1217,11 +1212,11 @@ class FillDatabaseData(ClusterTester): PRIMARY KEY (fn, ln) )"""], 'truncates': ["TRUNCATE set_test"], - 'inserts': ["UPDATE set_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "tags = tags + { 'foo' }", - "UPDATE set_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "tags = tags + { 'bar' }", - "UPDATE set_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "tags = tags + { 'foo' }", - "UPDATE set_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "tags = tags + { 'foobar' }", - "UPDATE set_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "tags = tags - { 'bar' }"], + 'inserts': ["UPDATE set_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("tags = tags + { 'foo' }"), + "UPDATE set_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("tags = tags + { 'bar' }"), + "UPDATE set_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("tags = tags + { 'foo' }"), + "UPDATE set_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("tags = tags + { 'foobar' }"), + "UPDATE set_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("tags = tags - { 'bar' }")], 'queries': ["SELECT tags FROM set_test", "UPDATE set_test SET {} WHERE fn='Bilbo' AND ln='Baggins'".format( "tags = { 'a', 'c', 'b' }"), @@ -1252,17 +1247,19 @@ class FillDatabaseData(ClusterTester): PRIMARY KEY (fn, ln) )"""], 'truncates': ["TRUNCATE map_test"], - 'inserts': ["UPDATE map_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "m['foo'] = 3", - "UPDATE map_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "m['bar'] = 4", - "UPDATE map_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "m['woot'] = 5", - "UPDATE map_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "m['bar'] = 6", + 'inserts': ["UPDATE map_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("m['foo'] = 3"), + "UPDATE map_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("m['bar'] = 4"), + "UPDATE map_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("m['woot'] = 5"), + "UPDATE map_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("m['bar'] = 6"), "DELETE m['foo'] FROM map_test WHERE fn='Tom' AND ln='Bombadil'"], 'queries': ["SELECT m FROM map_test", - "UPDATE map_test SET %s WHERE fn='Bilbo' AND ln='Baggins'" % "m = { 'a' : 4 , 'c' : 3, 'b' : 2 }", + "UPDATE map_test SET {} WHERE fn='Bilbo' AND ln='Baggins'".format( + "m = { 'a' : 4 , 'c' : 3, 'b' : 2 }"), "SELECT m FROM map_test WHERE fn='Bilbo' AND ln='Baggins'", - "UPDATE map_test SET %s WHERE fn='Bilbo' AND ln='Baggins'" % "m = { 'm' : 4 , 'n' : 1, 'o' : 2 }", + "UPDATE map_test SET {} WHERE fn='Bilbo' AND ln='Baggins'".format( + "m = { 'm' : 4 , 'n' : 1, 'o' : 2 }"), "SELECT m FROM map_test WHERE fn='Bilbo' AND ln='Baggins'", - "UPDATE map_test SET %s WHERE fn='Bilbo' AND ln='Baggins'" % "m = {}", + "UPDATE map_test SET {} WHERE fn='Bilbo' AND ln='Baggins'".format("m = {}"), "SELECT m FROM map_test WHERE fn='Bilbo' AND ln='Baggins'" ], 'results': [[[{'woot': 5, 'bar': 6}]], @@ -1285,20 +1282,23 @@ class FillDatabaseData(ClusterTester): PRIMARY KEY (fn, ln) )"""], 'truncates': ["TRUNCATE list_test"], - 'inserts': ["UPDATE list_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "tags = tags + [ 'foo' ]", - "UPDATE list_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "tags = tags + [ 'bar' ]", - "UPDATE list_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "tags = tags + [ 'foo' ]", - "UPDATE list_test SET %s WHERE fn='Tom' AND ln='Bombadil'" % "tags = tags + [ 'foobar' ]"], + 'inserts': ["UPDATE list_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("tags = tags + [ 'foo' ]"), + "UPDATE list_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("tags = tags + [ 'bar' ]"), + "UPDATE list_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("tags = tags + [ 'foo' ]"), + "UPDATE list_test SET {} WHERE fn='Tom' AND ln='Bombadil'".format("tags = tags + [ 'foobar' ]")], 'queries': ["SELECT tags FROM list_test", - "UPDATE list_test SET %s WHERE fn='Bilbo' AND ln='Baggins'" % "tags = [ 'a', 'c', 'b', 'c' ]", + "UPDATE list_test SET {} WHERE fn='Bilbo' AND ln='Baggins'".format( + "tags = [ 'a', 'c', 'b', 'c' ]"), "SELECT tags FROM list_test WHERE fn='Bilbo' AND ln='Baggins'", - "UPDATE list_test SET %s WHERE fn='Bilbo' AND ln='Baggins'" % "tags = [ 'm', 'n' ] + tags", + "UPDATE list_test SET {} WHERE fn='Bilbo' AND ln='Baggins'".format( + "tags = [ 'm', 'n' ] + tags"), "SELECT tags FROM list_test WHERE fn='Bilbo' AND ln='Baggins'", - "UPDATE list_test SET %s WHERE fn='Bilbo' AND ln='Baggins'" % "tags[2] = 'foo', tags[4] = 'bar'", + "UPDATE list_test SET {} WHERE fn='Bilbo' AND ln='Baggins'".format( + "tags[2] = 'foo', tags[4] = 'bar'"), "SELECT tags FROM list_test WHERE fn='Bilbo' AND ln='Baggins'", "DELETE tags[2] FROM list_test WHERE fn='Bilbo' AND ln='Baggins'", "SELECT tags FROM list_test WHERE fn='Bilbo' AND ln='Baggins'", - "UPDATE list_test SET %s WHERE fn='Bilbo' AND ln='Baggins'" % "tags = tags - [ 'bar' ]", + "UPDATE list_test SET {} WHERE fn='Bilbo' AND ln='Baggins'".format("tags = tags - [ 'bar' ]"), "SELECT tags FROM list_test WHERE fn='Bilbo' AND ln='Baggins'" ], 'results': [[[['foo', 'bar', 'foo', 'foobar']]], @@ -1443,7 +1443,7 @@ class FillDatabaseData(ClusterTester): ) """], 'truncates': ["TRUNCATE only_pk_test1", "TRUNCATE only_pk_test2"], - 'inserts': ["INSERT INTO only_pk_test1 (k, c) VALUES (%s, %s)" % (k, c) for k in range(0, 2) for c in + 'inserts': [f"INSERT INTO only_pk_test1 (k, c) VALUES ({k}, {c})" for k in range(0, 2) for c in range(0, 2)], 'queries': ["#SORTED SELECT * FROM only_pk_test1", "INSERT INTO only_pk_test2(k, c) VALUES(0, 0)", @@ -1466,7 +1466,7 @@ class FillDatabaseData(ClusterTester): 'name': 'no_clustering_test', 'create_tables': ["CREATE TABLE no_clustering_test (k int PRIMARY KEY, v int)"], 'truncates': [], - 'inserts': ["INSERT INTO no_clustering_test (k, v) VALUES (%s, %s)" % (i, i) for i in range(10)], + 'inserts': [f"INSERT INTO no_clustering_test (k, v) VALUES ({i}, {i})" for i in range(10)], 'queries': ["#SORTED SELECT * FROM no_clustering_test"], 'results': [[[i, i] for i in range(10)]], 'disable_paging': True, @@ -1747,7 +1747,7 @@ class FillDatabaseData(ClusterTester): """], 'truncates': ["TRUNCATE reversed_compact_test1", "TRUNCATE reversed_compact_test2"], 'inserts': [ - "INSERT INTO %s(k, c, v) VALUES ('foo', %s, %s)" % (k, i, i) for i in range(0, 10) for k in + f"INSERT INTO {k}(k, c, v) VALUES ('foo', {i}, {i})" for i in range(0, 10) for k in ['reversed_compact_test1', 'reversed_compact_test2'] ], 'queries': ["SELECT c FROM reversed_compact_test1 WHERE c > 2 AND c < 6 AND k = 'foo'", @@ -2027,7 +2027,7 @@ class FillDatabaseData(ClusterTester): range(0, 30)] + [ f"DELETE FROM range_with_deletes_test WHERE k = {i}" for i in random.sample(range(30), 5)], - 'queries': ["#LENGTH SELECT * FROM range_with_deletes_test LIMIT {}".format(15)], + 'queries': [f"#LENGTH SELECT * FROM range_with_deletes_test LIMIT {15}"], 'results': [15], 'min_version': '', 'max_version': '', @@ -2077,7 +2077,8 @@ class FillDatabaseData(ClusterTester): )"""], 'truncates': ["TRUNCATE multi_in_test"], 'inserts': [ - "INSERT INTO multi_in_test (group, zipcode, state, fips_regions, city) VALUES ('%s', '%s', '%s', %s, '%s')" % d + "INSERT INTO multi_in_test (group, zipcode, state, fips_regions, city) VALUES ('{}', '{}', '{}', {}, '{}')".format( + *d) for d in [ ('test', '06029', 'CT', 9, 'Ellington'), ('test', '06031', 'CT', 9, 'Falls Village'), @@ -2140,7 +2141,8 @@ class FillDatabaseData(ClusterTester): ) """], 'truncates': ["TRUNCATE multi_in_compact_test"], 'inserts': [ - "INSERT INTO multi_in_compact_test (group, zipcode, state, fips_regions, city) VALUES ('%s', '%s', '%s', %s, '%s')" % d + "INSERT INTO multi_in_compact_test (group, zipcode, state, fips_regions, city) VALUES ('{}', '{}', '{}', {}, '{}')".format( + *d) for d in [ ('test', '06029', 'CT', 9, 'Ellington'), ('test', '06031', 'CT', 9, 'Falls Village'), @@ -2617,11 +2619,9 @@ class FillDatabaseData(ClusterTester): 'create_tables': ["CREATE TABLE cas_simple_test (tkn int, consumed boolean, PRIMARY KEY (tkn))"], 'truncates': ["TRUNCATE cas_simple_test"], 'inserts': [], - 'queries': [[["INSERT INTO cas_simple_test (tkn, consumed) VALUES ({},FALSE);".format(k), - "UPDATE cas_simple_test SET consumed = TRUE WHERE tkn = {} IF consumed = FALSE;".format( - k), - "UPDATE cas_simple_test SET consumed = TRUE WHERE tkn = {} IF consumed = FALSE;".format( - k).format(i)] for k in range(1, 10)][j][i] for j in range(0, 9) for i in + 'queries': [[[f"INSERT INTO cas_simple_test (tkn, consumed) VALUES ({k},FALSE);", + f"UPDATE cas_simple_test SET consumed = TRUE WHERE tkn = {k} IF consumed = FALSE;", + f"UPDATE cas_simple_test SET consumed = TRUE WHERE tkn = {k} IF consumed = FALSE;".format(i)] for k in range(1, 10)][j][i] for j in range(0, 9) for i in range(0, 3)], 'results': [[], [[True]], [[False, True]]] * 3, 'min_version': '', diff --git a/sdcm/gemini_thread.py b/sdcm/gemini_thread.py index 7819cc8d44c..5fd5906ff62 100644 --- a/sdcm/gemini_thread.py +++ b/sdcm/gemini_thread.py @@ -11,20 +11,19 @@ # # Copyright (c) 2020 ScyllaDB +import json import logging import os -import uuid import random -import json import time +import uuid from sdcm.sct_events import Severity -from sdcm.utils.common import FileFollowerThread from sdcm.sct_events.loaders import GeminiStressEvent, GeminiStressLogEvent from sdcm.stress_thread import DockerBasedStressThread +from sdcm.utils.common import FileFollowerThread from sdcm.utils.docker_remote import RemoteDocker - LOGGER = logging.getLogger(__name__) @@ -74,7 +73,7 @@ def __init__(self, test_cluster, oracle_cluster, loaders, stress_cmd, timeout=No @property def gemini_result_file(self): if not self._gemini_result_file: - self._gemini_result_file = os.path.join("/", "gemini_result_{}.log".format(uuid.uuid4())) + self._gemini_result_file = os.path.join("/", f"gemini_result_{uuid.uuid4()}.log") return self._gemini_result_file def _generate_gemini_command(self): @@ -93,7 +92,7 @@ def _generate_gemini_command(self): self.gemini_request_timeout, self.gemini_connect_timeout) if oracle_nodes: - cmd += "--oracle-cluster={} ".format(oracle_nodes) + cmd += f"--oracle-cluster={oracle_nodes} " if table_options: cmd += " ".join([f"--table-options \"{table_opt}\"" for table_opt in table_options]) self.gemini_commands.append(cmd) @@ -111,8 +110,7 @@ def _run_stress(self, loader, loader_idx, cpu_idx): if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) - log_file_name = os.path.join(loader.logdir, 'gemini-l%s-c%s-%s.log' % - (loader_idx, cpu_idx, uuid.uuid4())) + log_file_name = os.path.join(loader.logdir, f'gemini-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log') LOGGER.debug('gemini local log: %s', log_file_name) gemini_cmd = self._generate_gemini_command() @@ -173,7 +171,7 @@ def verify_gemini_results(results): stats['results'].append(res) for err_type in ['write_errors', 'read_errors', 'errors']: if res.get(err_type, None): - LOGGER.error("Gemini {} errors: {}".format(err_type, res[err_type])) + LOGGER.error(f"Gemini {err_type} errors: {res[err_type]}") stats['status'] = 'FAILED' stats['errors'][err_type] = res[err_type] if not stats.get('status'): @@ -188,7 +186,7 @@ def _parse_gemini_summary_json(json_str): results = json.loads(json_str) except Exception as details: # pylint: disable=broad-except # noqa: BLE001 - LOGGER.error("Invalid json document {}".format(details)) + LOGGER.error(f"Invalid json document {details}") return results.get('result') diff --git a/sdcm/kcl_thread.py b/sdcm/kcl_thread.py index 2cb670dc7a2..ac7b2e4747f 100644 --- a/sdcm/kcl_thread.py +++ b/sdcm/kcl_thread.py @@ -11,23 +11,20 @@ # # Copyright (c) 2020 ScyllaDB +import logging import os -import time import random -import logging -import uuid import threading - +import time +import uuid from functools import cached_property -from typing import Dict -from sdcm.stress_thread import DockerBasedStressThread +from sdcm.cluster import BaseNode +from sdcm.sct_events.loaders import KclStressEvent +from sdcm.sct_events.system import InfoEvent from sdcm.stress.base import format_stress_cmd_error +from sdcm.stress_thread import DockerBasedStressThread from sdcm.utils.docker_remote import RemoteDocker -from sdcm.sct_events.system import InfoEvent -from sdcm.sct_events.loaders import KclStressEvent -from sdcm.cluster import BaseNode - LOGGER = logging.getLogger(__name__) @@ -58,18 +55,17 @@ def _run_stress(self, loader, loader_idx, cpu_idx): if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) - log_file_name = os.path.join(loader.logdir, 'kcl-l%s-c%s-%s.log' % - (loader_idx, cpu_idx, uuid.uuid4())) + log_file_name = os.path.join(loader.logdir, f'kcl-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log') LOGGER.debug('kcl-stress local log: %s', log_file_name) LOGGER.debug("'running: %s", stress_cmd) if self.stress_num > 1: - node_cmd = 'taskset -c %s bash -c "%s"' % (cpu_idx, stress_cmd) + node_cmd = f'taskset -c {cpu_idx} bash -c "{stress_cmd}"' else: node_cmd = stress_cmd - node_cmd = 'cd /hydra-kcl && {}'.format(node_cmd) + node_cmd = f'cd /hydra-kcl && {node_cmd}' KclStressEvent.start(node=loader, stress_cmd=stress_cmd).publish() @@ -113,7 +109,7 @@ def db_node_to_query(self, loader): return node_to_query @cached_property - def _options(self) -> Dict[str, str]: + def _options(self) -> dict[str, str]: return dict(item.strip().split("=") for item in self.stress_cmd.replace('table_compare', '').strip().split(";")) @property diff --git a/sdcm/keystore.py b/sdcm/keystore.py index f0c97f93aab..25bd4f25e14 100644 --- a/sdcm/keystore.py +++ b/sdcm/keystore.py @@ -11,8 +11,8 @@ # # Copyright (c) 2020 ScyllaDB -import os import json +import os from collections import namedtuple import boto3 diff --git a/sdcm/loader.py b/sdcm/loader.py index 21cf5bed72b..651c99c0ce1 100644 --- a/sdcm/loader.py +++ b/sdcm/loader.py @@ -11,16 +11,20 @@ # # Copyright (c) 2016 ScyllaDB +import logging import os import re -from abc import abstractmethod, ABCMeta import time -import logging +from abc import ABCMeta, abstractmethod from typing import NamedTuple from sdcm.prometheus import NemesisMetrics from sdcm.utils.common import FileFollowerThread, convert_metric_to_ms -from sdcm.utils.csrangehistogram import CSHistogramTags, CSWorkloadTypes, make_cs_range_histogram_summary_from_log_line +from sdcm.utils.csrangehistogram import ( + CSHistogramTags, + CSWorkloadTypes, + make_cs_range_histogram_summary_from_log_line, +) LOGGER = logging.getLogger(__name__) diff --git a/sdcm/localhost.py b/sdcm/localhost.py index 338cfc91b12..1150d8f9c16 100644 --- a/sdcm/localhost.py +++ b/sdcm/localhost.py @@ -11,28 +11,27 @@ # # Copyright (c) 2020 ScyllaDB -import os import logging -from typing import Optional +import os from sdcm.utils.docker_utils import ContainerManager -from sdcm.utils.k8s import HelmContainerMixin -from sdcm.utils.rsyslog import RSyslogContainerMixin from sdcm.utils.gce_utils import GcloudContainerMixin +from sdcm.utils.k8s import HelmContainerMixin from sdcm.utils.ldap import LDAP_PORT, LDAP_SSL_PORT, LdapContainerMixin +from sdcm.utils.rsyslog import RSyslogContainerMixin from sdcm.utils.syslogng import SyslogNGContainerMixin LOGGER = logging.getLogger(__name__) class LocalHost(SyslogNGContainerMixin, RSyslogContainerMixin, GcloudContainerMixin, HelmContainerMixin, LdapContainerMixin): - def __init__(self, user_prefix: Optional[str] = None, test_id: Optional[str] = None) -> None: + def __init__(self, user_prefix: str | None = None, test_id: str | None = None) -> None: self._containers = {} self.tags = {} self.name = (f"{user_prefix}-" if user_prefix else "") + "localhost" + (f"-{test_id}" if test_id else "") @property - def ldap_ports(self) -> Optional[dict]: + def ldap_ports(self) -> dict | None: return {'ldap_port': ContainerManager.get_container_port(self, "ldap", LDAP_PORT), 'ldap_ssl_port': ContainerManager.get_container_port(self, "ldap", LDAP_SSL_PORT)} diff --git a/sdcm/log.py b/sdcm/log.py index c9090df9382..1c2ef9ad941 100644 --- a/sdcm/log.py +++ b/sdcm/log.py @@ -21,4 +21,4 @@ class SDCMAdapter(logging.LoggerAdapter): """ def process(self, msg, kwargs): - return '%s: %s' % (self.extra['prefix'], msg), kwargs + return '{}: {}'.format(self.extra['prefix'], msg), kwargs diff --git a/sdcm/logcollector.py b/sdcm/logcollector.py index aa6fcb2a113..65d1640a207 100644 --- a/sdcm/logcollector.py +++ b/sdcm/logcollector.py @@ -12,56 +12,60 @@ # Copyright (c) 2020 ScyllaDB # pylint: disable=too-many-lines -import os +import datetime +import fnmatch import json +import logging +import os import re -import time import shutil -import fnmatch -import logging -import datetime import tarfile import tempfile +import time import traceback from collections import OrderedDict -from typing import Optional, Tuple, List -from pathlib import Path from functools import cached_property +from pathlib import Path import requests import sdcm.monitorstack.ui as monitoring_ui -from sdcm.paths import SCYLLA_YAML_PATH, SCYLLA_PROPERTIES_PATH, SCYLLA_MANAGER_AGENT_YAML_PATH, \ - SCYLLA_MANAGER_YAML_PATH +from sdcm.db_stats import PrometheusDBStats +from sdcm.paths import ( + SCYLLA_MANAGER_AGENT_YAML_PATH, + SCYLLA_MANAGER_YAML_PATH, + SCYLLA_PROPERTIES_PATH, + SCYLLA_YAML_PATH, +) from sdcm.provision import provisioner_factory from sdcm.provision.provisioner import ProvisionerError -from sdcm.remote import RemoteCmdRunnerBase, LocalCmdRunner -from sdcm.db_stats import PrometheusDBStats +from sdcm.remote import LocalCmdRunner, RemoteCmdRunnerBase from sdcm.sct_events.events_device import EVENTS_LOG_DIR, RAW_EVENTS_LOG from sdcm.test_config import TestConfig +from sdcm.utils.auto_ssh import AutoSshContainerMixin from sdcm.utils.common import ( - S3Storage, ParallelObject, + S3Storage, + create_remote_storage_dir, + filter_aws_instances_by_type, + filter_gce_instances_by_type, + get_builder_by_test_id, + get_sct_root_path, + get_testrun_dir, list_instances_aws, list_instances_gce, + normalize_ipv6_url, remove_files, - get_builder_by_test_id, - get_testrun_dir, search_test_id_in_latest, - filter_aws_instances_by_type, - filter_gce_instances_by_type, - get_sct_root_path, - normalize_ipv6_url, create_remote_storage_dir, ) -from sdcm.utils.auto_ssh import AutoSshContainerMixin from sdcm.utils.context_managers import environment from sdcm.utils.decorators import retrying from sdcm.utils.docker_utils import get_docker_bridge_gateway +from sdcm.utils.gce_utils import gce_private_addresses, gce_public_addresses from sdcm.utils.get_username import get_username from sdcm.utils.k8s import KubernetesOps from sdcm.utils.remotewebbrowser import RemoteBrowser, WebDriverContainerMixin from sdcm.utils.s3_remote_uploader import upload_remote_files_directly_to_s3 -from sdcm.utils.gce_utils import gce_public_addresses, gce_private_addresses LOGGER = logging.getLogger(__name__) @@ -125,18 +129,28 @@ def get_monitoring_base_dir(self, node): @staticmethod def get_monitoring_stack(backend): if backend == 'aws': - from sdcm.cluster_aws import MonitorSetAWS # pylint: disable=import-outside-toplevel + from sdcm.cluster_aws import ( + MonitorSetAWS, # pylint: disable=import-outside-toplevel + ) return MonitorSetAWS elif backend == 'docker': - from sdcm.cluster_docker import MonitorSetDocker # pylint: disable=import-outside-toplevel + from sdcm.cluster_docker import ( + MonitorSetDocker, # pylint: disable=import-outside-toplevel + ) return MonitorSetDocker elif backend == 'gce': - from sdcm.cluster_gce import MonitorSetGCE # pylint: disable=import-outside-toplevel + from sdcm.cluster_gce import ( + MonitorSetGCE, # pylint: disable=import-outside-toplevel + ) return MonitorSetGCE elif backend == 'baremetal': - from sdcm.cluster_baremetal import MonitorSetPhysical # pylint: disable=import-outside-toplevel + from sdcm.cluster_baremetal import ( + MonitorSetPhysical, # pylint: disable=import-outside-toplevel + ) return MonitorSetPhysical - from sdcm.cluster import BaseMonitorSet # pylint: disable=import-outside-toplevel + from sdcm.cluster import ( + BaseMonitorSet, # pylint: disable=import-outside-toplevel + ) return BaseMonitorSet def get_monitoring_version(self, node): @@ -172,7 +186,7 @@ class CommandLog(BaseLogEntity): # pylint: disable=too-few-public-methods BaseLogEntity """ - def collect(self, node, local_dst, remote_dst=None, local_search_path=None) -> Optional[str]: + def collect(self, node, local_dst, remote_dst=None, local_search_path=None) -> str | None: if not node or not node.remoter or remote_dst is None: return None remote_logfile = LogCollector.collect_log_remotely(node=node, @@ -203,7 +217,7 @@ def find_local_files(search_in_dir, search_pattern, except_patterns="collected_l full_path = os.path.join(root, f) if except_patterns in full_path: continue - if full_path.endswith(search_pattern) or fnmatch.fnmatch(full_path, "*{}".format(search_pattern)): + if full_path.endswith(search_pattern) or fnmatch.fnmatch(full_path, f"*{search_pattern}"): if os.path.islink(full_path): full_path = os.path.realpath(full_path) local_files.append(full_path) @@ -222,7 +236,7 @@ def find_on_builder(builder, file_name, search_in_dir="/"): # pylint: disable=u def _is_file_collected(self, local_dst): for collected_file in os.listdir(local_dst): - if self.name in collected_file or fnmatch.fnmatch(collected_file, "*{}".format(self.name)): + if self.name in collected_file or fnmatch.fnmatch(collected_file, f"*{self.name}"): return True return False @@ -307,7 +321,7 @@ def create_prometheus_snapshot(self, node): else: raise PrometheusSnapshotErrorException(result) - def get_prometheus_snapshot_remote(self, node) -> Optional[str]: + def get_prometheus_snapshot_remote(self, node) -> str | None: try: snapshot_dir = self.create_prometheus_snapshot(node) except (PrometheusSnapshotErrorException, Exception) as details: # pylint: disable=broad-except @@ -331,7 +345,7 @@ def setup_monitor_data_dir(self, node): base_dir = self.get_monitoring_base_dir(node) self.monitoring_data_dir = os.path.join(base_dir, self.monitoring_data_dir_name) - def collect(self, node, local_dst, remote_dst=None, local_search_path=None) -> Optional[str]: + def collect(self, node, local_dst, remote_dst=None, local_search_path=None) -> str | None: self.setup_monitor_data_dir(node) if remote_snapshot_archive := self.get_prometheus_snapshot_remote(node): LogCollector.receive_log(node, @@ -404,7 +418,7 @@ def search_dashboard(grafana_ip: str, port: int, query: str) -> list: return resp.json() @staticmethod - def get_dashboard_by_title(grafana_ip: str, port: int, title: str) -> Optional[dict]: + def get_dashboard_by_title(grafana_ip: str, port: int, title: str) -> dict | None: dashboards = MonitoringStack.search_dashboard(grafana_ip, port, title) if not dashboards: LOGGER.error("Dashboard with title '%s' was not found", title) @@ -506,10 +520,10 @@ def get_grafana_screenshot(self, node, local_dst): path=dashboard_metadata["url"], st=self.start_time) screenshot_path = os.path.join(local_dst, - "%s-%s-%s-%s.png" % (self.name, - dashboard.name, - datetime.datetime.now().strftime("%Y%m%d_%H%M%S"), - node.name)) + "{}-{}-{}-{}.png".format(self.name, + dashboard.name, + datetime.datetime.now().strftime("%Y%m%d_%H%M%S"), + node.name)) self.remote_browser.open(grafana_url, dashboard.resolution) dashboard.scroll_to_bottom(self.remote_browser.browser) dashboard.wait_panels_loading(self.remote_browser.browser) @@ -639,12 +653,12 @@ def __init__(self, nodes, test_id, storage_dir, params): def create_local_storage_dir(self, base_local_dir): local_dir = os.path.join(base_local_dir, self.current_run, - "{}-{}".format(self.cluster_log_type, self.test_id[:8])) + f"{self.cluster_log_type}-{self.test_id[:8]}") try: os.makedirs(local_dir, exist_ok=True) except OSError as details: if not os.path.exists(local_dir): - LOGGER.error("Folder is not created. {}".format(details)) + LOGGER.error(f"Folder is not created. {details}") raise return local_dir @@ -653,11 +667,11 @@ def create_remote_storage_dir(self, node, path=''): path = node.name try: remote_dir = os.path.join(self.node_remote_dir, path) - result = node.remoter.run('mkdir -p {}'.format(remote_dir), ignore_status=True) + result = node.remoter.run(f'mkdir -p {remote_dir}', ignore_status=True) if result.exited > 0: LOGGER.error( - 'Remote storing folder not created.\n{}'.format(result)) + f'Remote storing folder not created.\n{result}') remote_dir = self.node_remote_dir except Exception as details: # pylint: disable=broad-except # noqa: BLE001 @@ -667,7 +681,7 @@ def create_remote_storage_dir(self, node, path=''): return remote_dir @staticmethod - def collect_log_remotely(node, cmd: str, log_filename: str) -> Optional[str]: + def collect_log_remotely(node, cmd: str, log_filename: str) -> str | None: if not node.remoter: return None collect_log_command = f"{cmd} >& '{log_filename}'" @@ -676,7 +690,7 @@ def collect_log_remotely(node, cmd: str, log_filename: str) -> Optional[str]: return log_filename if result.ok else None @staticmethod - def archive_log_remotely(node, log_filename: str, archive_name: Optional[str] = None) -> Optional[str]: + def archive_log_remotely(node, log_filename: str, archive_name: str | None = None) -> str | None: if not node.remoter: return None archive_dir, log_filename = os.path.split(log_filename) @@ -697,7 +711,7 @@ def receive_log(node, remote_log_path, local_dir, timeout=300): timeout=timeout) return local_dir - def collect_logs(self, local_search_path: Optional[str] = None) -> list[str]: + def collect_logs(self, local_search_path: str | None = None) -> list[str]: def collect_logs_per_node(node): LOGGER.info('Collecting logs on host: %s', node.name) remote_node_dir = self.create_remote_storage_dir(node) @@ -863,7 +877,7 @@ def collect_diagnostic_data(node): collect_log_entities(node, log_entities) -def collect_log_entities(node, log_entities: List[BaseLogEntity]): +def collect_log_entities(node, log_entities: list[BaseLogEntity]): """Collects diagnostics data from node - used during the test. Log Entities should have unique names, otherwise won't be collected.""" @@ -1016,7 +1030,7 @@ class BaseSCTLogCollector(LogCollector): cluster_dir_prefix = 'sct-runner-events' too_big_log_size = 3*1024*1024*1024 - def collect_logs(self, local_search_path: Optional[str] = None) -> list[str]: + def collect_logs(self, local_search_path: str | None = None) -> list[str]: for ent in self.log_entities: ent.collect(None, self.local_dir, None, local_search_path=local_search_path) if not os.listdir(self.local_dir): @@ -1134,7 +1148,7 @@ class KubernetesAPIServerLogCollector(BaseSCTLogCollector): cluster_dir_prefix = "k8s-" collect_timeout = 600 - def collect_logs(self, local_search_path: Optional[str] = None) -> list[str]: + def collect_logs(self, local_search_path: str | None = None) -> list[str]: logfiles, apiserver_logdir = [], "" for ent in self.log_entities: output = ent.collect(None, self.local_dir, None, local_search_path=local_search_path) @@ -1171,7 +1185,7 @@ def generate_apiserver_call_stats_file(self, apiserver_logdir: str) -> None: # for log_file in os.listdir(apiserver_logdir): if not log_file.startswith(self.audit_file_name_prefix): continue - with open(f"{apiserver_logdir}/{log_file}", mode="r", encoding="utf-8") as current_log_file: + with open(f"{apiserver_logdir}/{log_file}", encoding="utf-8") as current_log_file: for line in current_log_file.readlines(): data = json.loads(line) username = data.get("user", {}).get('username') @@ -1223,7 +1237,7 @@ def _find_test_run_subdir_by_test_id(self, base_logdir) -> str: sub_test_id_file = os.path.join(base_logdir, sub_dir, "test_id") if not os.path.isfile(sub_test_id_file): continue - with open(sub_test_id_file, mode='r', encoding="utf8") as test_id_file: + with open(sub_test_id_file, encoding="utf8") as test_id_file: if test_id_file.read().strip() == self.test_id: return os.path.join(base_logdir, sub_dir) return "" @@ -1238,7 +1252,7 @@ def _find_k8s_subdirs(self, test_run_logdir) -> str: k8s_subdirs.append(os.path.join(test_run_logdir, sub_dir)) return k8s_subdirs - def collect_logs(self, local_search_path: Optional[str] = None) -> list[str]: + def collect_logs(self, local_search_path: str | None = None) -> list[str]: try: base_logdir = TestConfig.base_logdir() test_run_logdir = self._find_test_run_subdir_by_test_id(base_logdir) @@ -1271,7 +1285,7 @@ class KubernetesMustGatherLogCollector(BaseSCTLogCollector): cluster_dir_prefix = "k8s-" collect_timeout = 600 - def collect_logs(self, local_search_path: Optional[str] = None) -> list[str]: + def collect_logs(self, local_search_path: str | None = None) -> list[str]: return super().collect_logs(local_search_path=local_search_path) @@ -1279,7 +1293,7 @@ class JepsenLogCollector(LogCollector): cluster_log_type = "jepsen-data" cluster_dir_prefix = "jepsen-data" - def collect_logs(self, local_search_path: Optional[str] = None) -> list[str]: + def collect_logs(self, local_search_path: str | None = None) -> list[str]: s3_link = [] if self.nodes: jepsen_node = self.nodes[0] @@ -1317,15 +1331,15 @@ class SSTablesCollector(BaseSCTLogCollector): cluster_dir_prefix = "corrupted-sstables" sstable_path_regexp = re.compile(r'[./\w\-]+\.db') - def get_sstable_details(self, error_msg: str) -> Tuple[str, str, str, str]: + def get_sstable_details(self, error_msg: str) -> tuple[str, str, str, str]: sstable_path = self.sstable_path_regexp.findall(error_msg)[0] data_path, keyspace, table_dir, sstable_name = sstable_path.rsplit("/", 3) return f"{data_path}/{keyspace}/{table_dir}", keyspace, table_dir.split("-")[0], sstable_name - def collect_logs(self, local_search_path: Optional[str] = None) -> list[str]: # pylint: disable=too-many-locals + def collect_logs(self, local_search_path: str | None = None) -> list[str]: # pylint: disable=too-many-locals try: raw_events_file_path = Path(self.local_dir).parent.parent.parent / EVENTS_LOG_DIR / RAW_EVENTS_LOG - with open(raw_events_file_path, "r", encoding="utf-8") as events_file: + with open(raw_events_file_path, encoding="utf-8") as events_file: for raw_line in events_file.readlines(): event = json.loads(raw_line) if event.get("type") == "CORRUPTED_SSTABLE": @@ -1440,7 +1454,9 @@ def define_test_id(self): def find_and_append_cloud_manager_instance_to_collecting_nodes(self): try: - from cluster_cloud import get_manager_instance_by_cluster_id # pylint: disable=import-outside-toplevel + from cluster_cloud import ( + get_manager_instance_by_cluster_id, # pylint: disable=import-outside-toplevel + ) except ImportError: LOGGER.error("Couldn't collect Siren manager logs, cluster_cloud module isn't installed") return @@ -1704,7 +1720,7 @@ def check_archive(remoter, path: str) -> bool: return archive_is_ok -def upload_archive_to_s3(archive_path: str, storing_path: str) -> Optional[str]: +def upload_archive_to_s3(archive_path: str, storing_path: str) -> str | None: if not check_archive(LocalCmdRunner(), archive_path): LOGGER.error("File `%s' will not be uploaded", archive_path) return None diff --git a/sdcm/mgmt/__init__.py b/sdcm/mgmt/__init__.py index 4cc5b874fc2..52f1c5a23fd 100644 --- a/sdcm/mgmt/__init__.py +++ b/sdcm/mgmt/__init__.py @@ -1,12 +1,9 @@ -from typing import Union +from .cli import ManagerCluster, ScyllaManagerToolNonRedhat, ScyllaManagerToolRedhatLike +from .common import HostRestStatus, HostSsl, HostStatus, ScyllaManagerError, TaskStatus +from .operator import OperatorManagerCluster, ScyllaManagerToolOperator -from .common import ScyllaManagerError, TaskStatus, HostStatus, HostSsl, HostRestStatus -from .cli import ScyllaManagerToolRedhatLike, ScyllaManagerToolNonRedhat, ManagerCluster -from .operator import ScyllaManagerToolOperator, OperatorManagerCluster - - -AnyManagerTool = Union[ScyllaManagerToolOperator, ScyllaManagerToolRedhatLike, ScyllaManagerToolNonRedhat] -AnyManagerCluster = Union[OperatorManagerCluster, ManagerCluster] +AnyManagerTool = ScyllaManagerToolOperator | ScyllaManagerToolRedhatLike | ScyllaManagerToolNonRedhat +AnyManagerCluster = OperatorManagerCluster | ManagerCluster def get_scylla_manager_tool(manager_node, scylla_cluster=None) -> AnyManagerTool: diff --git a/sdcm/mgmt/cli.py b/sdcm/mgmt/cli.py index 4893889b68e..4f8a945017d 100644 --- a/sdcm/mgmt/cli.py +++ b/sdcm/mgmt/cli.py @@ -12,23 +12,30 @@ # Copyright (c) 2021 ScyllaDB # pylint: disable=too-many-lines +import datetime import json -import time import logging -import datetime -from re import findall -from textwrap import dedent -from statistics import mean +import time from contextlib import contextmanager from distutils.version import LooseVersion +from re import findall +from statistics import mean +from textwrap import dedent import requests from invoke.exceptions import Failure as InvokeFailure -from sdcm.remote.libssh2_client.exceptions import Failure as Libssh2Failure from sdcm import wait -from sdcm.mgmt.common import \ - TaskStatus, ScyllaManagerError, HostStatus, HostSsl, HostRestStatus, duration_to_timedelta, DEFAULT_TASK_TIMEOUT +from sdcm.mgmt.common import ( + DEFAULT_TASK_TIMEOUT, + HostRestStatus, + HostSsl, + HostStatus, + ScyllaManagerError, + TaskStatus, + duration_to_timedelta, +) +from sdcm.remote.libssh2_client.exceptions import Failure as Libssh2Failure from sdcm.utils.distro import Distro from sdcm.wait import WaitForTimeoutError @@ -70,17 +77,17 @@ def get_property(self, parsed_table, column_name): def stop(self): if self.sctool.is_v3_cli: - cmd = "stop {} -c {}".format(self.id, self.cluster_id) + cmd = f"stop {self.id} -c {self.cluster_id}" else: - cmd = "task stop {} -c {}".format(self.id, self.cluster_id) + cmd = f"task stop {self.id} -c {self.cluster_id}" self.sctool.run(cmd=cmd, is_verify_errorless_result=True) return self.wait_and_get_final_status(timeout=30, step=3) def start(self, continue_task=True): if self.sctool.is_v3_cli: - cmd = "start {} -c {}".format(self.id, self.cluster_id) + cmd = f"start {self.id} -c {self.cluster_id}" else: - cmd = "task start {} -c {}".format(self.id, self.cluster_id) + cmd = f"task start {self.id} -c {self.cluster_id}" if not continue_task: cmd += " --no-continue" self.sctool.run(cmd=cmd, is_verify_errorless_result=True) @@ -88,7 +95,7 @@ def start(self, continue_task=True): @staticmethod def _add_kwargs_to_cmd(cmd, **kwargs): for key, value in kwargs.items(): - cmd += ' --{}={}'.format(key, value) + cmd += f' --{key}={value}' return cmd def get_task_info_dict(self): @@ -106,12 +113,12 @@ def get_task_info_dict(self): # │ 13814000-1dd2-11b2-a009-02c33d089f9b │ 07 Jan 23 23:08:59 UTC │ 0s │ DONE │ # ╰──────────────────────────────────────┴────────────────────────┴──────────┴────────╯ info_dict = {} - cmd = "info {} -c {}".format(self.id, self.cluster_id) + cmd = f"info {self.id} -c {self.cluster_id}" res = self.sctool.run(cmd=cmd, is_verify_errorless_result=True) info_lines = [line[0] for line in res if len(line) == 1] for line in info_lines: if ":" in line: - name, value = [string.strip() for string in line.split(":", maxsplit=1)] + name, value = (string.strip() for string in line.split(":", maxsplit=1)) if name.startswith("-"): name = name[2:] info_dict[name] = value @@ -166,7 +173,7 @@ def history(self): # ╰──────────────────────────────────────┴────────────────────────┴──────────┴────────╯ if self.sctool.is_v3_cli: return self.get_task_info_dict()["history"] - cmd = "task history {} -c {}".format(self.id, self.cluster_id) + cmd = f"task history {self.id} -c {self.cluster_id}" res = self.sctool.run(cmd=cmd, is_verify_errorless_result=True) return res # or can be specified like: self.get_property(parsed_table=res, column_name='status') @@ -182,9 +189,9 @@ def next_run(self): # │ repair/22b68423-4332-443d-b8b4-713005ea6049 │ 19 Nov 18 00:00:00 UTC (+7d) │ 3 │ │ NEW │ # ╰──────────────────────────────────────────────────┴───────────────────────────────┴──────┴────────────┴────────╯ if self.sctool.is_v3_cli: - cmd = "tasks -c {}".format(self.cluster_id) + cmd = f"tasks -c {self.cluster_id}" else: - cmd = "task list -c {}".format(self.cluster_id) + cmd = f"task list -c {self.cluster_id}" res = self.sctool.run(cmd=cmd, is_verify_errorless_result=True) if self.sctool.is_v3_cli: return self.get_property(parsed_table=res, column_name='Next') @@ -219,9 +226,9 @@ def status(self) -> str: Gets the task's status """ if self.sctool.is_v3_cli: - cmd = "tasks -c {}".format(self.cluster_id) + cmd = f"tasks -c {self.cluster_id}" else: - cmd = "task list -c {}".format(self.cluster_id) + cmd = f"task list -c {self.cluster_id}" # expecting output of: # ╭─────────────────────────────────────────────┬───────────────────────────────┬──────┬────────────┬────────╮ # │ task │ next run │ ret. │ properties │ status │ @@ -394,7 +401,7 @@ def is_status_in_list(self, list_status, check_task_progress=False): return self.status in list_status def wait_for_status(self, list_status, check_task_progress=True, timeout=3600, step=120): - text = "Waiting until task: {} reaches status of: {}".format(self.id, list_status) + text = f"Waiting until task: {self.id} reaches status of: {list_status}" try: return wait.wait_for(func=self.is_status_in_list, step=step, throw_exc=True, text=text, list_status=list_status, check_task_progress=check_task_progress, @@ -434,10 +441,10 @@ def wait_and_get_final_status(self, timeout=DEFAULT_TASK_TIMEOUT, step=60, only_ else: list_final_status = [ TaskStatus.ERROR, TaskStatus.ERROR_FINAL, TaskStatus.STOPPED, TaskStatus.DONE, TaskStatus.ABORTED] - LOGGER.debug("Waiting for task: {} getting to a final status ({})..".format(self.id, str(list_final_status))) + LOGGER.debug(f"Waiting for task: {self.id} getting to a final status ({str(list_final_status)})..") res = self.wait_for_status(list_status=list_final_status, timeout=timeout, step=step) if not res: - raise ScyllaManagerError("Unexpected result on waiting for task {} status".format(self.id)) + raise ScyllaManagerError(f"Unexpected result on waiting for task {self.id} status") return self.status @@ -484,7 +491,7 @@ def is_task_in_uploading_stage(self): return "uploading data" in full_progress_string.lower() def wait_for_uploading_stage(self, timeout=1440, step=10): - text = "Waiting until backup task: {} starts to upload snapshots".format(self.id) + text = f"Waiting until backup task: {self.id} starts to upload snapshots" is_status_reached = wait.wait_for(func=self.is_task_in_uploading_stage, step=step, text=text, timeout=timeout, throw_exc=True) return is_status_reached @@ -523,7 +530,7 @@ def get_cluster_id_by_name(self, cluster_name: str): return self.sctool.get_table_value( parsed_table=cluster_list, column_name=column_to_search, identifier=cluster_name) except ScyllaManagerError as ex: - LOGGER.warning("Cluster name not found in Scylla-Manager: {}".format(ex)) + LOGGER.warning(f"Cluster name not found in Scylla-Manager: {ex}") return None def set_cluster_id(self, value: str): @@ -537,48 +544,48 @@ def create_restore_task(self, restore_schema=False, restore_data=False, location cmd += " --restore-tables" if location_list: locations_names = ','.join(location_list) - cmd += " --location {} ".format(locations_names) + cmd += f" --location {locations_names} " if snapshot_tag: cmd += f" --snapshot-tag {snapshot_tag}" res = self.sctool.run(cmd=cmd, parse_table_res=False) task_id = res.stdout.strip() - LOGGER.debug("Created task id is: {}".format(task_id)) + LOGGER.debug(f"Created task id is: {task_id}") return RestoreTask(task_id=task_id, cluster_id=self.id, manager_node=self.manager_node) def create_backup_task(self, dc_list=None, # pylint: disable=too-many-arguments,too-many-locals,too-many-branches # noqa: PLR0913, PLR0912 dry_run=None, interval=None, keyspace_list=None, cron=None, location_list=None, num_retries=None, rate_limit_list=None, retention=None, show_tables=None, snapshot_parallel_list=None, start_date=None, upload_parallel_list=None, legacy_args=None): - cmd = "backup -c {}".format(self.id) + cmd = f"backup -c {self.id}" if dc_list is not None: dc_names = ','.join(dc_list) - cmd += " --dc {} ".format(dc_names) + cmd += f" --dc {dc_names} " if dry_run is not None: cmd += " --dry-run" if interval is not None: - cmd += " --interval {}".format(interval) + cmd += f" --interval {interval}" if keyspace_list is not None: keyspaces_names = ','.join(keyspace_list) - cmd += " --keyspace {} ".format(keyspaces_names) + cmd += f" --keyspace {keyspaces_names} " if location_list is not None: locations_names = ','.join(location_list) - cmd += " --location {} ".format(locations_names) + cmd += f" --location {locations_names} " if num_retries is not None: - cmd += " --num-retries {}".format(num_retries) + cmd += f" --num-retries {num_retries}" if rate_limit_list is not None: rate_limit_string = ','.join(rate_limit_list) - cmd += " --rate-limit {} ".format(rate_limit_string) + cmd += f" --rate-limit {rate_limit_string} " if retention is not None: - cmd += " --retention {} ".format(retention) + cmd += f" --retention {retention} " if show_tables is not None: - cmd += " --show-tables {} ".format(show_tables) + cmd += f" --show-tables {show_tables} " if snapshot_parallel_list is not None: snapshot_parallel_string = ','.join(snapshot_parallel_list) - cmd += " --snapshot-parallel {} ".format(snapshot_parallel_string) + cmd += f" --snapshot-parallel {snapshot_parallel_string} " if start_date is not None: - cmd += " --start-date {} ".format(start_date) + cmd += f" --start-date {start_date} " # Since currently we support both manager 2.6 and 3.0, I left the start-date parameter in, # even though it's deprecated in 3.0 # TODO: remove start-date and interval once 2.6 is no longer supported @@ -586,13 +593,13 @@ def create_backup_task(self, dc_list=None, # pylint: disable=too-many-arguments cmd += " --cron '{}' ".format(" ".join(cron)) if upload_parallel_list is not None: upload_parallel_string = ','.join(upload_parallel_list) - cmd += " --upload-parallel {} ".format(upload_parallel_string) + cmd += f" --upload-parallel {upload_parallel_string} " if legacy_args: cmd += f" {legacy_args}" res = self.sctool.run(cmd=cmd, parse_table_res=False) task_id = res.stdout.strip() - LOGGER.debug("Created task id is: {}".format(task_id)) + LOGGER.debug(f"Created task id is: {task_id}") return BackupTask(task_id=task_id, cluster_id=self.id, manager_node=self.manager_node) def create_repair_task(self, dc_list=None, # pylint: disable=too-many-arguments # noqa: PLR0913 @@ -605,16 +612,16 @@ def create_repair_task(self, dc_list=None, # pylint: disable=too-many-arguments # h - hours, # m - minutes, # s - seconds. - cmd = "repair -c {}".format(self.id) + cmd = f"repair -c {self.id}" if dc_list is not None: dc_names = ','.join(dc_list) - cmd += " --dc {} ".format(dc_names) + cmd += f" --dc {dc_names} " if keyspace is not None: - cmd += " --keyspace {} ".format(keyspace) + cmd += f" --keyspace {keyspace} " if interval is not None: - cmd += " --interval {}".format(interval) + cmd += f" --interval {interval}" if num_retries is not None: - cmd += " --num-retries {}".format(num_retries) + cmd += f" --num-retries {num_retries}" if fail_fast is not None: cmd += " --fail-fast" if intensity is not None: @@ -622,7 +629,7 @@ def create_repair_task(self, dc_list=None, # pylint: disable=too-many-arguments if parallel is not None: cmd += f" --parallel {parallel}" if start_date is not None: - cmd += " --start-date {} ".format(start_date) + cmd += f" --start-date {start_date} " # Since currently we support both manager 2.6 and 3.0, I left the start-date parameter in, even though it's # deprecated in 3.0 # TODO: remove start-date once 2.6 is no longer supported @@ -631,24 +638,24 @@ def create_repair_task(self, dc_list=None, # pylint: disable=too-many-arguments res = self.sctool.run(cmd=cmd, parse_table_res=False) if not res: - raise ScyllaManagerError("Unknown failure for sctool {} command".format(cmd)) + raise ScyllaManagerError(f"Unknown failure for sctool {cmd} command") if "no matching units found" in res.stderr: raise ScyllaManagerError("Manager cannot run repair where no keyspace exists.") # expected result output is to have a format of: "repair/2a4125d6-5d5a-45b9-9d8d-dec038b3732d" if 'repair' not in res.stdout: - LOGGER.error("Encountered an error on '{}' command response".format(cmd)) + LOGGER.error(f"Encountered an error on '{cmd}' command response") raise ScyllaManagerError(res.stderr) task_id = res.stdout.split('\n')[0] - LOGGER.debug("Created task id is: {}".format(task_id)) + LOGGER.debug(f"Created task id is: {task_id}") return RepairTask(task_id=task_id, cluster_id=self.id, manager_node=self.manager_node) # return the manager's object with new repair-task-id def control_repair(self, intensity=None, parallel=None): - cmd = " repair control -c {} ".format(self.id) + cmd = f" repair control -c {self.id} " if intensity is not None: cmd += f" --intensity {intensity}" if parallel is not None: @@ -656,7 +663,7 @@ def control_repair(self, intensity=None, parallel=None): res = self.sctool.run(cmd=cmd, parse_table_res=False) if not res: - raise ScyllaManagerError("Unknown failure for sctool {} command".format(cmd)) + raise ScyllaManagerError(f"Unknown failure for sctool {cmd} command") def get_backup_files_dict(self, snapshot_tag): command = f" -c {self.id} backup files --snapshot-tag {snapshot_tag}" @@ -672,7 +679,7 @@ def get_backup_files_dict(self, snapshot_tag): def snapshot_files_to_dict(snapshot_file_lines): per_node_keyspaces_and_tables_backup_files = {} for line in snapshot_file_lines: - s3_file_path, keyspace_and_table = [string.strip() for string in line.split(' ')] + s3_file_path, keyspace_and_table = (string.strip() for string in line.split(' ')) node_id = s3_file_path[s3_file_path.find("/node/") + len("/node/"):s3_file_path.find("/keyspace")] keyspace, table = keyspace_and_table.split('/') if node_id not in per_node_keyspaces_and_tables_backup_files: @@ -689,7 +696,7 @@ def delete(self): $ sctool cluster delete """ - cmd = "cluster delete -c {}".format(self.id) + cmd = f"cluster delete -c {self.id}" self.sctool.run(cmd=cmd, is_verify_errorless_result=True) def update(self, name=None, host=None, client_encrypt=None): # pylint: disable=too-many-arguments @@ -705,24 +712,24 @@ def update(self, name=None, host=None, client_encrypt=None): # pylint: disable= --host string hostname or IP of one of the cluster nodes -n, --name alias alias you can give to your cluster """ - cmd = "cluster update -c {}".format(self.id) + cmd = f"cluster update -c {self.id}" if name: - cmd += " --name={}".format(name) + cmd += f" --name={name}" if host: - cmd += " --host={}".format(host) + cmd += f" --host={host}" if client_encrypt: - cmd += " --ssl-user-cert-file {} --ssl-user-key-file {}".format(SSL_USER_CERT_FILE, SSL_USER_KEY_FILE) + cmd += f" --ssl-user-cert-file {SSL_USER_CERT_FILE} --ssl-user-key-file {SSL_USER_KEY_FILE}" self.sctool.run(cmd=cmd, is_verify_errorless_result=True) def delete_task(self, task: ManagerTask): task_id = task.id if self.sctool.is_v3_cli: - cmd = "stop --delete {} -c {}".format(task_id, self.id) + cmd = f"stop --delete {task_id} -c {self.id}" else: - cmd = "-c {} task delete {}".format(self.id, task_id) - LOGGER.debug("Task Delete command to execute is: {}".format(cmd)) + cmd = f"-c {self.id} task delete {task_id}" + LOGGER.debug(f"Task Delete command to execute is: {cmd}") self.sctool.run(cmd=cmd, parse_table_res=False) - LOGGER.debug("Deleted the task '{}' successfully!". format(task_id)) + LOGGER.debug(f"Deleted the task '{task_id}' successfully!") def delete_automatic_repair_task(self): repair_tasks_list = self.repair_task_list @@ -753,9 +760,9 @@ def name(self): def _get_task_list(self): if self.sctool.is_v3_cli: - cmd = "tasks -c {}".format(self.id) + cmd = f"tasks -c {self.id}" else: - cmd = "task list -c {}".format(self.id) + cmd = f"task list -c {self.id}" return self.sctool.run(cmd=cmd, is_verify_errorless_result=True) def _get_task_list_filtered(self, prefix, task_class): @@ -813,13 +820,13 @@ def get_hosts_health(self): # │ UN │ TIMEOUT SSL │ TIMEOUT │ 192.168.100.22 │ 56d2f4c0-9327-487e-b115-c96d3e5c014b │ # │ UN │ UP SSL (40ms) │ HTTP (503) (7ms) │ 192.168.100.23 │ 08152d3d-ed30-469e-bc19-5ab9f4248e9a │ # ╰────┴─────────────────────────┴───────────────────┴────────────────┴──────────────────────────────────────╯ - cmd = "status -c {}".format(self.id) + cmd = f"status -c {self.id}" dict_status_tables = self.sctool.run(cmd=cmd, is_verify_errorless_result=True, is_multiple_tables=True) dict_hosts_health = {} for dc_name, hosts_table in dict_status_tables.items(): if len(hosts_table) < 2: # noqa: PLR2004 - LOGGER.debug("Cluster: {} - {} has no hosts health report".format(self.id, dc_name)) + LOGGER.debug(f"Cluster: {self.id} - {dc_name} has no hosts health report") else: list_titles_row = hosts_table[0] host_col_idx = list_titles_row.index("Address") @@ -846,13 +853,13 @@ def get_hosts_health(self): rest_status=HostRestStatus.from_str(rest_status), rest_rtt=rest_rtt, ssl=HostSsl.from_str(ssl), rest_http_status_code=rest_http_status_code) - LOGGER.debug("Cluster {} Hosts Health is:".format(self.id)) + LOGGER.debug(f"Cluster {self.id} Hosts Health is:") for ip, health in dict_hosts_health.items(): LOGGER.debug("{}: {},{},{},{},{}".format(ip, health.status, health.rtt, health.rest_status, health.rest_rtt, health.ssl)) return dict_hosts_health - class _HostHealth(): # pylint: disable=too-few-public-methods + class _HostHealth: # pylint: disable=too-few-public-methods def __init__(self, status, rtt, ssl, rest_status, rest_rtt, rest_http_status_code=None): # pylint: disable=too-many-arguments # noqa: PLR0913 self.status = status self.rtt = rtt @@ -896,7 +903,7 @@ def verify_errorless_result(cmd, res): raise ScyllaManagerError("Encountered an error on '{}' command response {}\ncommand exit code:{}\nstderr:{}".format( cmd, res, res.exited, res.stderr)) if res.stderr: - LOGGER.error("Encountered an error on '{}' stderr: {}".format(cmd, str(res.stderr))) # TODO: just for checking + LOGGER.error(f"Encountered an error on '{cmd}' stderr: {str(res.stderr)}") # TODO: just for checking class ScyllaManagerTool(ScyllaManagerBase): @@ -909,14 +916,14 @@ class ScyllaManagerTool(ScyllaManagerBase): def __init__(self, manager_node): ScyllaManagerBase.__init__(self, id="MANAGER", manager_node=manager_node) self._initial_wait(20) - LOGGER.info("Initiating Scylla-Manager, version: {}".format(self.sctool.version)) + LOGGER.info(f"Initiating Scylla-Manager, version: {self.sctool.version}") list_supported_distros = [Distro.CENTOS7, Distro.DEBIAN10, Distro.DEBIAN11, Distro.UBUNTU20, Distro.UBUNTU22] self.default_user = "centos" if manager_node.distro not in list_supported_distros: raise ScyllaManagerError( - "Non-Manager-supported Distro found on Monitoring Node: {}".format(manager_node.distro)) + f"Non-Manager-supported Distro found on Monitoring Node: {manager_node.distro}") @staticmethod def _initial_wait(seconds: int): @@ -997,8 +1004,7 @@ def add_cluster(self, name, host=None, db_cluster=None, client_encrypt=None, dis raise ScyllaManagerError("Neither host or db_cluster parameter were given to Manager add_cluster") host = host or self.get_cluster_hosts_ip(db_cluster=db_cluster)[0] # FIXME: if cluster already added, print a warning, but not fail - cmd = 'cluster add --host={} --name={} --auth-token {}'.format( - host, name, auth_token) + cmd = f'cluster add --host={host} --name={name} --auth-token {auth_token}' # Adding client-encryption parameters if required if client_encrypt: if not db_cluster: @@ -1030,7 +1036,7 @@ def upgrade(self, scylla_mgmt_upgrade_to_repo): manager_from_version, scylla_mgmt_upgrade_to_repo)) self.manager_node.upgrade_mgmt(scylla_mgmt_address=scylla_mgmt_upgrade_to_repo) new_manager_version = self.sctool.version - LOGGER.debug('The Manager version after upgrade is: {}'.format(new_manager_version)) + LOGGER.debug(f'The Manager version after upgrade is: {new_manager_version}') return new_manager_version def rollback_upgrade(self, scylla_mgmt_address): @@ -1045,13 +1051,13 @@ def __init__(self, manager_node): def rollback_upgrade(self, scylla_mgmt_address): - remove_post_upgrade_repo = dedent(""" + remove_post_upgrade_repo = dedent(f""" sudo systemctl stop scylla-manager cqlsh -e 'DROP KEYSPACE scylla_manager' - sudo rm -rf {} + sudo rm -rf {self.manager_repo_path} sudo yum clean all sudo rm -rf /var/cache/yum - """.format(self.manager_repo_path)) + """) self.manager_node.remoter.run('sudo bash -cxe "%s"' % remove_post_upgrade_repo) # Downgrade to pre-upgrade scylla-manager repository @@ -1076,16 +1082,16 @@ def __init__(self, manager_node): def rollback_upgrade(self, scylla_mgmt_address): manager_from_version = self.sctool.version[0] - remove_post_upgrade_repo = dedent(""" + remove_post_upgrade_repo = dedent(f""" cqlsh -e 'DROP KEYSPACE scylla_manager' sudo systemctl stop scylla-manager sudo systemctl stop scylla-server.service sudo apt-get remove scylla-manager -y sudo apt-get remove scylla-manager-server -y sudo apt-get remove scylla-manager-client -y - sudo rm -rf {} + sudo rm -rf {self.manager_repo_path} sudo apt-get clean - """.format(self.manager_repo_path)) # +" /var/lib/scylla-manager/*")) + """) # +" /var/lib/scylla-manager/*")) self.manager_node.remoter.run('sudo bash -cxe "%s"' % remove_post_upgrade_repo) self.manager_node.remoter.run('sudo apt-get update', ignore_status=True) @@ -1094,7 +1100,7 @@ def rollback_upgrade(self, scylla_mgmt_address): res = self.manager_node.remoter.run('sudo apt-get update', ignore_status=True) res = self.manager_node.remoter.run('apt-cache show scylla-manager-client | grep Version:') rollback_to_version = res.stdout.split()[1] - LOGGER.debug("Rolling back manager version from: {} to: {}".format(manager_from_version, rollback_to_version)) + LOGGER.debug(f"Rolling back manager version from: {manager_from_version} to: {rollback_to_version}") # self.manager_node.install_mgmt(scylla_mgmt_address=scylla_mgmt_address) downgrade_to_pre_upgrade_repo = dedent(""" @@ -1242,7 +1248,7 @@ def get_table_value(self, parsed_table, identifier, column_name=None, is_search_ column_titles = [title.upper() for title in parsed_table[0]] # get all table column titles capital (for comparison) if column_name and column_name.upper() not in column_titles: - raise ScyllaManagerError("Column name: {} not found in table: {}".format(column_name, parsed_table)) + raise ScyllaManagerError(f"Column name: {column_name} not found in table: {parsed_table}") column_name_index = column_titles.index( column_name.upper()) if column_name else 1 # "1" is used in a case like "task progress" where no column names exist. ret_val = 'N/A' @@ -1254,7 +1260,7 @@ def get_table_value(self, parsed_table, identifier, column_name=None, is_search_ elif identifier in row: ret_val = row[column_name_index] break - LOGGER.debug("{} {} value is:{}".format(identifier, column_name, ret_val)) + LOGGER.debug(f"{identifier} {column_name} value is:{ret_val}") return ret_val @staticmethod @@ -1266,7 +1272,7 @@ def get_all_column_values_from_table(parsed_table, column_name): column_titles = [title.upper() for title in parsed_table[0]] # get all table column titles capital (for comparison) if column_name and column_name.upper() not in column_titles: - raise ScyllaManagerError("Column name: {} not found in table: {}".format(column_name, parsed_table)) + raise ScyllaManagerError(f"Column name: {column_name} not found in table: {parsed_table}") column_name_index = column_titles.index(column_name.upper()) column_values = [row[column_name_index] for row in parsed_table[1:]] @@ -1306,14 +1312,14 @@ class ScyllaMgmt: """ def __init__(self, server, port=9090): - self._url = 'http://{}:{}/api/v1/'.format(server, port) + self._url = f'http://{server}:{port}/api/v1/' def get(self, path, params=None): if not params: params = {} resp = requests.get(url=self._url + path, params=params) if resp.status_code not in [200, 201, 202]: - err_msg = 'GET request to scylla-manager failed! error: {}'.format(resp.content) + err_msg = f'GET request to scylla-manager failed! error: {resp.content}' LOGGER.error(err_msg) raise Exception(err_msg) try: @@ -1325,7 +1331,7 @@ def get(self, path, params=None): def post(self, path, data): resp = requests.post(url=self._url + path, data=json.dumps(data)) if resp.status_code not in [200, 201]: - err_msg = 'POST request to scylla-manager failed! error: {}'.format(resp.content) + err_msg = f'POST request to scylla-manager failed! error: {resp.content}' LOGGER.error(err_msg) raise Exception(err_msg) return resp @@ -1335,7 +1341,7 @@ def put(self, path, data=None): data = {} resp = requests.put(url=self._url + path, data=json.dumps(data)) if resp.status_code not in [200, 201]: - err_msg = 'PUT request to scylla-manager failed! error: {}'.format(resp.content) + err_msg = f'PUT request to scylla-manager failed! error: {resp.content}' LOGGER.error(err_msg) raise Exception(err_msg) return resp @@ -1343,7 +1349,7 @@ def put(self, path, data=None): def delete(self, path): resp = requests.delete(url=self._url + path) if resp.status_code not in [200, 204]: - err_msg = 'DELETE request to scylla-manager failed! error: {}'.format(resp.content) + err_msg = f'DELETE request to scylla-manager failed! error: {resp.content}' LOGGER.error(err_msg) raise Exception(err_msg) @@ -1378,7 +1384,7 @@ def delete_cluster(self, cluster_id): :param cluster_id: cluster id/name :return: nothing """ - self.delete('cluster/{}'.format(cluster_id)) + self.delete(f'cluster/{cluster_id}') def get_schedule_task(self, cluster_id): """ @@ -1388,7 +1394,7 @@ def get_schedule_task(self, cluster_id): """ resp = [] while not resp: - resp = self.get(path='cluster/{}/tasks'.format(cluster_id), params={'type': 'repair_auto_schedule'}) + resp = self.get(path=f'cluster/{cluster_id}/tasks', params={'type': 'repair_auto_schedule'}) return resp[0] def disable_task_schedule(self, cluster_id, task): @@ -1401,7 +1407,7 @@ def disable_task_schedule(self, cluster_id, task): self.put(path='cluster/{}/task/repair_auto_schedule/{}'.format(cluster_id, task['id']), data=task) def start_repair_task(self, cluster_id, task_id, task_type='repair'): - self.put(path='cluster/{}/task/{}/{}/start'.format(cluster_id, task_type, task_id)) + self.put(path=f'cluster/{cluster_id}/task/{task_type}/{task_id}/start') def get_repair_tasks(self, cluster_id): """ @@ -1411,7 +1417,7 @@ def get_repair_tasks(self, cluster_id): """ resp = [] while not resp: - resp = self.get(path='cluster/{}/tasks'.format(cluster_id), params={'type': 'repair'}) + resp = self.get(path=f'cluster/{cluster_id}/tasks', params={'type': 'repair'}) tasks = {} for task in resp: unit_id = task['properties']['unit_id'] @@ -1421,7 +1427,7 @@ def get_repair_tasks(self, cluster_id): def get_task_progress(self, cluster_id, repair_unit): try: - return self.get(path='cluster/{}/repair/unit/{}/progress'.format(cluster_id, repair_unit)) + return self.get(path=f'cluster/{cluster_id}/repair/unit/{repair_unit}/progress') except Exception as ex: # pylint: disable=broad-except LOGGER.exception('Failed to get repair progress: %s', ex) return None diff --git a/sdcm/mgmt/common.py b/sdcm/mgmt/common.py index b179c345a69..6c94ed58405 100644 --- a/sdcm/mgmt/common.py +++ b/sdcm/mgmt/common.py @@ -1,11 +1,11 @@ -from enum import Enum -import logging import datetime +import logging +from enum import Enum + import yaml from sdcm.utils.distro import Distro - DEFAULT_TASK_TIMEOUT = 7200 # 2 hours LOGGER = logging.getLogger(__name__) @@ -132,7 +132,7 @@ def from_str(cls, output_str): return cls.DOWN return getattr(cls, output_str) except AttributeError as err: - raise ScyllaManagerError("Could not recognize returned host status: {}".format(output_str)) from err + raise ScyllaManagerError(f"Could not recognize returned host status: {output_str}") from err class HostRestStatus(Enum): @@ -150,7 +150,7 @@ def from_str(cls, output_str): return cls.DOWN return getattr(cls, output_str) except AttributeError as err: - raise ScyllaManagerError("Could not recognize returned host rest status: {}".format(output_str)) from err + raise ScyllaManagerError(f"Could not recognize returned host rest status: {output_str}") from err class TaskStatus: # pylint: disable=too-few-public-methods @@ -173,7 +173,7 @@ def from_str(cls, output_str) -> str: output_str = output_str.upper() return getattr(cls, output_str) except AttributeError as err: - raise ScyllaManagerError("Could not recognize returned task status: {}".format(output_str)) from err + raise ScyllaManagerError(f"Could not recognize returned task status: {output_str}") from err @classmethod def all_statuses(cls): diff --git a/sdcm/mgmt/operator.py b/sdcm/mgmt/operator.py index 1daefd11640..44877ee4cc9 100644 --- a/sdcm/mgmt/operator.py +++ b/sdcm/mgmt/operator.py @@ -12,21 +12,19 @@ # Copyright (c) 2021 ScyllaDB import logging -from typing import List, Optional -from dataclasses import dataclass, asdict, fields +from dataclasses import asdict, dataclass, fields from sdcm.mgmt.cli import ( BackupTask, HealthcheckTask, ManagerCluster, + ManagerTask, RepairTask, ScyllaManagerTool, - ManagerTask, ) from sdcm.mgmt.common import TaskStatus from sdcm.wait import wait_for - LOGGER = logging.getLogger(__name__) @@ -72,7 +70,7 @@ class ScyllaOperatorTaskBaseClass(BaseClass): class ScyllaOperatorRepairTask(ScyllaOperatorTaskBaseClass): # DC list of datacenter glob patterns, e.g. 'dc1', '!otherdc*' used to specify the DCs # to include or exclude from backup. - dc: List[str] = None + dc: list[str] = None # fail_fast stop repair on first error. fail_fast: bool = None # Intensity integer >= 1 or a decimal between (0,1), higher values may result in higher speed and cluster load. @@ -83,7 +81,7 @@ class ScyllaOperatorRepairTask(ScyllaOperatorTaskBaseClass): parallel: int = None # Keyspace a list of keyspace/tables glob patterns, e.g. 'keyspace,!keyspace.table_prefix_*' # used to include or exclude keyspaces from repair. - keyspace: List[str] = None + keyspace: list[str] = None # small_table_threshold enable small table optimization for tables of size lower than given threshold. # Supported units [B, MiB, GiB, TiB] (default "1GiB"). small_table_threshold: str = None @@ -111,7 +109,7 @@ class ScyllaOperatorRepairTaskStatus(ScyllaOperatorRepairTask): class ScyllaOperatorBackupTask(ScyllaOperatorTaskBaseClass): # DC list of datacenter glob patterns, e.g. 'dc1', '!otherdc*' used to specify the DCs # to include or exclude from backup. - dc: List[str] = None + dc: list[str] = None # fail_fast stop repair on first error. fail_fast: bool = None # Intensity integer >= 1 or a decimal between (0,1), higher values may result in higher speed and cluster load. @@ -122,29 +120,29 @@ class ScyllaOperatorBackupTask(ScyllaOperatorTaskBaseClass): parallel: int = None # Keyspace a list of keyspace/tables glob patterns, e.g. 'keyspace,!keyspace.table_prefix_*' # used to include or exclude keyspaces from repair. - keyspace: List[str] = None + keyspace: list[str] = None # small_table_threshold enable small table optimization for tables of size lower than given threshold. # Supported units [B, MiB, GiB, TiB] (default "1GiB"). small_table_threshold: str = None # List of locations where backup is going to be stored, location is string in following format: # : , where provider could be gcs or s3 - location: List[str] = None + location: list[str] = None # RateLimit a list of megabytes (MiB) per second rate limits expressed in the format [:]. # The : part is optional and only needed when different datacenters need different upload limits. # Set to 0 for no limit (default 100). - rate_limit: List[str] = None + rate_limit: list[str] = None # Retention The number of backups which are to be stored (default 3). retention: int = None # SnapshotParallel a list of snapshot parallelism limits in the format [:]. # The : part is optional and allows for specifying different limits in selected datacenters. # If The : part is not set, the limit is global (e.g. 'dc1:2,5') the runs are parallel in n nodes (2 in dc1) # and n nodes in all the other datacenters. - snapshot_parallel: List[str] = None + snapshot_parallel: list[str] = None # UploadParallel a list of upload parallelism limits in the format [:]. # The : part is optional and allows for specifying different limits in selected datacenters. # If The : part is not set the limit is global (e.g. 'dc1:2,5') the runs are parallel in n nodes (2 in dc1) # and n nodes in all the other datacenters. - upload_parallel: List[str] = None + upload_parallel: list[str] = None @dataclass @@ -212,7 +210,7 @@ def are_healthchecks_done(self): # | healthcheck_rest/uuid-quuz | | 06 May 22 12:28:30 UTC (+1m) | DONE | # +---------------------------------+-----------+-------------------------------+--------+ healthchecks = self._get_task_list_filtered(prefix="healthcheck", task_class=HealthcheckTask) - return all((healthcheck.status == TaskStatus.DONE for healthcheck in healthchecks)) + return all(healthcheck.status == TaskStatus.DONE for healthcheck in healthchecks) def wait_for_healthchecks(self): wait_for( @@ -325,7 +323,7 @@ def create_repair_task(self, dc_list=None, # pylint: disable=too-many-arguments intensity=intensity, parallel=parallel, name=name) return wait_for(lambda: self.get_mgr_repair_task(so_task), step=2, timeout=300) - def get_mgr_repair_task(self, so_repair_task: ScyllaOperatorRepairTask) -> Optional[RepairTask]: + def get_mgr_repair_task(self, so_repair_task: ScyllaOperatorRepairTask) -> RepairTask | None: so_repair_task_status = wait_for( func=self.get_operator_repair_task_status, text=f"Waiting until operator repair task: {so_repair_task.name} get it's status", @@ -338,7 +336,7 @@ def get_mgr_repair_task(self, so_repair_task: ScyllaOperatorRepairTask) -> Optio return mgr_task return None - def get_mgr_backup_task(self, so_backup_task: ScyllaOperatorBackupTask) -> Optional[BackupTask]: + def get_mgr_backup_task(self, so_backup_task: ScyllaOperatorBackupTask) -> BackupTask | None: so_backup_task_status = wait_for( func=self.get_operator_backup_task_status, text=f"Waiting until operator backup task '{so_backup_task.name}' get it's status", @@ -351,13 +349,13 @@ def get_mgr_backup_task(self, so_backup_task: ScyllaOperatorBackupTask) -> Optio return mgr_task return None - def get_operator_repair_task_status(self, task_name: str) -> Optional[ScyllaOperatorRepairTaskStatus]: + def get_operator_repair_task_status(self, task_name: str) -> ScyllaOperatorRepairTaskStatus | None: for task_status in self.operator_repair_task_statuses: if task_status.name == task_name: return task_status return None - def get_operator_backup_task_status(self, task_name: str) -> Optional[ScyllaOperatorBackupTaskStatus]: + def get_operator_backup_task_status(self, task_name: str) -> ScyllaOperatorBackupTaskStatus | None: for task_status in self.operator_backup_task_statuses: if task_status.name == task_name: return task_status @@ -370,19 +368,19 @@ def _get_list_of_entities_from_operator(self, path, entity_class): return [entity_class.from_dict(task_status) for task_status in repair_task_status_infos] @property - def operator_repair_task_statuses(self) -> List[ScyllaOperatorRepairTaskStatus]: + def operator_repair_task_statuses(self) -> list[ScyllaOperatorRepairTaskStatus]: return self._get_list_of_entities_from_operator('/status/repairs', ScyllaOperatorRepairTaskStatus) @property - def operator_backup_task_statuses(self) -> List[ScyllaOperatorBackupTaskStatus]: + def operator_backup_task_statuses(self) -> list[ScyllaOperatorBackupTaskStatus]: return self._get_list_of_entities_from_operator('/status/backups', ScyllaOperatorBackupTaskStatus) @property - def operator_repair_tasks(self) -> List[ScyllaOperatorRepairTask]: + def operator_repair_tasks(self) -> list[ScyllaOperatorRepairTask]: return self._get_list_of_entities_from_operator('/spec/repairs', ScyllaOperatorRepairTask) @property - def operator_backup_tasks(self) -> List[ScyllaOperatorBackupTask]: + def operator_backup_tasks(self) -> list[ScyllaOperatorBackupTask]: return self._get_list_of_entities_from_operator('/spec/backups', ScyllaOperatorBackupTask) def update(self, name=None, host=None, client_encrypt=None): diff --git a/sdcm/microbenchmarking.py b/sdcm/microbenchmarking.py index ca6430dcd2e..83927287ba7 100755 --- a/sdcm/microbenchmarking.py +++ b/sdcm/microbenchmarking.py @@ -13,17 +13,17 @@ # # Copyright (c) 2020 ScyllaDB -import os -import sys -import logging +import argparse +import contextlib import datetime import json -import argparse +import logging +import os import socket +import sys import tempfile -from collections import defaultdict -import contextlib import warnings +from collections import defaultdict # disable InsecureRequestWarning import urllib3 @@ -31,10 +31,13 @@ # HACK: since cryptography==37.0.0 CryptographyDeprecationWarning is being raised # this is until https://github.com/paramiko/paramiko/issues/2038 would be solved from cryptography.utils import CryptographyDeprecationWarning + warnings.filterwarnings(action='ignore', category=CryptographyDeprecationWarning) sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from sdcm.results_analyze import BaseResultsAnalyzer # pylint: disable=wrong-import-position +from sdcm.results_analyze import ( + BaseResultsAnalyzer, # pylint: disable=wrong-import-position +) from sdcm.utils.log import setup_stdout_logger # pylint: disable=wrong-import-position urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -63,7 +66,7 @@ def __init__(self, msg, *args, **kwargs): self.message = msg def __str__(self): - return "MBM: {0.message}".format(self) + return f"MBM: {self.message}" class EmptyResultFolder(Exception): @@ -72,7 +75,7 @@ def __init__(self, msg, *args, **kwargs): self.message = msg def __str__(self): - return "MBM: {0.message}".format(self) + return f"MBM: {self.message}" class MicroBenchmarkingResultsAnalyzer(BaseResultsAnalyzer): # pylint: disable=too-many-instance-attributes @@ -145,8 +148,8 @@ def check_regression(self, current_results): # pylint: disable=arguments-differ sorted_by_type = defaultdict(list) for res in results: - test_type = "%s_%s" % (res["_source"]["test_group_properties"]["name"], - res["_source"]["test_args"]) + test_type = "{}_{}".format(res["_source"]["test_group_properties"]["name"], + res["_source"]["test_args"]) sorted_by_type[test_type].append(res) report_results = defaultdict(dict) @@ -339,9 +342,9 @@ def get_results(self, results_path, update_db): if not subdirs: dataset_name = os.path.basename(fullpath) - self.log.info('Dataset name: {}'.format(dataset_name)) + self.log.info(f'Dataset name: {dataset_name}') dirname = os.path.basename(os.path.dirname(fullpath)) - self.log.info("Test set: {}".format(dirname)) + self.log.info(f"Test set: {dirname}") for filename in files: if filename.startswith('.'): continue @@ -360,7 +363,7 @@ def get_results(self, results_path, update_db): }) if update_db: self._es.create_doc(index=self._es_index, doc_type=self._es_doc_type, - doc_id="%s_%s" % (self.test_run_date, test_type), body=datastore) + doc_id=f"{self.test_run_date}_{test_type}", body=datastore) results[test_type] = datastore if not results: raise EmptyResultFolder("perf_fast_forward_output folder is empty") @@ -381,7 +384,7 @@ def exclude_test_run(self, testrun_id=''): self.log.info("Nothing to exclude") return - self.log.info('Exclude testrun {} from results'.format(testrun_id)) + self.log.info(f'Exclude testrun {testrun_id} from results') filter_path = ( "hits.hits._id", # '2018-04-02_18:36:47_large-partition-skips_[64-32.1)' "hits.hits._source.hostname", # 'godzilla.cloudius-systems.com' @@ -414,7 +417,7 @@ def exclude_by_test_id(self, test_id=''): self.log.info("Nothing to exclude") return - self.log.info('Exclude test id {} from results'.format(test_id)) + self.log.info(f'Exclude test id {test_id} from results') doc = self._es.get_doc(index=self._es_index, doc_id=test_id) if doc: self._es.update_doc(index=self._es_index, @@ -455,10 +458,9 @@ def exclude_before_date(self, date=''): "hits.hits._source.hostname", "hits.hits._source.versions.scylla-server.run_date_time" ) - self.log.info('Exclude tests before date {}'.format(date)) + self.log.info(f'Exclude tests before date {date}') results = self._es.search(index=self._es_index, filter_path=filter_path, size=self._limit, # pylint: disable=unexpected-keyword-arg - q="hostname:'%s' AND versions.scylla-server.version:%s*" % - (self.hostname, self.db_version[:3])) + q=f"hostname:'{self.hostname}' AND versions.scylla-server.version:{self.db_version[:3]}*") if not results: self.log.info('Nothing to exclude') return @@ -488,14 +490,14 @@ def exclude_testrun_by_commit_id(self, commit_id=None): "hits.hits._source.test_run_date" ) - self.log.info('Exclude tests by commit id #{}'.format(commit_id)) + self.log.info(f'Exclude tests by commit id #{commit_id}') results = self._es.search(index=self._es_index, filter_path=filter_path, size=self._limit, # pylint: disable=unexpected-keyword-arg - q="hostname:'{}' \ - AND versions.scylla-server.version:{}*\ - AND versions.scylla-server.commit_id:'{}'".format(self.hostname, self.db_version[:3], commit_id)) + q=f"hostname:'{self.hostname}' \ + AND versions.scylla-server.version:{self.db_version[:3]}*\ + AND versions.scylla-server.commit_id:'{commit_id}'") if not results: - self.log.info('There is no testrun results for commit id #{}'.format(commit_id)) + self.log.info(f'There is no testrun results for commit id #{commit_id}') return for doc in results['hits']['hits']: diff --git a/sdcm/monitorstack/__init__.py b/sdcm/monitorstack/__init__.py index 8a1a3a2ea01..435f264ab1f 100644 --- a/sdcm/monitorstack/__init__.py +++ b/sdcm/monitorstack/__init__.py @@ -1,22 +1,26 @@ +import datetime +import json import logging import os import tarfile -import zipfile import tempfile -import json -import datetime import time -from textwrap import dedent +import zipfile from pathlib import Path +from textwrap import dedent import requests import yaml from sdcm.remote import LocalCmdRunner -from sdcm.utils.common import list_logs_by_test_id, S3Storage, remove_files, get_free_port +from sdcm.utils.common import ( + S3Storage, + get_free_port, + list_logs_by_test_id, + remove_files, +) from sdcm.utils.decorators import retrying - LOGGER = logging.getLogger(name='monitoringstack') GRAFANA_DOCKER_NAME = "agraf" @@ -163,15 +167,13 @@ def create_monitoring_data_dir(base_dir, archive, tenant_dir=""): if tenant_dir: monitoring_data_base_dir = os.path.join(monitoring_data_base_dir, tenant_dir) - cmd = dedent(""" - mkdir -p {data_dir} - cd {data_dir} + cmd = dedent(f""" + mkdir -p {monitoring_data_base_dir} + cd {monitoring_data_base_dir} cp {archive} ./ - tar -xvf {archive_name} - chmod -R 777 {data_dir} - """.format(data_dir=monitoring_data_base_dir, - archive=archive, - archive_name=os.path.basename(archive))) + tar -xvf {os.path.basename(archive)} + chmod -R 777 {monitoring_data_base_dir} + """) result = LocalCmdRunner().run(cmd, timeout=COMMAND_TIMEOUT, ignore_status=True) if result.exited > 0: LOGGER.error("Error during extracting prometheus snapshot. Switch to next archive") @@ -203,14 +205,12 @@ def get_nemesis_dashboard_file_for_cluster(base_dir, archive, file_name_for_sear def create_monitoring_stack_dir(base_dir, archive): - cmd = dedent(""" - cd {data_dir} + cmd = dedent(f""" + cd {base_dir} cp {archive} ./ - tar -xvf {archive_name} - chmod -R 777 {data_dir} - """.format(data_dir=base_dir, - archive_name=os.path.basename(archive), - archive=archive)) + tar -xvf {os.path.basename(archive)} + chmod -R 777 {base_dir} + """) result = LocalCmdRunner().run(cmd, timeout=COMMAND_TIMEOUT, ignore_status=True) if result.exited > 0: @@ -382,8 +382,7 @@ def restore_annotations_data(monitoring_stack_dir, grafana_docker_port): res = requests.post(annotations_url, data=json.dumps(an), headers={'Content-Type': 'application/json'}) if res.status_code != 200: # noqa: PLR2004 LOGGER.info('Error during uploading annotation %s. Error message %s', an, res.text) - raise ErrorUploadAnnotations('Error during uploading annotation {}. Error message {}'.format(an, - res.text)) + raise ErrorUploadAnnotations(f'Error during uploading annotation {an}. Error message {res.text}') LOGGER.info('Annotations loaded successfully') return True except Exception as details: # pylint: disable=broad-except @@ -480,7 +479,7 @@ def verify_grafana_is_available(grafana_docker_port=GRAFANA_DOCKER_PORT): port=grafana_docker_port, title=dashboard.title) grafana_statuses.append(result) - LOGGER.info("Dashboard {} is available".format(dashboard.title)) + LOGGER.info(f"Dashboard {dashboard.title} is available") except Exception as details: # pylint: disable=broad-except # noqa: BLE001 LOGGER.error("Dashboard %s is not available. Error: %s", dashboard.title, details) grafana_statuses.append(False) diff --git a/sdcm/monitorstack/ui.py b/sdcm/monitorstack/ui.py index d6ee8322219..732ba009c93 100644 --- a/sdcm/monitorstack/ui.py +++ b/sdcm/monitorstack/ui.py @@ -1,16 +1,13 @@ import logging -from typing import Tuple, List - -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC +from selenium.common import exceptions from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement -from selenium.common import exceptions +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait from sdcm.utils.ci_tools import get_test_name - LOGGER = logging.getLogger(__name__) UI_ELEMENT_LOAD_TIMEOUT = 180 GRAFANA_USERNAME = "admin" @@ -120,8 +117,8 @@ class Dashboard: name: str path: str resolution: str - scroll_ready_locator: Tuple[By, str] = (By.XPATH, "//div[@class='scrollbar-view']") - panels: List[Panel] + scroll_ready_locator: tuple[By, str] = (By.XPATH, "//div[@class='scrollbar-view']") + panels: list[Panel] scroll_step: int = 1000 title: str diff --git a/sdcm/ndbench_thread.py b/sdcm/ndbench_thread.py index 1ae653c6efb..a46ba881cd0 100644 --- a/sdcm/ndbench_thread.py +++ b/sdcm/ndbench_thread.py @@ -11,19 +11,19 @@ # # Copyright (c) 2020 ScyllaDB +import logging import os import re -import logging import time import uuid from typing import Any from sdcm.prometheus import nemesis_metrics_obj -from sdcm.sct_events.loaders import NdBenchStressEvent, NDBENCH_ERROR_EVENTS_PATTERNS +from sdcm.sct_events.loaders import NDBENCH_ERROR_EVENTS_PATTERNS, NdBenchStressEvent +from sdcm.stress.base import format_stress_cmd_error +from sdcm.stress_thread import DockerBasedStressThread from sdcm.utils.common import FileFollowerThread from sdcm.utils.docker_remote import RemoteDocker -from sdcm.stress_thread import DockerBasedStressThread -from sdcm.stress.base import format_stress_cmd_error LOGGER = logging.getLogger(__name__) @@ -107,7 +107,7 @@ def run(self): self.set_metric(operation, name, float(value)) except Exception as exc: # pylint: disable=broad-except # noqa: BLE001 - LOGGER.warning("Failed to send metric. Failed with exception {exc}".format(exc=exc)) + LOGGER.warning(f"Failed to send metric. Failed with exception {exc}") class NdBenchStressThread(DockerBasedStressThread): # pylint: disable=too-many-instance-attributes diff --git a/sdcm/nemesis.py b/sdcm/nemesis.py index d63478d5c3c..66f95b1dd9c 100644 --- a/sdcm/nemesis.py +++ b/sdcm/nemesis.py @@ -20,6 +20,7 @@ import copy import datetime import inspect +import json import logging import math import os @@ -27,36 +28,37 @@ import re import time import traceback -import json -from distutils.version import LooseVersion -from contextlib import ExitStack -from typing import Any, List, Optional, Type, Tuple, Callable, Dict, Set, Union, Iterable -from functools import wraps, partial -from collections import defaultdict, Counter, namedtuple +from collections import Counter, defaultdict, namedtuple +from collections.abc import Callable, Iterable from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from distutils.version import LooseVersion +from functools import partial, wraps from threading import Lock from types import MethodType # pylint: disable=no-name-in-module +from typing import Any +from argus.backend.util.enums import NemesisStatus from cassandra import ConsistencyLevel, InvalidRequest from cassandra.query import SimpleStatement # pylint: disable=no-name-in-module -from invoke import UnexpectedExit from elasticsearch.exceptions import ConnectionTimeout as ElasticSearchConnectionTimeout -from argus.backend.util.enums import NemesisStatus +from invoke import UnexpectedExit from sdcm import wait from sdcm.audit import Audit, AuditConfiguration, AuditStore from sdcm.cluster import ( + DB_LOG_PATTERN_RESHARDING_FINISH, + DB_LOG_PATTERN_RESHARDING_START, + HOUR_IN_SEC, + MAX_TIME_WAIT_FOR_DECOMMISSION, + MAX_TIME_WAIT_FOR_NEW_NODE_UP, BaseCluster, BaseNode, BaseScyllaCluster, ClusterNodesNotReady, - DB_LOG_PATTERN_RESHARDING_START, - DB_LOG_PATTERN_RESHARDING_FINISH, - MAX_TIME_WAIT_FOR_NEW_NODE_UP, - MAX_TIME_WAIT_FOR_DECOMMISSION, - NodeSetupFailed, - NodeSetupTimeout, HOUR_IN_SEC, NodeCleanedAfterDecommissionAborted, + NodeSetupFailed, + NodeSetupTimeout, NodeStayInClusterAfterDecommission, ) from sdcm.cluster_k8s import ( @@ -64,87 +66,142 @@ PodCluster, ) from sdcm.db_stats import PrometheusDBStats +from sdcm.exceptions import ( + AuditLogTestFailure, + BootstrapStreamErrorFailure, + CdcStreamsWasNotUpdated, + FilesNotCorrupted, + KillNemesis, + LdapNotRunning, + LogContentNotFound, + NemesisSubTestFailure, + NoFilesFoundToDestroy, + NoKeyspaceFound, + PartitionNotFound, + QuotaConfigurationFailure, + TimestampNotFound, + UnsupportedNemesis, + WatcherCallableException, +) from sdcm.log import SDCMAdapter from sdcm.logcollector import save_kallsyms_map -from sdcm.mgmt.common import TaskStatus, ScyllaManagerError, get_persistent_snapshots +from sdcm.mgmt.common import ScyllaManagerError, TaskStatus, get_persistent_snapshots from sdcm.nemesis_publisher import NemesisElasticSearchPublisher from sdcm.paths import SCYLLA_YAML_PATH from sdcm.prometheus import nemesis_metrics_obj from sdcm.provision.scylla_yaml import SeedProvider -from sdcm.remote.libssh2_client.exceptions import UnexpectedExit as Libssh2UnexpectedExit +from sdcm.remote.libssh2_client.exceptions import ( + UnexpectedExit as Libssh2UnexpectedExit, +) from sdcm.sct_events import Severity from sdcm.sct_events.database import DatabaseLogEvent from sdcm.sct_events.decorators import raise_event_on_failure -from sdcm.sct_events.filters import DbEventsFilter, EventsSeverityChangerFilter, EventsFilter -from sdcm.sct_events.group_common_events import (ignore_alternator_client_errors, ignore_no_space_errors, - ignore_scrub_invalid_errors, ignore_view_error_gate_closed_exception, - ignore_stream_mutation_fragments_errors, - ignore_ycsb_connection_refused, decorate_with_context, - ignore_reactor_stall_errors, ignore_disk_quota_exceeded_errors, - ignore_error_apply_view_update) +from sdcm.sct_events.filters import ( + DbEventsFilter, + EventsFilter, + EventsSeverityChangerFilter, +) +from sdcm.sct_events.group_common_events import ( + decorate_with_context, + ignore_alternator_client_errors, + ignore_disk_quota_exceeded_errors, + ignore_error_apply_view_update, + ignore_no_space_errors, + ignore_reactor_stall_errors, + ignore_scrub_invalid_errors, + ignore_stream_mutation_fragments_errors, + ignore_view_error_gate_closed_exception, + ignore_ycsb_connection_refused, +) from sdcm.sct_events.health import DataValidatorEvent from sdcm.sct_events.loaders import CassandraStressLogEvent, ScyllaBenchEvent from sdcm.sct_events.nemesis import DisruptionEvent from sdcm.sct_events.system import InfoEvent from sdcm.sla.sla_tests import SlaTests -from sdcm.utils.aws_kms import AwsKms from sdcm.utils import cdc -from sdcm.utils.adaptive_timeouts import adaptive_timeout, Operations -from sdcm.utils.common import (get_db_tables, generate_random_string, - update_certificates, reach_enospc_on_node, clean_enospc_on_node, - parse_nodetool_listsnapshots, - update_authenticator, ParallelObject, - ParallelObjectResult, sleep_for_percent_of_duration, get_views_of_base_table) -from sdcm.utils.quota import configure_quota_on_node_for_scylla_user_context, is_quota_enabled_on_node, enable_quota_on_node, \ - write_data_to_reach_end_of_quota +from sdcm.utils.adaptive_timeouts import Operations, adaptive_timeout +from sdcm.utils.aws_kms import AwsKms +from sdcm.utils.common import ( + ParallelObject, + ParallelObjectResult, + clean_enospc_on_node, + generate_random_string, + get_db_tables, + get_views_of_base_table, + parse_nodetool_listsnapshots, + reach_enospc_on_node, + sleep_for_percent_of_duration, + update_authenticator, + update_certificates, +) from sdcm.utils.compaction_ops import CompactionOps, StartStopCompactionArgs from sdcm.utils.context_managers import nodetool_context -from sdcm.utils.decorators import retrying, latency_calculator_decorator +from sdcm.utils.decorators import latency_calculator_decorator, retrying from sdcm.utils.decorators import timeout as timeout_decor from sdcm.utils.docker_utils import ContainerManager from sdcm.utils.k8s import ( convert_cpu_units_to_k8s_value, - convert_cpu_value_from_k8s_to_units, convert_memory_value_from_k8s_to_units, + convert_cpu_value_from_k8s_to_units, + convert_memory_value_from_k8s_to_units, +) +from sdcm.utils.k8s.chaos_mesh import ( + DiskError, + IOFaultChaosExperiment, + MemoryStressExperiment, + NetworkBandwidthLimitExperiment, + NetworkCorruptExperiment, + NetworkDelayExperiment, + NetworkPacketLossExperiment, ) -from sdcm.utils.k8s.chaos_mesh import MemoryStressExperiment, IOFaultChaosExperiment, DiskError, NetworkDelayExperiment, \ - NetworkPacketLossExperiment, NetworkCorruptExperiment, NetworkBandwidthLimitExperiment from sdcm.utils.ldap import SASLAUTHD_AUTHENTICATOR, LdapServerType -from sdcm.utils.loader_utils import DEFAULT_USER, DEFAULT_USER_PASSWORD, SERVICE_LEVEL_NAME_TEMPLATE -from sdcm.utils.nemesis_utils.indexes import get_random_column_name, create_index, \ - wait_for_index_to_be_built, verify_query_by_index_works, drop_index, get_column_names, \ - wait_for_view_to_be_built, drop_materialized_view, is_cf_a_view +from sdcm.utils.loader_utils import ( + DEFAULT_USER, + DEFAULT_USER_PASSWORD, + SERVICE_LEVEL_NAME_TEMPLATE, +) +from sdcm.utils.nemesis_utils.indexes import ( + create_index, + drop_index, + drop_materialized_view, + get_column_names, + get_random_column_name, + is_cf_a_view, + verify_query_by_index_works, + wait_for_index_to_be_built, + wait_for_view_to_be_built, +) from sdcm.utils.node import build_node_api_command -from sdcm.utils.replication_strategy_utils import temporary_replication_strategy_setter, \ - NetworkTopologyReplicationStrategy, ReplicationStrategy, SimpleReplicationStrategy +from sdcm.utils.quota import ( + configure_quota_on_node_for_scylla_user_context, + enable_quota_on_node, + is_quota_enabled_on_node, + write_data_to_reach_end_of_quota, +) +from sdcm.utils.raft import ( + Group0MembersNotConsistentWithTokenRingMembersException, + TopologyOperations, +) +from sdcm.utils.raft.common import NodeBootstrapAbortManager +from sdcm.utils.replication_strategy_utils import ( + NetworkTopologyReplicationStrategy, + ReplicationStrategy, + SimpleReplicationStrategy, + temporary_replication_strategy_setter, +) from sdcm.utils.sstable.load_utils import SstableLoadUtils from sdcm.utils.sstable.sstable_utils import SstableUtils from sdcm.utils.toppartition_util import NewApiTopPartitionCmd, OldApiTopPartitionCmd from sdcm.utils.version_utils import MethodVersionNotFound, scylla_versions -from sdcm.utils.raft import Group0MembersNotConsistentWithTokenRingMembersException, TopologyOperations -from sdcm.utils.raft.common import NodeBootstrapAbortManager from sdcm.wait import wait_for, wait_for_log_lines -from sdcm.exceptions import ( - KillNemesis, - NoFilesFoundToDestroy, - NoKeyspaceFound, - FilesNotCorrupted, - LogContentNotFound, - LdapNotRunning, - TimestampNotFound, - PartitionNotFound, - WatcherCallableException, - UnsupportedNemesis, - CdcStreamsWasNotUpdated, - NemesisSubTestFailure, - AuditLogTestFailure, - BootstrapStreamErrorFailure, - QuotaConfigurationFailure, +from test_lib.compaction import ( + CompactionStrategy, + GcMode, + get_compaction_random_additional_params, + get_compaction_strategy, + get_gc_mode, ) -from test_lib.compaction import CompactionStrategy, get_compaction_strategy, get_compaction_random_additional_params, \ - get_gc_mode, GcMode from test_lib.cql_types import CQLTypeBuilder -from test_lib.sla import ServiceLevel, MAX_ALLOWED_SERVICE_LEVELS - +from test_lib.sla import MAX_ALLOWED_SERVICE_LEVELS, ServiceLevel LOGGER = logging.getLogger(__name__) # NOTE: following lock is needed in the K8S multitenant case @@ -196,7 +253,7 @@ def __init__(self, tester_obj, termination_event, *args, nemesis_selector=None, # *args - compatible with CategoricalMonkey self.tester = tester_obj # ClusterTester object - self.cluster: Union[BaseCluster, BaseScyllaCluster] = tester_obj.db_cluster + self.cluster: BaseCluster | BaseScyllaCluster = tester_obj.db_cluster self.loaders = tester_obj.loaders self.monitoring_set = tester_obj.monitors self.target_node: BaseNode = None @@ -321,9 +378,9 @@ def unset_current_running_nemesis(node): def _get_target_nodes( self, - is_seed: Optional[Union[bool, DefaultValue]] = DefaultValue, - dc_idx: Optional[int] = None, - rack: Optional[int] = None) -> list: + is_seed: bool | DefaultValue = DefaultValue, + dc_idx: int | None = None, + rack: int | None = None) -> list: """ Filters and return nodes in the cluster that has no running nemesis on them It can filter node by following criteria: is_seed, dc_idx, rack @@ -346,8 +403,8 @@ def _get_target_nodes( nodes = [node for node in nodes if node.rack == rack] return nodes - def set_target_node(self, dc_idx: Optional[int] = None, rack: Optional[int] = None, - is_seed: Union[bool, DefaultValue, None] = DefaultValue, + def set_target_node(self, dc_idx: int | None = None, rack: int | None = None, + is_seed: bool | DefaultValue | None = DefaultValue, allow_only_last_node_in_rack: bool = False): """Set a Scylla node as target node. @@ -417,16 +474,16 @@ def report(self): # pylint: disable=too-many-arguments,unused-argument def get_list_of_methods_compatible_with_backend( # noqa: PLR0913 self, - disruptive: Optional[bool] = None, - run_with_gemini: Optional[bool] = None, - networking: Optional[bool] = None, - limited: Optional[bool] = None, - topology_changes: Optional[bool] = None, - schema_changes: Optional[bool] = None, - config_changes: Optional[bool] = None, - free_tier_set: Optional[bool] = None, - manager_operation: Optional[bool] = None, - ) -> List[str]: + disruptive: bool | None = None, + run_with_gemini: bool | None = None, + networking: bool | None = None, + limited: bool | None = None, + topology_changes: bool | None = None, + schema_changes: bool | None = None, + config_changes: bool | None = None, + free_tier_set: bool | None = None, + manager_operation: bool | None = None, + ) -> list[str]: return self.get_list_of_methods_by_flags( disruptive=disruptive, run_with_gemini=run_with_gemini, @@ -446,18 +503,18 @@ def _is_it_on_kubernetes(self) -> bool: # pylint: disable=too-many-arguments,unused-argument def get_list_of_methods_by_flags( # pylint: disable=too-many-locals # noqa: PLR0913 self, - disruptive: Optional[bool] = None, - run_with_gemini: Optional[bool] = None, - networking: Optional[bool] = None, - kubernetes: Optional[bool] = None, - limited: Optional[bool] = None, - topology_changes: Optional[bool] = None, - schema_changes: Optional[bool] = None, - config_changes: Optional[bool] = None, - free_tier_set: Optional[bool] = None, - sla: Optional[bool] = None, - manager_operation: Optional[bool] = None, - ) -> List[str]: + disruptive: bool | None = None, + run_with_gemini: bool | None = None, + networking: bool | None = None, + kubernetes: bool | None = None, + limited: bool | None = None, + topology_changes: bool | None = None, + schema_changes: bool | None = None, + config_changes: bool | None = None, + free_tier_set: bool | None = None, + sla: bool | None = None, + manager_operation: bool | None = None, + ) -> list[str]: subclasses_list = self._get_subclasses( disruptive=disruptive, run_with_gemini=run_with_gemini, @@ -477,7 +534,7 @@ def get_list_of_methods_by_flags( # pylint: disable=too-many-locals # noqa: PL r'self\.(?Pdisrupt_[A-Za-z_]+?)\(.*\)', inspect.getsource(subclass), flags=re.MULTILINE) if method_name: disrupt_methods_list.append(method_name.group('method_name')) - self.log.debug("Gathered subclass methods: {}".format(disrupt_methods_list)) + self.log.debug(f"Gathered subclass methods: {disrupt_methods_list}") return disrupt_methods_list def get_list_of_subclasses_by_property_name(self, list_of_properties_to_include): @@ -510,14 +567,14 @@ def get_list_of_disrupt_methods(self, subclasses_list, export_properties=False): all_methods_with_properties.append(per_method_properties) all_methods_with_properties = sorted(all_methods_with_properties, key=lambda d: list(d.keys())) nemesis_classes.sort() - self.log.debug("list of matching disrupions: {}".format(disrupt_methods_names_list)) + self.log.debug(f"list of matching disrupions: {disrupt_methods_names_list}") for _ in disrupt_methods_names_list: disrupt_methods_objects_list = [attr[1] for attr in inspect.getmembers(self) if attr[0] in disrupt_methods_names_list and callable(attr[1])] return disrupt_methods_objects_list, all_methods_with_properties, nemesis_classes @classmethod - def _get_subclasses(cls, **flags) -> List[Type['Nemesis']]: + def _get_subclasses(cls, **flags) -> list[type['Nemesis']]: tmp = Nemesis.__subclasses__() subclasses = [] while tmp: @@ -529,8 +586,8 @@ def _get_subclasses(cls, **flags) -> List[Type['Nemesis']]: @staticmethod def _get_subclasses_from_list( - list_of_nemesis: List[Type['Nemesis']], - **flags) -> List[Type['Nemesis']]: + list_of_nemesis: list[type['Nemesis']], + **flags) -> list[type['Nemesis']]: """ It apply 'and' logic to filter, if any value in the filter does not match what nemeses have, @@ -991,14 +1048,13 @@ def replace_full_file_name_to_prefix(self, one_file, ks_cf_for_destroy): try: file_name_template = re.search(r"([^-]+-[^-]+)-", file_name).group(1) except Exception as error: # pylint: disable=broad-except # noqa: BLE001 - self.log.debug('File name "{file_name}" is not as expected for Scylla data files. ' - 'Search files for "{ks_cf_for_destroy}" table'.format(file_name=file_name, - ks_cf_for_destroy=ks_cf_for_destroy)) - self.log.debug('Error: {}'.format(error)) + self.log.debug(f'File name "{file_name}" is not as expected for Scylla data files. ' + f'Search files for "{ks_cf_for_destroy}" table') + self.log.debug(f'Error: {error}') return "" file_for_destroy = one_file.replace(file_name, file_name_template + '-*') - self.log.debug('Selected files for destroy: {}'.format(file_for_destroy)) + self.log.debug(f'Selected files for destroy: {file_for_destroy}') return file_for_destroy @retrying(n=10, allowed_exceptions=(NoKeyspaceFound, NoFilesFoundToDestroy)) @@ -1023,14 +1079,14 @@ def _choose_file_for_destroy(self, ks_cfs, return_one_file=True): if not (file_for_destroy := self.replace_full_file_name_to_prefix(one_file, ks_cf_for_destroy)): continue - self.log.debug('Selected files for destroy: {}'.format(file_for_destroy)) + self.log.debug(f'Selected files for destroy: {file_for_destroy}') if file_for_destroy: if return_one_file: break all_files.append(file_for_destroy) if not file_for_destroy: - raise NoFilesFoundToDestroy('Data file for destroy is not found in {}'.format(ks_cf_for_destroy)) + raise NoFilesFoundToDestroy(f'Data file for destroy is not found in {ks_cf_for_destroy}') return file_for_destroy if return_one_file else all_files @@ -1084,10 +1140,10 @@ def _destroy_data_and_restart_scylla(self, keyspaces_for_destroy: list = None, s result = self.target_node.remoter.sudo('rm -f %s' % file_group_for_destroy) if result.stderr: raise FilesNotCorrupted( - 'Files were not removed. The nemesis can\'t be run. Error: {}'.format(result)) + f'Files were not removed. The nemesis can\'t be run. Error: {result}') all_files_to_destroy.remove(file_for_destroy) sstables_amount_to_destroy -= 1 - self.log.debug('Files {} were destroyed'.format(file_for_destroy)) + self.log.debug(f'Files {file_for_destroy} were destroyed') finally: self.target_node.start_scylla_server(verify_up=True, verify_down=False) @@ -1105,7 +1161,7 @@ def _set_current_disruption(self, label=None, node=None): self.target_node = node if node else self.target_node if not label: - label = "%s on target node %s" % (self.__class__.__name__, self.target_node) + label = f"{self.__class__.__name__} on target node {self.target_node}" self.log.debug('Set current_disruption -> %s', label) self.current_disruption = label @@ -1498,7 +1554,7 @@ def _kubernetes_wait_till_node_up_after_been_recreated(self, node, old_uid=None) def disrupt_terminate_and_replace_node(self): # pylint: disable=invalid-name - def get_node_state(node_ip: str) -> List["str"] | None: + def get_node_state(node_ip: str) -> list["str"] | None: """Gets node state by IP address from nodetool status response""" status = self.cluster.get_nodetool_status() states = [val['state'] for dc in status.values() for ip, val in dc.items() if ip == node_ip] @@ -1666,7 +1722,7 @@ def disrupt_nodetool_enospc(self, sleep_time=30, all_nodes=False): if all_nodes: nodes = self.cluster.nodes - InfoEvent('Enospc test on {}'.format([n.name for n in nodes])).publish() + InfoEvent(f'Enospc test on {[n.name for n in nodes]}').publish() else: nodes = [self.target_node] @@ -1974,7 +2030,7 @@ def disrupt_truncate_large_partition(self): def _modify_table_property(self, name, val, filter_out_table_with_counter=False, keyspace_table=None): disruption_name = "".join([p.strip().capitalize() for p in name.split("_")]) - InfoEvent('ModifyTableProperties%s %s' % (disruption_name, self.target_node)).publish() + InfoEvent(f'ModifyTableProperties{disruption_name} {self.target_node}').publish() if not keyspace_table: self.use_nemesis_seed() @@ -1989,8 +2045,7 @@ def _modify_table_property(self, name, val, filter_out_table_with_counter=False, raise UnsupportedNemesis( 'Non-system keyspace and table are not found. ModifyTableProperties nemesis can\'t be run') - cmd = "ALTER TABLE {keyspace_table} WITH {name} = {val};".format( - keyspace_table=keyspace_table, name=name, val=val) + cmd = f"ALTER TABLE {keyspace_table} WITH {name} = {val};" self.log.debug('_modify_table_property: %s', cmd) with self.cluster.cql_connection_patient(self.target_node) as session: session.execute(cmd) @@ -2132,7 +2187,7 @@ def _add_drop_column(self, drop=True, add=True): # pylint: disable=too-many-bra del added_columns_info['column_names'][column_name] if add: cmd = f"ALTER TABLE {self._add_drop_column_target_table[1]} " \ - f"ADD ( {', '.join(['%s %s' % (col[0], col[1]) for col in add])} );" + f"ADD ( {', '.join([f'{col[0]} {col[1]}' for col in add])} );" if self._add_drop_column_run_cql_query(cmd, self._add_drop_column_target_table[0]): for column_name, column_type in add: added_columns_info['column_names'][column_name] = column_type @@ -2508,7 +2563,7 @@ def toggle_table_ics(self): # pylint: disable=too-many-locals node=self.target_node, ks=keyspace, cf=table) else 'ALTER MATERIALIZED VIEW ' cmd = alter_command_prefix + \ " {keyspace_table} WITH compaction = {new_compaction_strategy_as_dict};".format(**locals()) - self.log.debug("Toggle table ICS query to execute: {}".format(cmd)) + self.log.debug(f"Toggle table ICS query to execute: {cmd}") try: self.target_node.run_cqlsh(cmd) except (UnexpectedExit, Libssh2UnexpectedExit) as unexpected_exit: @@ -2670,7 +2725,7 @@ def modify_table_twcs_window_size(self): # noqa: PLR0915 """ self.use_nemesis_seed() - def set_new_twcs_settings(settings: Dict[str, Any]) -> Dict[str, Any]: + def set_new_twcs_settings(settings: dict[str, Any]) -> dict[str, Any]: """ Recommended number of sstables for twcs is 20 - 30 if number of sstables more than 32, sstables are picked up in bucket by 32 @@ -2966,7 +3021,7 @@ def _mgmt_repair_cli(self): raise ScyllaManagerError( f'Task: {mgr_task.id} final status is: {str(task_final_status)}.\nTask progress string: ' f'{progress_full_string}') - self.log.info('Task: {} is done.'.format(mgr_task.id)) + self.log.info(f'Task: {mgr_task.id} is done.') def disrupt_abort_repair(self): """ @@ -3241,7 +3296,7 @@ def _disrupt_show_toppartitions(self, allow_new_api: bool): sub_cmd='toppartitions', args=top_partition_api.get_cmd_args()) top_partition_api.verify_output(result.stdout) - def get_rate_limit_for_network_disruption(self) -> Optional[str]: + def get_rate_limit_for_network_disruption(self) -> str | None: if not self.monitoring_set.nodes: return None @@ -3266,11 +3321,11 @@ def get_rate_limit_for_network_disruption(self) -> Optional[str]: max_limit = int(round(avg_kbps_per_node * 0.70)) rate_limit_suffix = "kbps" - return "{}{}".format(random.randrange(min_limit, max_limit), rate_limit_suffix) + return f"{random.randrange(min_limit, max_limit)}{rate_limit_suffix}" def _disrupt_network_random_interruptions_k8s(self, list_of_timeout_options): interruptions = ["delay", "loss", "corrupt"] - rate_limit: Optional[str] = self.get_rate_limit_for_network_disruption() + rate_limit: str | None = self.get_rate_limit_for_network_disruption() if not rate_limit: self.log.warning("NetworkRandomInterruption won't limit network bandwidth due to lack of monitoring nodes.") else: @@ -3321,7 +3376,7 @@ def disrupt_network_random_interruptions(self): # pylint: disable=invalid-name if not self.target_node.install_traffic_control(): raise UnsupportedNemesis("Traffic control package not installed on system") - rate_limit: Optional[str] = self.get_rate_limit_for_network_disruption() + rate_limit: str | None = self.get_rate_limit_for_network_disruption() if not rate_limit: self.log.warning("NetworkRandomInterruption won't limit network bandwidth due to lack of monitoring nodes.") @@ -3335,14 +3390,14 @@ def disrupt_network_random_interruptions(self): # pylint: disable=invalid-name delay_in_secs = random.randrange(1, 30) list_of_tc_options = [ - ("NetworkRandomInterruption_{}pct_loss".format(loss_percentage), "--loss {}%".format(loss_percentage)), - ("NetworkRandomInterruption_{}pct_corrupt".format(corrupt_percentage), - "--corrupt {}%".format(corrupt_percentage)), - ("NetworkRandomInterruption_{}sec_delay".format(delay_in_secs), - "--delay {}s --delay-distro 500ms".format(delay_in_secs))] + (f"NetworkRandomInterruption_{loss_percentage}pct_loss", f"--loss {loss_percentage}%"), + (f"NetworkRandomInterruption_{corrupt_percentage}pct_corrupt", + f"--corrupt {corrupt_percentage}%"), + (f"NetworkRandomInterruption_{delay_in_secs}sec_delay", + f"--delay {delay_in_secs}s --delay-distro 500ms")] if rate_limit: list_of_tc_options.append( - ("NetworkRandomInterruption_{}_limit".format(rate_limit), "--rate {}".format(rate_limit))) + (f"NetworkRandomInterruption_{rate_limit}_limit", f"--rate {rate_limit}")) option_name, selected_option = random.choice(list_of_tc_options) wait_time = random.choice(list_of_timeout_options) @@ -3444,7 +3499,7 @@ def remove_node(): self.log.info("Running removenode command on {}, Removing node with the following host_id: {}" .format(rnd_node.ip_address, host_id)) with adaptive_timeout(Operations.REMOVE_NODE, rnd_node, timeout=HOUR_IN_SEC * 48): - res = rnd_node.run_nodetool("removenode {}".format(host_id), ignore_status=True, verbose=True) + res = rnd_node.run_nodetool(f"removenode {host_id}", ignore_status=True, verbose=True) if res.failed and re.match(removenode_reject_msg, res.stdout + res.stderr): raise Exception(f"Removenode was rejected {res.stdout}\n{res.stderr}") @@ -3495,7 +3550,7 @@ def remove_node(): removed_node_status = self.cluster.get_node_status_dictionary( ip_address=node_to_remove.ip_address, verification_node=verification_node) assert removed_node_status is None, \ - "Node was not removed properly (Node status:{})".format(removed_node_status) + f"Node was not removed properly (Node status:{removed_node_status})" # add new node new_node = self._add_and_init_new_cluster_node(rack=self.target_node.rack) @@ -3657,8 +3712,8 @@ def _iptables_randomly_get_disrupting_target(): return 'dropped', f'{target_type}' def _run_commands_wait_and_cleanup( # pylint: disable=too-many-arguments # noqa: PLR0913 - self, node, name: str, start_commands: List[str], - cleanup_commands: List[str] = None, wait_time: int = 0): + self, node, name: str, start_commands: list[str], + cleanup_commands: list[str] = None, wait_time: int = 0): """ Runs command/commands on target node wait and run cleanup commands :param node: target node @@ -3913,10 +3968,10 @@ def _corrupt_data_file(self): # Corrupt data file data_file_pattern = self._choose_file_for_destroy(ks_cfs) - res = self.target_node.remoter.run('sudo find {}-Data.db'.format(data_file_pattern)) + res = self.target_node.remoter.run(f'sudo find {data_file_pattern}-Data.db') for sstable_file in res.stdout.split(): - self.target_node.remoter.run('sudo dd if=/dev/urandom of={} count=1024'.format(sstable_file)) - self.log.debug('File {} was corrupted by dd'.format(sstable_file)) + self.target_node.remoter.run(f'sudo dd if=/dev/urandom of={sstable_file} count=1024') + self.log.debug(f'File {sstable_file} was corrupted by dd') def disrupt_corrupt_then_scrub(self): """ @@ -3938,8 +3993,8 @@ def add_new_node(self, rack=0): def decommission_node(self, node): self.cluster.decommission(node) - def decommission_nodes(self, add_nodes_number, rack, is_seed: Optional[Union[bool, DefaultValue]] = DefaultValue, - dc_idx: Optional[int] = None): + def decommission_nodes(self, add_nodes_number, rack, is_seed: bool | DefaultValue = DefaultValue, + dc_idx: int | None = None): for idx in range(add_nodes_number): if self._is_it_on_kubernetes(): if rack is None and self._is_it_on_kubernetes(): @@ -4358,7 +4413,7 @@ def _alter_table_with_cdc_properties(self, keyspace: str, table: str, cdc_settin :param postimage: is postimage enabled for base table, defaults to False :type postimage: bool, optional :param ttl: set ttl for scylla_cdc_log table, defaults to None - :type ttl: Optional[int], optional + :type ttl: int | None, optional """ cmd = f"ALTER TABLE {keyspace}.{table} WITH cdc = {cdc_settings};" self.log.debug(f"Alter command: {cmd}") @@ -4401,7 +4456,7 @@ def _add_new_node_in_new_dc(self) -> BaseNode: self.monitoring_set.reconfigure_scylla_monitoring() return new_node - def _write_read_data_to_multi_dc_keyspace(self, datacenters: List[str]) -> None: + def _write_read_data_to_multi_dc_keyspace(self, datacenters: list[str]) -> None: InfoEvent(message='Writing and reading data with new dc').publish() write_cmd = f"cassandra-stress write no-warmup cl=ALL n=10000 -schema 'keyspace=keyspace_new_dc " \ f"replication(strategy=NetworkTopologyStrategy,{datacenters[0]}=3,{datacenters[1]}=1) " \ @@ -4418,7 +4473,7 @@ def _verify_multi_dc_keyspace_data(self, consistency_level: str = "ALL"): read_thread = self.tester.run_stress_thread(stress_cmd=read_cmd, round_robin=True, stop_test_on_failure=False) self.tester.verify_stress_thread(cs_thread_pool=read_thread) - def _switch_to_network_replication_strategy(self, keyspaces: List[str]) -> None: + def _switch_to_network_replication_strategy(self, keyspaces: list[str]) -> None: """Switches replication strategy to NetworkTopology for given keyspaces. """ node = self.cluster.nodes[0] @@ -5112,8 +5167,8 @@ def wrapper(*args, **kwargs): # pylint: disable=too-many-statements # noqa: PL args[0].cluster.check_cluster_health() num_nodes_after = len(args[0].cluster.nodes) if num_nodes_before != num_nodes_after: - args[0].log.error('num nodes before %s and nodes after %s does not match' % - (num_nodes_before, num_nodes_after)) + args[0].log.error( + f'num nodes before {num_nodes_before} and nodes after {num_nodes_after} does not match') # TODO: Temporary print. Will be removed later data_validation_prints(args=args) finally: @@ -5480,7 +5535,7 @@ class CategoricalMonkey(Nemesis): """ @staticmethod - def get_disruption_distribution(dist: dict, default_weight: float) -> Tuple[List[Callable], List[float]]: + def get_disruption_distribution(dist: dict, default_weight: float) -> tuple[list[Callable], list[float]]: def is_nonnegative_number(val): try: val = float(val) @@ -5496,9 +5551,9 @@ def prefixed(pref: str, val: str) -> str: all_methods = CategoricalMonkey.get_disrupt_methods() - population: List[Callable] = [] - weights: List[float] = [] - listed_methods: Set[str] = set() + population: list[Callable] = [] + weights: list[float] = [] + listed_methods: set[str] = set() for _name, _weight in dist.items(): name = str(_name) @@ -5528,7 +5583,7 @@ def prefixed(pref: str, val: str) -> str: return population, weights @staticmethod - def get_disrupt_methods() -> Dict[str, Callable]: + def get_disrupt_methods() -> dict[str, Callable]: return {attr[0]: attr[1] for attr in inspect.getmembers(CategoricalMonkey) if attr[0].startswith('disrupt_') and callable(attr[1])} @@ -5702,11 +5757,11 @@ def rollback_node(self, node): 'sudo yum downgrade scylla scylla-server scylla-jmx scylla-tools scylla-conf scylla-kernel-conf scylla-debuginfo -y') # flush all memtables to SSTables node.run_nodetool("drain", timeout=15*60, coredump_on_timeout=True) - node.remoter.run('sudo cp {0}-backup {0}'.format(SCYLLA_YAML_PATH)) + node.remoter.run(f'sudo cp {SCYLLA_YAML_PATH}-backup {SCYLLA_YAML_PATH}') node.remoter.run('sudo systemctl restart scylla-server.service') node.wait_db_up(verbose=True) new_ver = node.remoter.run('rpm -qa scylla-server') - self.log.debug('original scylla-server version is %s, latest: %s' % (orig_ver, new_ver)) + self.log.debug(f'original scylla-server version is {orig_ver}, latest: {new_ver}') if orig_ver == new_ver: raise ValueError('scylla-server version isn\'t changed') diff --git a/sdcm/nemesis_publisher.py b/sdcm/nemesis_publisher.py index 49ad51e07db..32fa04db80d 100644 --- a/sdcm/nemesis_publisher.py +++ b/sdcm/nemesis_publisher.py @@ -12,9 +12,9 @@ # Copyright (c) 2020 ScyllaDB import logging +import sys from datetime import datetime from functools import cached_property -import sys from elasticsearch import Elasticsearch diff --git a/sdcm/node_exporter_setup.py b/sdcm/node_exporter_setup.py index 8b58b3df7ad..a46f58b7b73 100644 --- a/sdcm/node_exporter_setup.py +++ b/sdcm/node_exporter_setup.py @@ -1,6 +1,5 @@ from sdcm.remote import shell_script_cmd - NODE_EXPORTER_VERSION = '1.7.0' diff --git a/sdcm/nosql_thread.py b/sdcm/nosql_thread.py index be07056a611..2dd28eee418 100644 --- a/sdcm/nosql_thread.py +++ b/sdcm/nosql_thread.py @@ -11,15 +11,15 @@ # # Copyright (c) 2021 ScyllaDB -import os import logging +import os +import threading import time import uuid -import threading from sdcm.cluster import BaseNode +from sdcm.sct_events.loaders import NOSQLBENCH_EVENT_PATTERNS, NoSQLBenchStressEvent from sdcm.stress.base import DockerBasedStressThread -from sdcm.sct_events.loaders import NoSQLBenchStressEvent, NOSQLBENCH_EVENT_PATTERNS from sdcm.utils.common import FileFollowerThread LOGGER = logging.getLogger(__name__) @@ -80,8 +80,7 @@ def _run_stress(self, loader, loader_idx, cpu_idx): if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) - log_file_name = os.path.join(loader.logdir, 'nosql-bench-l%s-c%s-%s.log' % - (loader_idx, cpu_idx, uuid.uuid4())) + log_file_name = os.path.join(loader.logdir, f'nosql-bench-{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log') LOGGER.debug('nosql-bench-stress local log: %s', log_file_name) LOGGER.debug("'running: %s", stress_cmd) with NoSQLBenchStressEvent(node=loader, stress_cmd=stress_cmd, log_file_name=log_file_name) as stress_event, \ diff --git a/sdcm/parallel_timeline_report/generate_pt_report.py b/sdcm/parallel_timeline_report/generate_pt_report.py index 20265e2ed07..3e00c2fbf7e 100644 --- a/sdcm/parallel_timeline_report/generate_pt_report.py +++ b/sdcm/parallel_timeline_report/generate_pt_report.py @@ -16,13 +16,12 @@ import logging import re import sys -from pathlib import Path from dataclasses import dataclass, field from datetime import datetime -from typing import List from enum import Enum -from jinja2 import Environment, FileSystemLoader +from pathlib import Path +from jinja2 import Environment, FileSystemLoader LOGGER = logging.getLogger(__name__) @@ -220,7 +219,7 @@ def prepare_stress_event_data(self) -> None: x.stress_cmd)) self._process_chart_data(event_list=stress_events_data_sorted, group_name="Stress events") - def _process_chart_data(self, event_list: List[Event], group_name: str) -> None: + def _process_chart_data(self, event_list: list[Event], group_name: str) -> None: """ The structure of self.chart_data should look like this: [ @@ -273,7 +272,7 @@ def _append_group_data(self, group_name: str, event_data: Event) -> None: event_data.end_timestamp], "val": event_data.chart_value}) - def _process_raw_data(self, events_to_process: list) -> List[Event]: + def _process_raw_data(self, events_to_process: list) -> list[Event]: """ Finds continuous events with 'begin' records only and evaluates 'end_timestamp' for them. If continuous event has both 'begin' and 'end' records, then only 'end' record will be processed. diff --git a/sdcm/prometheus.py b/sdcm/prometheus.py index 46c6463574f..89c332e3014 100644 --- a/sdcm/prometheus.py +++ b/sdcm/prometheus.py @@ -11,21 +11,20 @@ # # Copyright (c) 2020 ScyllaDB -import time -import logging import datetime +import logging import threading -from typing import Optional +import time from http.server import HTTPServer from socketserver import ThreadingMixIn -import requests import prometheus_client +import requests from sdcm.sct_events.base import EventPeriod from sdcm.sct_events.continuous_event import ContinuousEventsRegistry from sdcm.sct_events.monitors import PrometheusAlertManagerEvent -from sdcm.utils.decorators import retrying, log_run_info +from sdcm.utils.decorators import log_run_info, retrying from sdcm.utils.net import get_my_ip START = 'start' @@ -60,7 +59,7 @@ def start_metrics_server(): port = httpd.server_port ip = get_my_ip() LOGGER.info('prometheus API server running on port: %s', port) - return '{}:{}'.format(ip, port) + return f'{ip}:{port}' except Exception as ex: # pylint: disable=broad-except # noqa: BLE001 LOGGER.error('Cannot start local http metrics server: %s', ex) @@ -234,9 +233,9 @@ def run(self): def silence(self, alert_name: str, - duration: Optional[int] = None, - start: Optional[datetime.datetime] = None, - end: Optional[datetime.datetime] = None) -> str: + duration: int | None = None, + start: datetime.datetime | None = None, + end: datetime.datetime | None = None) -> str: """ Silence an alert for a duration of time @@ -288,9 +287,9 @@ class AlertSilencer: def __init__(self, # noqa: PLR0913 alert_manager: PrometheusAlertManagerListener, alert_name: str, - duration: Optional[int] = None, - start: Optional[datetime.datetime] = None, - end: Optional[datetime.datetime] = None): + duration: int | None = None, + start: datetime.datetime | None = None, + end: datetime.datetime | None = None): self.alert_manager = alert_manager self.alert_name = alert_name self.duration = duration or 86400 # 24h diff --git a/sdcm/provision/aws/configuration_script.py b/sdcm/provision/aws/configuration_script.py index 13b70a21737..0dc6f073b70 100644 --- a/sdcm/provision/aws/configuration_script.py +++ b/sdcm/provision/aws/configuration_script.py @@ -11,7 +11,10 @@ # # Copyright (c) 2021 ScyllaDB -from sdcm.provision.aws.utils import network_config_ipv6_workaround_script, configure_eth1_script +from sdcm.provision.aws.utils import ( + configure_eth1_script, + network_config_ipv6_workaround_script, +) from sdcm.provision.common.configuration_script import ConfigurationScriptBuilder diff --git a/sdcm/provision/aws/instance_parameters.py b/sdcm/provision/aws/instance_parameters.py index ea3f1c3022e..cb7ca8d0972 100644 --- a/sdcm/provision/aws/instance_parameters.py +++ b/sdcm/provision/aws/instance_parameters.py @@ -12,7 +12,7 @@ # Copyright (c) 2021 ScyllaDB import base64 -from typing import List, Optional, Literal, Union +from typing import Literal from pydantic import BaseModel @@ -22,7 +22,7 @@ class AWSNetworkInterfaces(BaseModel): DeviceIndex: int SubnetId: str - Groups: List[str] + Groups: list[str] class AWSInstanceProfile(BaseModel): @@ -60,12 +60,12 @@ class AWSInstanceParams(InstanceParamsBase): KeyName: str InstanceType: str UserData: str = None - NetworkInterfaces: List[AWSNetworkInterfaces] = None - IamInstanceProfile: Optional[AWSInstanceProfile] = None - BlockDeviceMappings: List[AWSDiskMapping] = None + NetworkInterfaces: list[AWSNetworkInterfaces] = None + IamInstanceProfile: AWSInstanceProfile | None = None + BlockDeviceMappings: list[AWSDiskMapping] = None Placement: AWSPlacementInfo = None SubnetId: str = None - SecurityGroups: List[str] = None + SecurityGroups: list[str] = None AddressingType: str = None EbsOptimized: bool = None @@ -73,8 +73,8 @@ class AWSInstanceParams(InstanceParamsBase): def dict( # noqa: PLR0913 self, *, - include: Union['AbstractSetIntStr', 'MappingIntStrAny'] = None, - exclude: Union['AbstractSetIntStr', 'MappingIntStrAny'] = None, + include: 'AbstractSetIntStr | MappingIntStrAny' = None, + exclude: 'AbstractSetIntStr | MappingIntStrAny' = None, by_alias: bool = False, skip_defaults: bool = None, exclude_unset: bool = False, diff --git a/sdcm/provision/aws/instance_parameters_builder.py b/sdcm/provision/aws/instance_parameters_builder.py index 011138a665d..54e015a1a6c 100644 --- a/sdcm/provision/aws/instance_parameters_builder.py +++ b/sdcm/provision/aws/instance_parameters_builder.py @@ -12,21 +12,20 @@ # Copyright (c) 2021 ScyllaDB import abc -from typing import List, Optional -from sdcm.provision.aws.instance_parameters import AWSPlacementInfo, AWSDiskMapping +from sdcm.provision.aws.instance_parameters import AWSDiskMapping, AWSPlacementInfo from sdcm.provision.common.builders import AttrBuilder class AWSInstanceParamsBuilderBase(AttrBuilder, metaclass=abc.ABCMeta): @property @abc.abstractmethod - def BlockDeviceMappings(self) -> List[AWSDiskMapping]: # pylint: disable=invalid-name + def BlockDeviceMappings(self) -> list[AWSDiskMapping]: # pylint: disable=invalid-name pass @property @abc.abstractmethod - def ImageId(self) -> Optional[str]: # pylint: disable=invalid-name + def ImageId(self) -> str | None: # pylint: disable=invalid-name pass @property @@ -36,7 +35,7 @@ def KeyName(self) -> str: # pylint: disable=invalid-name @property @abc.abstractmethod - def NetworkInterfaces(self) -> List[dict]: # pylint: disable=invalid-name + def NetworkInterfaces(self) -> list[dict]: # pylint: disable=invalid-name pass @property @@ -51,5 +50,5 @@ def InstanceType(self) -> str: # pylint: disable=invalid-name @property @abc.abstractmethod - def Placement(self) -> Optional[AWSPlacementInfo]: # pylint: disable=invalid-name + def Placement(self) -> AWSPlacementInfo | None: # pylint: disable=invalid-name pass diff --git a/sdcm/provision/aws/provisioner.py b/sdcm/provision/aws/provisioner.py index 72f3f383996..d382763e62e 100644 --- a/sdcm/provision/aws/provisioner.py +++ b/sdcm/provision/aws/provisioner.py @@ -15,18 +15,34 @@ import datetime import logging import time -from typing import List, Optional, Union from mypy_boto3_ec2 import EC2Client from mypy_boto3_ec2.service_resource import Instance +from sdcm.provision.aws.constants import ( + FLEET_LIMIT_EXCEEDED_ERROR, + SPOT_CAPACITY_NOT_AVAILABLE_ERROR, + SPOT_CNT_LIMIT, + SPOT_FLEET_LIMIT, + SPOT_REQUEST_TIMEOUT, + SPOT_STATUS_UNEXPECTED_ERROR, + STATUS_FULFILLED, +) from sdcm.provision.aws.instance_parameters import AWSInstanceParams -from sdcm.provision.aws.utils import ec2_services, ec2_clients, find_instance_by_id, set_tags_on_instances, \ - wait_for_provision_request_done, create_spot_fleet_instance_request, \ - create_spot_instance_request -from sdcm.provision.aws.constants import SPOT_CNT_LIMIT, SPOT_FLEET_LIMIT, SPOT_REQUEST_TIMEOUT, STATUS_FULFILLED, \ - SPOT_STATUS_UNEXPECTED_ERROR, FLEET_LIMIT_EXCEEDED_ERROR, SPOT_CAPACITY_NOT_AVAILABLE_ERROR -from sdcm.provision.common.provisioner import TagsType, ProvisionParameters, InstanceProvisionerBase +from sdcm.provision.aws.utils import ( + create_spot_fleet_instance_request, + create_spot_instance_request, + ec2_clients, + ec2_services, + find_instance_by_id, + set_tags_on_instances, + wait_for_provision_request_done, +) +from sdcm.provision.common.provisioner import ( + InstanceProvisionerBase, + ProvisionParameters, + TagsType, +) LOGGER = logging.getLogger(__name__) @@ -41,8 +57,8 @@ def provision( # pylint: disable=too-many-arguments # noqa: PLR0913 provision_parameters: ProvisionParameters, instance_parameters: AWSInstanceParams, count: int, - tags: Union[List[TagsType], TagsType] = None, - names: List[str] = None) -> List[Instance]: + tags: list[TagsType] | TagsType = None, + names: list[str] = None) -> list[Instance]: if tags is None: tags = {} if isinstance(tags, dict): @@ -101,7 +117,7 @@ def _provision_on_demand_instances( provision_parameters: ProvisionParameters, instance_parameters: AWSInstanceParams, count: int, - tags: List[TagsType]) -> List[Instance]: + tags: list[TagsType]) -> list[Instance]: instance_parameters_dict = instance_parameters.dict( exclude_none=True, exclude_defaults=True, exclude_unset=True, encode_user_data=False) LOGGER.info("[%s] Creating {count} on-demand instances using AMI id '%s' with following parameters:\n%s", @@ -118,7 +134,7 @@ def _provision_on_demand_instances( set_tags_on_instances( region_name=provision_parameters.region_name, instance_ids=[instance.instance_id], - tags={'Name': 'spot_fleet_{}_{}'.format(instance.instance_id, ind)} | instance_tags, + tags={'Name': f'spot_fleet_{instance.instance_id}_{ind}'} | instance_tags, ) return instances @@ -127,7 +143,7 @@ def _provision_spot_instances( provision_parameters: ProvisionParameters, instance_parameters: AWSInstanceParams, count: int, - tags: Union[List[TagsType], TagsType]) -> List[Instance]: + tags: list[TagsType] | TagsType) -> list[Instance]: rest_to_provision = count provisioned_instances = [] while rest_to_provision: @@ -157,7 +173,7 @@ def _execute_spot_fleet_instance_request( provision_parameters: ProvisionParameters, instance_parameters: AWSInstanceParams, count: int, - tags: List[TagsType]) -> List[Instance]: + tags: list[TagsType]) -> list[Instance]: request_id = create_spot_fleet_instance_request( region_name=provision_parameters.region_name, count=count, @@ -186,7 +202,7 @@ def _execute_spot_fleet_instance_request( set_tags_on_instances( region_name=provision_parameters.region_name, instance_ids=[instance_id], - tags={'Name': 'spot_fleet_{}_{}'.format(instance_id, ind)} | instance_tags, + tags={'Name': f'spot_fleet_{instance_id}_{ind}'} | instance_tags, ) self._ec2_client(provision_parameters).cancel_spot_fleet_requests( SpotFleetRequestIds=[request_id], TerminateInstances=False) @@ -196,7 +212,7 @@ def _execute_spot_fleet_instance_request( def _get_provisioned_fleet_instance_ids( self, provision_parameters: ProvisionParameters, - request_ids: List[str]) -> Optional[List[str]]: + request_ids: list[str]) -> list[str] | None: try: resp = self._ec2_client(provision_parameters).describe_spot_fleet_requests(SpotFleetRequestIds=request_ids) LOGGER.info("%s: - %s", request_ids, resp) @@ -250,7 +266,7 @@ def _execute_spot_instance_request( provision_parameters: ProvisionParameters, instance_parameters: AWSInstanceParams, count: int, - tags: List[TagsType]) -> List[Instance]: + tags: list[TagsType]) -> list[Instance]: request_ids = create_spot_instance_request( region_name=provision_parameters.region_name, count=count, @@ -278,7 +294,7 @@ def _execute_spot_instance_request( set_tags_on_instances( region_name=provision_parameters.region_name, instance_ids=[instance_id], - tags={'Name': 'spot_{}_{}'.format(instance_id, ind)} | instance_tags, + tags={'Name': f'spot_{instance_id}_{ind}'} | instance_tags, ) self._ec2_client(provision_parameters).cancel_spot_instance_requests(SpotInstanceRequestIds=request_ids) return [find_instance_by_id( diff --git a/sdcm/provision/aws/utils.py b/sdcm/provision/aws/utils.py index aec25c40f77..95da5c8b116 100644 --- a/sdcm/provision/aws/utils.py +++ b/sdcm/provision/aws/utils.py @@ -15,18 +15,30 @@ import contextlib import datetime import time +from collections.abc import Callable from textwrap import dedent -from typing import Any, Callable, List, Dict, Optional +from typing import Any import boto3 from botocore.exceptions import ClientError -from mypy_boto3_ec2 import EC2ServiceResource, EC2Client +from mypy_boto3_ec2 import EC2Client, EC2ServiceResource from mypy_boto3_ec2.service_resource import Instance -from mypy_boto3_ec2.type_defs import InstanceTypeDef, SpotFleetLaunchSpecificationTypeDef, \ - RequestSpotLaunchSpecificationTypeDef, SpotFleetRequestConfigDataTypeDef - -from sdcm.provision.aws.constants import SPOT_REQUEST_TIMEOUT, SPOT_REQUEST_WAITING_TIME, STATUS_FULFILLED, \ - SPOT_STATUS_UNEXPECTED_ERROR, SPOT_PRICE_TOO_LOW, FLEET_LIMIT_EXCEEDED_ERROR, SPOT_CAPACITY_NOT_AVAILABLE_ERROR +from mypy_boto3_ec2.type_defs import ( + InstanceTypeDef, + RequestSpotLaunchSpecificationTypeDef, + SpotFleetLaunchSpecificationTypeDef, + SpotFleetRequestConfigDataTypeDef, +) + +from sdcm.provision.aws.constants import ( + FLEET_LIMIT_EXCEEDED_ERROR, + SPOT_CAPACITY_NOT_AVAILABLE_ERROR, + SPOT_PRICE_TOO_LOW, + SPOT_REQUEST_TIMEOUT, + SPOT_REQUEST_WAITING_TIME, + SPOT_STATUS_UNEXPECTED_ERROR, + STATUS_FULFILLED, +) from sdcm.provision.common.provisioner import TagsType @@ -83,22 +95,22 @@ def get_subnet_info(region_name: str, subnet_id: str): return [subnet for subnet in resp['Subnets'] if subnet['SubnetId'] == subnet_id][0] -def convert_tags_to_aws_format(tags: TagsType) -> List[Dict[str, str]]: +def convert_tags_to_aws_format(tags: TagsType) -> list[dict[str, str]]: return [{'Key': str(name), 'Value': str(value)} for name, value in tags.items()] -def convert_tags_to_filters(tags: TagsType) -> List[Dict[str, str]]: - return [{'Name': 'tag:{}'.format(name), 'Values': value if isinstance( +def convert_tags_to_filters(tags: TagsType) -> list[dict[str, str]]: + return [{'Name': f'tag:{name}', 'Values': value if isinstance( value, list) else [value]} for name, value in tags.items()] -def find_instance_descriptions_by_tags(region_name: str, tags: TagsType) -> List[InstanceTypeDef]: +def find_instance_descriptions_by_tags(region_name: str, tags: TagsType) -> list[InstanceTypeDef]: client: EC2Client = ec2_clients[region_name] response = client.describe_instances(Filters=convert_tags_to_filters(tags)) return [instance for reservation in response['Reservations'] for instance in reservation['Instances']] -def find_instances_by_tags(region_name: str, tags: TagsType, states: List[str] = None) -> List[Instance]: +def find_instances_by_tags(region_name: str, tags: TagsType, states: list[str] = None) -> list[Instance]: instances = [] for instance_description in find_instance_descriptions_by_tags(region_name=region_name, tags=tags): if states and instance_description['State']['Name'] not in states: @@ -111,7 +123,7 @@ def find_instance_by_id(region_name: str, instance_id: str) -> Instance: return ec2_resources[region_name].Instance(id=instance_id) # pylint: disable=no-member -def set_tags_on_instances(region_name: str, instance_ids: List[str], tags: TagsType): +def set_tags_on_instances(region_name: str, instance_ids: list[str], tags: TagsType): end_time = time.perf_counter() + 20 while end_time > time.perf_counter(): with contextlib.suppress(ClientError): @@ -123,7 +135,7 @@ def set_tags_on_instances(region_name: str, instance_ids: List[str], tags: TagsT def wait_for_provision_request_done( - region_name: str, request_ids: List[str], is_fleet: bool, + region_name: str, request_ids: list[str], is_fleet: bool, timeout: float = SPOT_REQUEST_TIMEOUT, wait_interval: float = SPOT_REQUEST_WAITING_TIME): waiting_time = 0 @@ -142,7 +154,7 @@ def wait_for_provision_request_done( return provisioned_instance_ids -def get_provisioned_fleet_instance_ids(region_name: str, request_ids: List[str]) -> Optional[List[str]]: +def get_provisioned_fleet_instance_ids(region_name: str, request_ids: list[str]) -> list[str] | None: try: resp = ec2_clients[region_name].describe_spot_fleet_requests(SpotFleetRequestIds=request_ids) except Exception: # pylint: disable=broad-except # noqa: BLE001 @@ -174,7 +186,7 @@ def get_provisioned_fleet_instance_ids(region_name: str, request_ids: List[str]) return provisioned_instances -def get_provisioned_spot_instance_ids(region_name: str, request_ids: List[str]) -> Optional[List[str]]: +def get_provisioned_spot_instance_ids(region_name: str, request_ids: list[str]) -> list[str] | None: """ Return list of provisioned instances if all requests where fulfilled if any of the requests failed it will return empty list @@ -220,11 +232,11 @@ def create_spot_fleet_instance_request( # noqa: PLR0913 def create_spot_instance_request( # noqa: PLR0913 region_name: str, count: int, - price: Optional[float], + price: float | None, instance_parameters: RequestSpotLaunchSpecificationTypeDef, full_availability_zone: str, valid_until: datetime.datetime = None, -) -> List[str]: +) -> list[str]: params = { 'DryRun': False, 'InstanceCount': count, diff --git a/sdcm/provision/azure/ip_provider.py b/sdcm/provision/azure/ip_provider.py index d996b37f88b..ad8a5bbf18e 100644 --- a/sdcm/provision/azure/ip_provider.py +++ b/sdcm/provision/azure/ip_provider.py @@ -14,8 +14,6 @@ import logging from dataclasses import dataclass, field -from typing import Dict, List - from azure.core.exceptions import ResourceNotFoundError from azure.mgmt.network.models import PublicIPAddress @@ -30,7 +28,7 @@ class IpAddressProvider: _region: str _az: str _azure_service: AzureService = AzureService() - _cache: Dict[str, PublicIPAddress] = field(default_factory=dict) + _cache: dict[str, PublicIPAddress] = field(default_factory=dict) def __post_init__(self): """Discover existing ip addresses for resource group.""" @@ -42,7 +40,7 @@ def __post_init__(self): except ResourceNotFoundError: pass - def get_or_create(self, names: List[str] = "default", version: str = "IPV4") -> List[PublicIPAddress]: + def get_or_create(self, names: list[str] = "default", version: str = "IPV4") -> list[PublicIPAddress]: addresses = [] pollers = [] for name in names: diff --git a/sdcm/provision/azure/network_interface_provider.py b/sdcm/provision/azure/network_interface_provider.py index eb257e97e9f..14ae73625cd 100644 --- a/sdcm/provision/azure/network_interface_provider.py +++ b/sdcm/provision/azure/network_interface_provider.py @@ -14,8 +14,6 @@ import logging from dataclasses import dataclass, field -from typing import Dict, List - from azure.core.exceptions import ResourceNotFoundError from azure.mgmt.network.models import NetworkInterface @@ -29,7 +27,7 @@ class NetworkInterfaceProvider: _resource_group_name: str _region: str _azure_service: AzureService = AzureService() - _cache: Dict[str, NetworkInterface] = field(default_factory=dict) + _cache: dict[str, NetworkInterface] = field(default_factory=dict) def __post_init__(self): """Discover existing network interfaces for resource group.""" @@ -44,7 +42,7 @@ def __post_init__(self): def get(self, name: str) -> NetworkInterface: return self._cache[self.get_nic_name(name)] - def get_or_create(self, subnet_id: str, ip_addresses_ids: List[str], names: List[str]) -> List[NetworkInterface]: + def get_or_create(self, subnet_id: str, ip_addresses_ids: list[str], names: list[str]) -> list[NetworkInterface]: """Creates or gets (if already exists) network interface""" nics = [] pollers = [] diff --git a/sdcm/provision/azure/network_security_group_provider.py b/sdcm/provision/azure/network_security_group_provider.py index 42057238b41..44e29b98ba1 100644 --- a/sdcm/provision/azure/network_security_group_provider.py +++ b/sdcm/provision/azure/network_security_group_provider.py @@ -12,10 +12,9 @@ # Copyright (c) 2022 ScyllaDB import logging +from collections.abc import Iterable from dataclasses import dataclass, field -from typing import Dict, Iterable, Union, List - from azure.core.exceptions import ResourceNotFoundError from azure.mgmt.network.models import NetworkSecurityGroup @@ -24,7 +23,7 @@ LOGGER = logging.getLogger(__name__) -def rules_to_payload(rules: Iterable) -> List[Dict[str, Union[str, int]]]: +def rules_to_payload(rules: Iterable) -> list[dict[str, str | int]]: """convert iterable rules to format accepted by provisioner""" template = { "name": "", @@ -53,7 +52,7 @@ class NetworkSecurityGroupProvider: _resource_group_name: str _region: str _azure_service: AzureService = AzureService() - _cache: Dict[str, NetworkSecurityGroup] = field(default_factory=dict) + _cache: dict[str, NetworkSecurityGroup] = field(default_factory=dict) def __post_init__(self): """Discover existing security groups for resource group.""" diff --git a/sdcm/provision/azure/provisioner.py b/sdcm/provision/azure/provisioner.py index ac634bab1b1..f7f51c4b807 100644 --- a/sdcm/provision/azure/provisioner.py +++ b/sdcm/provision/azure/provisioner.py @@ -13,8 +13,7 @@ import logging import string -from datetime import datetime, timezone -from typing import Dict, List +from datetime import UTC, datetime, timezone from azure.mgmt.compute.models import VirtualMachine, VirtualMachinePriorityTypes from azure.mgmt.resource.resources.models import ResourceGroup @@ -22,12 +21,19 @@ from sdcm.provision.azure.ip_provider import IpAddressProvider from sdcm.provision.azure.network_interface_provider import NetworkInterfaceProvider -from sdcm.provision.azure.network_security_group_provider import NetworkSecurityGroupProvider +from sdcm.provision.azure.network_security_group_provider import ( + NetworkSecurityGroupProvider, +) from sdcm.provision.azure.resource_group_provider import ResourceGroupProvider from sdcm.provision.azure.subnet_provider import SubnetProvider from sdcm.provision.azure.virtual_machine_provider import VirtualMachineProvider from sdcm.provision.azure.virtual_network_provider import VirtualNetworkProvider -from sdcm.provision.provisioner import Provisioner, InstanceDefinition, VmInstance, PricingModel +from sdcm.provision.provisioner import ( + InstanceDefinition, + PricingModel, + Provisioner, + VmInstance, +) from sdcm.provision.security import ScyllaOpenPorts from sdcm.utils.azure_utils import AzureService @@ -42,7 +48,7 @@ def __init__(self, test_id: str, region: str, availability_zone: str, availability_zone = self._convert_az_to_zone(availability_zone) super().__init__(test_id, region, availability_zone) self._azure_service: AzureService = azure_service - self._cache: Dict[str, VmInstance] = {} + self._cache: dict[str, VmInstance] = {} LOGGER.debug("getting resources for %s...", self._resource_group_name) self._rg_provider = ResourceGroupProvider( self._resource_group_name, self._region, self._az, self._azure_service) @@ -84,13 +90,13 @@ def _get_az_from_name(resource_group: ResourceGroup) -> str: @classmethod def discover_regions(cls, test_id: str = "", regions: list = None, - azure_service: AzureService = AzureService(), **kwargs) -> List["AzureProvisioner"]: + azure_service: AzureService = AzureService(), **kwargs) -> list["AzureProvisioner"]: # pylint: disable=arguments-differ,unused-argument """Discovers provisioners for in each region for given test id. If test_id is not provided, it discovers all related to SCT provisioners.""" - all_resource_groups: List[ResourceGroup] = [ + all_resource_groups: list[ResourceGroup] = [ rg for rg in azure_service.resource.resource_groups.list() if rg.name.startswith("SCT-") and (rg.location in regions if regions else True) @@ -112,9 +118,9 @@ def get_or_create_instance(self, definition: InstanceDefinition, return self.get_or_create_instances(definitions=[definition], pricing_model=pricing_model)[0] def get_or_create_instances(self, - definitions: List[InstanceDefinition], + definitions: list[InstanceDefinition], pricing_model: PricingModel = PricingModel.SPOT - ) -> List[VmInstance]: + ) -> list[VmInstance]: """Create a set of instances specified by a list of InstanceDefinition. If instances already exist, returns them.""" provisioned_vm_instances = [] @@ -156,7 +162,7 @@ def terminate_instance(self, name: str, wait: bool = True) -> None: def reboot_instance(self, name: str, wait: bool, hard: bool = False) -> None: self._vm_provider.reboot(name, wait, hard) - def list_instances(self) -> List[VmInstance]: + def list_instances(self) -> list[VmInstance]: """List virtual machines for given provisioner.""" return list(self._cache.values()) @@ -176,7 +182,7 @@ def cleanup(self, wait: bool = False) -> None: for task in tasks: task.wait() - def add_instance_tags(self, name: str, tags: Dict[str, str]) -> None: + def add_instance_tags(self, name: str, tags: dict[str, str]) -> None: """Adds tags to instance.""" LOGGER.info("Adding tags '%s' to intance '%s'...", tags, name) instance = self._vm_to_instance(self._vm_provider.add_tags(name, tags)) @@ -202,7 +208,7 @@ def _vm_to_instance(self, v_m: VirtualMachine) -> VmInstance: ssh_user = tags.pop("ssh_user", "") ssh_key = tags.pop("ssh_key", "") creation_time = datetime.fromisoformat(tags.pop("creation_time")).replace( - tzinfo=timezone.utc) if 'creation_time' in tags else None + tzinfo=UTC) if 'creation_time' in tags else None image = str(v_m.storage_profile.image_reference) pricing_model = self._get_pricing_model(v_m) diff --git a/sdcm/provision/azure/resource_group_provider.py b/sdcm/provision/azure/resource_group_provider.py index a21cc9b69f1..eedd5d31b7c 100644 --- a/sdcm/provision/azure/resource_group_provider.py +++ b/sdcm/provision/azure/resource_group_provider.py @@ -14,7 +14,6 @@ import logging from dataclasses import dataclass, field from datetime import datetime -from typing import Optional from azure.core.exceptions import ResourceNotFoundError from azure.mgmt.resource.resources.models import ResourceGroup @@ -31,7 +30,7 @@ class ResourceGroupProvider: _region: str _az: str _azure_service: AzureService = AzureService() - _cache: Optional[ResourceGroup] = field(default=None) + _cache: ResourceGroup | None = field(default=None) def __post_init__(self): """Discover existing resource group for this provider.""" diff --git a/sdcm/provision/azure/subnet_provider.py b/sdcm/provision/azure/subnet_provider.py index 2890af6bf8e..77903010e26 100644 --- a/sdcm/provision/azure/subnet_provider.py +++ b/sdcm/provision/azure/subnet_provider.py @@ -14,8 +14,6 @@ import logging from dataclasses import dataclass, field -from typing import Dict - from azure.core.exceptions import ResourceNotFoundError from azure.mgmt.network.models import Subnet @@ -28,7 +26,7 @@ class SubnetProvider: _resource_group_name: str _azure_service: AzureService = AzureService() - _cache: Dict[str, Subnet] = field(default_factory=dict) + _cache: dict[str, Subnet] = field(default_factory=dict) def __post_init__(self): """Discover existing subnets for resource group.""" diff --git a/sdcm/provision/azure/utils.py b/sdcm/provision/azure/utils.py index d36dbe73e3f..4f0e8966ab8 100644 --- a/sdcm/provision/azure/utils.py +++ b/sdcm/provision/azure/utils.py @@ -21,7 +21,6 @@ from sdcm.utils.azure_utils import AzureService from sdcm.utils.version_utils import SCYLLA_VERSION_GROUPED_RE - LOGGER = logging.getLogger(__name__) diff --git a/sdcm/provision/azure/virtual_machine_provider.py b/sdcm/provision/azure/virtual_machine_provider.py index 24bcb6d3433..0a1aa92624b 100644 --- a/sdcm/provision/azure/virtual_machine_provider.py +++ b/sdcm/provision/azure/virtual_machine_provider.py @@ -11,17 +11,16 @@ # # Copyright (c) 2022 ScyllaDB import base64 -import time -from datetime import datetime +import binascii import logging import os +import time from dataclasses import dataclass, field +from datetime import datetime +from typing import Any -from typing import Dict, Optional, Any, List - -import binascii -from azure.core.exceptions import ResourceNotFoundError, AzureError -from azure.mgmt.compute.models import VirtualMachine, RunCommandInput +from azure.core.exceptions import AzureError, ResourceNotFoundError +from azure.mgmt.compute.models import RunCommandInput, VirtualMachine from invoke import Result from sdcm.provision.provisioner import InstanceDefinition, PricingModel, ProvisionError @@ -37,7 +36,7 @@ class VirtualMachineProvider: _region: str _az: str _azure_service: AzureService = AzureService() - _cache: Dict[str, VirtualMachine] = field(default_factory=dict) + _cache: dict[str, VirtualMachine] = field(default_factory=dict) def __post_init__(self): """Discover existing virtual machines for resource group.""" @@ -50,8 +49,8 @@ def __post_init__(self): except ResourceNotFoundError: pass - def get_or_create(self, definitions: List[InstanceDefinition], nics_ids: List[str], pricing_model: PricingModel - ) -> List[VirtualMachine]: + def get_or_create(self, definitions: list[InstanceDefinition], nics_ids: list[str], pricing_model: PricingModel + ) -> list[VirtualMachine]: # pylint: disable=too-many-locals v_ms = [] pollers = [] @@ -156,7 +155,7 @@ def reboot(self, name: str, wait: bool = True, hard: bool = False) -> None: if instance_view and instance_view.statuses[-1].display_status == 'VM running': break - def add_tags(self, name: str, tags: Dict[str, str]) -> VirtualMachine: + def add_tags(self, name: str, tags: dict[str, str]) -> VirtualMachine: """Adds tags to instance (with waiting for completion)""" if name not in self._cache: raise AttributeError(f"Instance '{name}' does not exist in resource group '{self._resource_group_name}'") @@ -172,7 +171,7 @@ def add_tags(self, name: str, tags: Dict[str, str]) -> VirtualMachine: @staticmethod def _get_os_profile(computer_name: str, admin_username: str, - admin_password: str, ssh_public_key: str, custom_data: str) -> Dict[str, Any]: + admin_password: str, ssh_public_key: str, custom_data: str) -> dict[str, Any]: os_profile = {"os_profile": { "computer_name": computer_name, "admin_username": admin_username, @@ -192,7 +191,7 @@ def _get_os_profile(computer_name: str, admin_username: str, return os_profile @staticmethod - def _get_scylla_storage_profile(image_id: str, name: str, disk_size: Optional[int] = None) -> Dict[str, Any]: + def _get_scylla_storage_profile(image_id: str, name: str, disk_size: int | None = None) -> dict[str, Any]: """Creates storage profile based on image_id. image_id may refer to scylla-crafted images (starting with '/subscription') or to 'Urn' of image (see output of e.g. `az vm image list --output table`)""" storage_profile = {"storage_profile": { @@ -264,7 +263,7 @@ def clear_cache(self): self._cache = {} @staticmethod - def _replace_null_value_from_tags_with_empty_string(tags: Dict[str, str]) -> Dict[str, str]: + def _replace_null_value_from_tags_with_empty_string(tags: dict[str, str]) -> dict[str, str]: """Azure API does not accept 'null' as value for tags, so we replace it with empty string.""" for key, value in tags.items(): if value == "null": diff --git a/sdcm/provision/azure/virtual_network_provider.py b/sdcm/provision/azure/virtual_network_provider.py index cb510cd56af..ef2886726bf 100644 --- a/sdcm/provision/azure/virtual_network_provider.py +++ b/sdcm/provision/azure/virtual_network_provider.py @@ -14,8 +14,6 @@ import logging from dataclasses import dataclass, field -from typing import Dict - from azure.core.exceptions import ResourceNotFoundError from azure.mgmt.network.models import VirtualNetwork @@ -30,7 +28,7 @@ class VirtualNetworkProvider: _region: str _az: str _azure_service: AzureService = AzureService() - _cache: Dict[str, VirtualNetwork] = field(default_factory=dict) + _cache: dict[str, VirtualNetwork] = field(default_factory=dict) def __post_init__(self): """Discover existing virtual networks for resource group.""" diff --git a/sdcm/provision/common/builders.py b/sdcm/provision/common/builders.py index f53739182c0..949ae7753eb 100644 --- a/sdcm/provision/common/builders.py +++ b/sdcm/provision/common/builders.py @@ -11,14 +11,9 @@ # # Copyright (c) 2021 ScyllaDB -from typing import Union - from pydantic import BaseModel # pylint: disable=no-name-in-module -OptionalType = type(Union[str, None]) - - class AttrBuilder(BaseModel): @classmethod def get_properties(cls): @@ -35,8 +30,8 @@ def _exclude_by_default(self): def dict( # noqa: PLR0913 self, *, - include: Union['MappingIntStrAny', 'AbstractSetIntStr'] = None, - exclude: Union['MappingIntStrAny', 'AbstractSetIntStr'] = None, + include: 'MappingIntStrAny | AbstractSetIntStr' = None, + exclude: 'MappingIntStrAny | AbstractSetIntStr' = None, by_alias: bool = False, skip_defaults: bool = None, exclude_unset: bool = False, diff --git a/sdcm/provision/common/configuration_script.py b/sdcm/provision/common/configuration_script.py index 6aec58cd33e..a941d3a8d19 100644 --- a/sdcm/provision/common/configuration_script.py +++ b/sdcm/provision/common/configuration_script.py @@ -15,10 +15,17 @@ from sdcm.provision.common.builders import AttrBuilder from sdcm.provision.common.utils import ( - configure_rsyslog_target_script, configure_sshd_script, restart_sshd_service, restart_rsyslog_service, - install_syslogng_service, configure_syslogng_target_script, restart_syslogng_service, - configure_rsyslog_rate_limits_script, configure_rsyslog_set_hostname_script, configure_ssh_accept_rsa) - + configure_rsyslog_rate_limits_script, + configure_rsyslog_set_hostname_script, + configure_rsyslog_target_script, + configure_ssh_accept_rsa, + configure_sshd_script, + configure_syslogng_target_script, + install_syslogng_service, + restart_rsyslog_service, + restart_sshd_service, + restart_syslogng_service, +) RSYSLOG_SSH_TUNNEL_LOCAL_PORT = 5000 RSYSLOG_IMJOURNAL_RATE_LIMIT_INTERVAL = 600 diff --git a/sdcm/provision/common/provision_plan.py b/sdcm/provision/common/provision_plan.py index 4515a1f1f7a..3c96b27de34 100644 --- a/sdcm/provision/common/provision_plan.py +++ b/sdcm/provision/common/provision_plan.py @@ -12,17 +12,21 @@ # Copyright (c) 2021 ScyllaDB import logging -from typing import List from pydantic import BaseModel -from sdcm.provision.common.provisioner import ProvisionParameters, InstanceProvisionerBase, InstanceParamsBase, TagsType +from sdcm.provision.common.provisioner import ( + InstanceParamsBase, + InstanceProvisionerBase, + ProvisionParameters, + TagsType, +) LOGGER = logging.getLogger(__name__) class ProvisionPlan(BaseModel): - provision_steps: List[ProvisionParameters] + provision_steps: list[ProvisionParameters] provisioner: InstanceProvisionerBase @property @@ -31,10 +35,10 @@ def name(self): def provision_instances( self, - instance_parameters: InstanceParamsBase | List[InstanceParamsBase], + instance_parameters: InstanceParamsBase | list[InstanceParamsBase], node_count: int, - node_tags: List[TagsType], - node_names: List[str], + node_tags: list[TagsType], + node_names: list[str], ): for provision_parameters in self.provision_steps: if instances := self.provisioner.provision( diff --git a/sdcm/provision/common/provision_plan_builder.py b/sdcm/provision/common/provision_plan_builder.py index daa5adde9fe..8a5207dd4e7 100644 --- a/sdcm/provision/common/provision_plan_builder.py +++ b/sdcm/provision/common/provision_plan_builder.py @@ -13,13 +13,15 @@ import logging from enum import Enum -from typing import List, Optional from pydantic import BaseModel from sdcm.provision.aws.provisioner import AWSInstanceProvisioner from sdcm.provision.common.provision_plan import ProvisionPlan -from sdcm.provision.common.provisioner import ProvisionParameters, InstanceProvisionerBase +from sdcm.provision.common.provisioner import ( + InstanceProvisionerBase, + ProvisionParameters, +) LOGGER = logging.getLogger(__name__) @@ -40,7 +42,7 @@ class ProvisionPlanBuilder(BaseModel): provisioner: InstanceProvisionerBase @property - def _provision_request_on_demand(self) -> Optional[ProvisionParameters]: + def _provision_request_on_demand(self) -> ProvisionParameters | None: return ProvisionParameters( name='On Demand', spot=False, @@ -68,7 +70,7 @@ def _provision_request_spot_low_price(self) -> ProvisionParameters: ) @property - def _provision_steps(self) -> List[ProvisionParameters]: + def _provision_steps(self) -> list[ProvisionParameters]: if self.initial_provision_type == ProvisionType.ON_DEMAND: return [self._provision_request_on_demand] provision_plan = [] diff --git a/sdcm/provision/common/provisioner.py b/sdcm/provision/common/provisioner.py index 309e8c8ef42..19552289ae8 100644 --- a/sdcm/provision/common/provisioner.py +++ b/sdcm/provision/common/provisioner.py @@ -1,11 +1,9 @@ - import abc -from typing import Dict, Any, List, Union +from typing import Any from pydantic import BaseModel # pylint: disable=no-name-in-module - -TagsType = Dict[str, str] +TagsType = dict[str, str] class ProvisionParameters(BaseModel): # pylint: disable=too-few-public-methods @@ -31,8 +29,8 @@ class InstanceProvisionerBase(BaseModel, metaclass=abc.ABCMeta): # pylint: disa def provision( # pylint: disable=too-many-arguments # noqa: PLR0913 self, provision_parameters: ProvisionParameters, - instance_parameters: InstanceParamsBase | List[InstanceParamsBase], + instance_parameters: InstanceParamsBase | list[InstanceParamsBase], count: int, - tags: Union[List[TagsType], TagsType] = None, - names: List[str] = None) -> List[Any]: + tags: list[TagsType] | TagsType = None, + names: list[str] = None) -> list[Any]: pass diff --git a/sdcm/provision/common/utils.py b/sdcm/provision/common/utils.py index 17d0a2a0798..94e96282c14 100644 --- a/sdcm/provision/common/utils.py +++ b/sdcm/provision/common/utils.py @@ -13,7 +13,6 @@ from textwrap import dedent - # pylint: disable=anomalous-backslash-in-string @@ -39,7 +38,7 @@ def configure_rsyslog_target_script(host: str, port: int) -> str: def configure_syslogng_target_script(host: str, port: int, throttle_per_second: int, hostname: str = "") -> str: - return dedent(""" + return dedent(f""" source_name=`cat /etc/syslog-ng/syslog-ng.conf | tr -d "\\n" | tr -d "\\r" | sed -r "s/\\}};/\\}};\\n/g; \ s/source /\\nsource /g" | grep -P "^source.*system\\(\\)" | cut -d" " -f2` disk_buffer_option="" @@ -87,7 +86,7 @@ def configure_syslogng_target_script(host: str, port: int, throttle_per_second: sed -i -r "s/destination\\(remote_sct\\);[ \\t]*\\}};/destination\\(remote_sct\\); rewrite\\(r_host\\); \\}};/" /etc/syslog-ng/syslog-ng.conf fi fi - """.format(host=host, port=port, hostname=hostname, throttle_per_second=throttle_per_second)) + """) def configure_rsyslog_set_hostname_script(hostname: str) -> str: diff --git a/sdcm/provision/provisioner.py b/sdcm/provision/provisioner.py index cb85b2c17d3..00460611628 100644 --- a/sdcm/provision/provisioner.py +++ b/sdcm/provision/provisioner.py @@ -15,7 +15,6 @@ from dataclasses import dataclass, field from datetime import datetime from enum import Enum -from typing import List, Dict from invoke import Result @@ -42,11 +41,11 @@ class InstanceDefinition: # pylint: disable=too-many-instance-attributes type: str # instance_type from yaml user_name: str ssh_key: SSHKey = field(repr=False) - tags: Dict[str, str] = field(default_factory=dict) + tags: dict[str, str] = field(default_factory=dict) arch: VmArch = VmArch.X86 root_disk_size: int | None = None - data_disks: List[DataDisk] | None = None - user_data: List[UserDataObject] | None = field( + data_disks: list[DataDisk] | None = None + user_data: list[UserDataObject] | None = field( default_factory=list, repr=False) # None when no cloud-init use at all @@ -76,7 +75,7 @@ class VmInstance: # pylint: disable=too-many-instance-attributes ssh_key_name: str public_ip_address: str private_ip_address: str - tags: Dict[str, str] + tags: dict[str, str] pricing_model: PricingModel image: str creation_time: datetime | None @@ -93,7 +92,7 @@ def reboot(self, wait: bool = True, hard: bool = False) -> None: If wait is set to True, waits until machine is up, otherwise, returns when reboot was triggered.""" self._provisioner.reboot_instance(self.name, wait, hard) - def add_tags(self, tags: Dict[str, str]) -> None: + def add_tags(self, tags: dict[str, str]) -> None: """Adds tags to the instance.""" self._provisioner.add_instance_tags(self.name, tags) self.tags.update(tags) @@ -129,7 +128,7 @@ def availability_zone(self) -> str: return self._az @classmethod - def discover_regions(cls, test_id: str) -> List["Provisioner"]: + def discover_regions(cls, test_id: str) -> list["Provisioner"]: """Returns provisioner class instance for each region where resources exist.""" raise NotImplementedError() @@ -142,9 +141,9 @@ def get_or_create_instance(self, raise NotImplementedError() def get_or_create_instances(self, - definitions: List[InstanceDefinition], + definitions: list[InstanceDefinition], pricing_model: PricingModel = PricingModel.SPOT - ) -> List[VmInstance]: + ) -> list[VmInstance]: """Create a set of instances specified by a list of InstanceDefinition. If instances already exist, returns them.""" raise NotImplementedError() @@ -157,7 +156,7 @@ def reboot_instance(self, name: str, wait: bool, hard: bool = False) -> None: """Reboot instance by name. """ raise NotImplementedError() - def list_instances(self) -> List[VmInstance]: + def list_instances(self) -> list[VmInstance]: """List instances for given provisioner.""" raise NotImplementedError() @@ -165,7 +164,7 @@ def cleanup(self, wait: bool = False) -> None: """Cleans up all the resources. If wait == True, waits till cleanup fully completes.""" raise NotImplementedError() - def add_instance_tags(self, name: str, tags: Dict[str, str]) -> None: + def add_instance_tags(self, name: str, tags: dict[str, str]) -> None: """Adds tags to instance.""" raise NotImplementedError() @@ -190,7 +189,7 @@ def create_provisioner(self, backend: str, test_id: str, region: str, availabili f"Register it with provisioner_factory.register_provisioner method.") return provisioner(test_id, region, availability_zone, **config) - def discover_provisioners(self, backend: str, test_id: str, **config) -> List[Provisioner]: + def discover_provisioners(self, backend: str, test_id: str, **config) -> list[Provisioner]: """Discovers regions where resources for given test_id are created. Returning provisioner class instance for each region.""" provisioner = self._classes.get(backend) diff --git a/sdcm/provision/scylla_yaml/__init__.py b/sdcm/provision/scylla_yaml/__init__.py index bf5e742d1c9..ed32b51b273 100644 --- a/sdcm/provision/scylla_yaml/__init__.py +++ b/sdcm/provision/scylla_yaml/__init__.py @@ -11,8 +11,13 @@ # # Copyright (c) 2021 ScyllaDB -from .scylla_yaml import ScyllaYaml -from .auxiliaries import ServerEncryptionOptions, ClientEncryptionOptions, SeedProvider, RequestSchedulerOptions +from .auxiliaries import ( + ClientEncryptionOptions, + RequestSchedulerOptions, + SeedProvider, + ServerEncryptionOptions, +) from .certificate_builder import ScyllaYamlCertificateAttrBuilder from .cluster_builder import ScyllaYamlClusterAttrBuilder from .node_builder import ScyllaYamlNodeAttrBuilder +from .scylla_yaml import ScyllaYaml diff --git a/sdcm/provision/scylla_yaml/auxiliaries.py b/sdcm/provision/scylla_yaml/auxiliaries.py index da10ea9f88f..e1397eb2956 100644 --- a/sdcm/provision/scylla_yaml/auxiliaries.py +++ b/sdcm/provision/scylla_yaml/auxiliaries.py @@ -13,14 +13,13 @@ import logging -from typing import Literal, List, Union, Optional +from typing import Literal -from pydantic import Field, validator, BaseModel # pylint: disable=no-name-in-module +from pydantic import BaseModel, Field, validator # pylint: disable=no-name-in-module from sdcm.provision.common.builders import AttrBuilder from sdcm.sct_config import SCTConfiguration - SEED_PROVIDERS = [ 'org.apache.cassandra.locator.SimpleSeedProvider', 'org.apache.cassandra.locator.GossipingPropertyFileSnitch', @@ -54,7 +53,7 @@ class SeedProvider(BaseModel): # pylint: disable=too-few-public-methods 'GoogleCloudSnitch', 'RackInferringSnitch', ] - parameters: List[dict] = None + parameters: list[dict] = None # pylint: disable=no-self-argument,no-self-use @validator("class_name", pre=True, always=True) @@ -109,7 +108,7 @@ class RequestSchedulerOptions(BaseModel): # pylint: disable=too-few-public-meth class ScyllaYamlAttrBuilderBase(AttrBuilder): - params: Union[SCTConfiguration, dict] = Field(as_dict=False) + params: SCTConfiguration | dict = Field(as_dict=False) @property def _cluster_backend(self) -> str: @@ -127,7 +126,7 @@ def _cloud_provider(self) -> Literal['aws', 'gce', 'azure', None]: return None @property - def _regions(self) -> List[str]: + def _regions(self) -> list[str]: if self._cloud_provider == 'aws': regions = self.params.get('region_name') elif self._cloud_provider == 'gce': @@ -160,7 +159,7 @@ def _intra_node_comm_public(self) -> bool: return self.params.get('intra_node_comm_public') @property - def _authenticator(self) -> Optional[str]: + def _authenticator(self) -> str | None: return self.params.get('authenticator') @property @@ -168,7 +167,7 @@ def _is_authenticator_valid(self) -> bool: return self._authenticator in ['AllowAllAuthenticator', 'PasswordAuthenticator', SASLAUTHD_AUTHENTICATOR] @property - def _authorizer(self) -> Optional[str]: + def _authorizer(self) -> str | None: return self.params.get('authorizer') @property diff --git a/sdcm/provision/scylla_yaml/certificate_builder.py b/sdcm/provision/scylla_yaml/certificate_builder.py index 8285d509f8d..d75aa414a20 100644 --- a/sdcm/provision/scylla_yaml/certificate_builder.py +++ b/sdcm/provision/scylla_yaml/certificate_builder.py @@ -12,13 +12,16 @@ # Copyright (c) 2021 ScyllaDB from functools import cached_property -from typing import Optional, Any +from typing import Any from pydantic import Field from sdcm.provision.helpers.certificate import install_client_certificate -from sdcm.provision.scylla_yaml.auxiliaries import ScyllaYamlAttrBuilderBase, ClientEncryptionOptions, \ - ServerEncryptionOptions +from sdcm.provision.scylla_yaml.auxiliaries import ( + ClientEncryptionOptions, + ScyllaYamlAttrBuilderBase, + ServerEncryptionOptions, +) # Disabling no-member since can't import BaseNode from 'sdcm.cluster' due to a circular import @@ -35,7 +38,7 @@ def _ssl_files_path(self) -> str: return '/etc/scylla/ssl_conf' @property - def client_encryption_options(self) -> Optional[ClientEncryptionOptions]: + def client_encryption_options(self) -> ClientEncryptionOptions | None: if not self.params.get('client_encrypt'): return None return ClientEncryptionOptions( @@ -46,7 +49,7 @@ def client_encryption_options(self) -> Optional[ClientEncryptionOptions]: ) @property - def server_encryption_options(self) -> Optional[ServerEncryptionOptions]: + def server_encryption_options(self) -> ServerEncryptionOptions | None: if not self.params.get('internode_encryption') or not self.params.get('server_encrypt'): return None return ServerEncryptionOptions( diff --git a/sdcm/provision/scylla_yaml/cluster_builder.py b/sdcm/provision/scylla_yaml/cluster_builder.py index d3ef44a2577..a8e484bd8e5 100644 --- a/sdcm/provision/scylla_yaml/cluster_builder.py +++ b/sdcm/provision/scylla_yaml/cluster_builder.py @@ -12,7 +12,7 @@ # Copyright (c) 2021 ScyllaDB from functools import cached_property -from typing import Optional, Any, List +from typing import Any from pydantic import Field @@ -29,7 +29,7 @@ class ScyllaYamlClusterAttrBuilder(ScyllaYamlAttrBuilderBase): msldap_server_info: dict = Field(as_dict=False, default=None) @property - def hinted_handoff_enabled(self) -> Optional[str]: + def hinted_handoff_enabled(self) -> str | None: param_hinted_handoff = str(self.params.get('hinted_handoff')).lower() if param_hinted_handoff in ('enabled', 'true', '1'): return True @@ -38,7 +38,7 @@ def hinted_handoff_enabled(self) -> Optional[str]: return None @property - def experimental_features(self) -> List[str]: + def experimental_features(self) -> list[str]: features = self.params.get('experimental_features') if features is None: return [] @@ -49,29 +49,29 @@ def enable_ipv6_dns_lookup(self) -> bool: return self._is_ip_ssh_connections_ipv6 @property - def authenticator(self) -> Optional[str]: + def authenticator(self) -> str | None: if self._is_authenticator_valid: return self._authenticator return None @property - def saslauthd_socket_path(self) -> Optional[str]: + def saslauthd_socket_path(self) -> str | None: if self._is_authenticator_valid and self.params.get('prepare_saslauthd'): return '/run/saslauthd/mux' return None @property - def authorizer(self) -> Optional[str]: + def authorizer(self) -> str | None: if self._authorizer in ['AllowAllAuthorizer', 'CassandraAuthorizer']: return self._authorizer return None @property - def alternator_port(self) -> Optional[str]: + def alternator_port(self) -> str | None: return self.params.get('alternator_port') @property - def alternator_write_isolation(self) -> Optional[str]: + def alternator_write_isolation(self) -> str | None: return "always_use_lwt" if self.params.get('alternator_port') else None @property @@ -79,11 +79,11 @@ def alternator_enforce_authorization(self) -> bool: return bool(self.params.get('alternator_enforce_authorization')) @property - def internode_compression(self) -> Optional[str]: + def internode_compression(self) -> str | None: return self.params.get('internode_compression') @property - def endpoint_snitch(self) -> Optional[str]: + def endpoint_snitch(self) -> str | None: """ Comes from get_endpoint_snitch """ @@ -94,11 +94,11 @@ def endpoint_snitch(self) -> Optional[str]: return None @property - def ldap_attr_role(self) -> Optional[str]: + def ldap_attr_role(self) -> str | None: return 'cn' if self._is_ldap_authorization else None @property - def ldap_bind_dn(self) -> Optional[str]: + def ldap_bind_dn(self) -> str | None: if self._is_msldap_authorization: return self._ms_ldap_bind_dn if self._is_openldap_authorization: @@ -106,11 +106,11 @@ def ldap_bind_dn(self) -> Optional[str]: return None @property - def role_manager(self) -> Optional[str]: + def role_manager(self) -> str | None: return 'com.scylladb.auth.LDAPRoleManager' if self._is_ldap_authorization else None @property - def ldap_bind_passwd(self) -> Optional[str]: + def ldap_bind_passwd(self) -> str | None: if self._is_msldap_authorization: return self._ms_ldap_bind_passwd if self._is_openldap_authorization: @@ -118,7 +118,7 @@ def ldap_bind_passwd(self) -> Optional[str]: return None @property - def ldap_url_template(self) -> Optional[str]: + def ldap_url_template(self) -> str | None: if self._is_msldap_authorization: server_port = self._ms_ldap_server_address_port ldap_filter = 'member=CN={USER}' diff --git a/sdcm/provision/scylla_yaml/node_builder.py b/sdcm/provision/scylla_yaml/node_builder.py index e7d0531b24c..306f72c3718 100644 --- a/sdcm/provision/scylla_yaml/node_builder.py +++ b/sdcm/provision/scylla_yaml/node_builder.py @@ -12,11 +12,14 @@ # Copyright (c) 2021 ScyllaDB -from typing import Optional, List, Any +from typing import Any from pydantic import Field -from sdcm.provision.scylla_yaml.auxiliaries import ScyllaYamlAttrBuilderBase, SeedProvider +from sdcm.provision.scylla_yaml.auxiliaries import ( + ScyllaYamlAttrBuilderBase, + SeedProvider, +) # Disabling no-member since can't import BaseNode from 'sdcm.cluster' due to a circular import @@ -44,7 +47,7 @@ def _ipv6_ip_address(self) -> str: return self.node.ipv6_ip_address @property - def seed_provider(self) -> Optional[List[SeedProvider]]: + def seed_provider(self) -> list[SeedProvider] | None: if not self._seed_address: return None return [ @@ -55,7 +58,7 @@ def seed_provider(self) -> Optional[List[SeedProvider]]: ] @property - def listen_address(self) -> Optional[str]: + def listen_address(self) -> str | None: if self.params.get('use_dns_names'): return self.node.private_dns_name if self._is_ip_ssh_connections_ipv6: @@ -66,7 +69,7 @@ def listen_address(self) -> Optional[str]: return self._private_ip_address @property - def rpc_address(self) -> Optional[str]: + def rpc_address(self) -> str | None: if self.params.get('use_dns_names'): return self.node.private_dns_name if self._is_ip_ssh_connections_ipv6: @@ -77,7 +80,7 @@ def rpc_address(self) -> Optional[str]: return self._private_ip_address @property - def broadcast_rpc_address(self) -> Optional[str]: + def broadcast_rpc_address(self) -> str | None: if self.params.get('use_dns_names'): return self.node.private_dns_name if self._is_ip_ssh_connections_ipv6: @@ -90,7 +93,7 @@ def broadcast_rpc_address(self) -> Optional[str]: return None @property - def broadcast_address(self) -> Optional[str]: + def broadcast_address(self) -> str | None: if self._is_ip_ssh_connections_ipv6: return self._ipv6_ip_address if self.params.get('extra_network_interface'): @@ -101,7 +104,7 @@ def broadcast_address(self) -> Optional[str]: return None @property - def prometheus_address(self) -> Optional[str]: + def prometheus_address(self) -> str | None: if self._is_ip_ssh_connections_ipv6: return self._ipv6_ip_address return "0.0.0.0" diff --git a/sdcm/provision/scylla_yaml/scylla_yaml.py b/sdcm/provision/scylla_yaml/scylla_yaml.py index 006b8e9ff68..2d6654db682 100644 --- a/sdcm/provision/scylla_yaml/scylla_yaml.py +++ b/sdcm/provision/scylla_yaml/scylla_yaml.py @@ -10,16 +10,20 @@ # See LICENSE for more details. # # Copyright (c) 2021 ScyllaDB +import logging from difflib import unified_diff -from typing import List, Literal, Union +from typing import Literal -import logging import yaml -from pydantic import validator, BaseModel, Extra # pylint: disable=no-name-in-module - -from sdcm.provision.scylla_yaml.auxiliaries import RequestSchedulerOptions, EndPointSnitchType, SeedProvider, \ - ServerEncryptionOptions, ClientEncryptionOptions +from pydantic import BaseModel, Extra, validator # pylint: disable=no-name-in-module +from sdcm.provision.scylla_yaml.auxiliaries import ( + ClientEncryptionOptions, + EndPointSnitchType, + RequestSchedulerOptions, + SeedProvider, + ServerEncryptionOptions, +) logger = logging.getLogger(__name__) @@ -43,7 +47,7 @@ class Config: # pylint: disable=too-few-public-methods listen_interface: str = None # "eth0" listen_interface_prefer_ipv6: bool = None # False commitlog_directory: str = None # "" - data_file_directories: List[str] = None # None + data_file_directories: list[str] = None # None hints_directory: str = None # "" view_hints_directory: str = None # "" saved_caches_directory: str = None # "" @@ -64,7 +68,7 @@ def set_endpoint_snitch(cls, endpoint_snitch: str): rpc_interface: str = None # "eth1" rpc_interface_prefer_ipv6: bool = None # False # [SeedProvider(class_name='org.apache.cassandra.locator.SimpleSeedProvider')] - seed_provider: List[SeedProvider] = None + seed_provider: list[SeedProvider] = None consistent_cluster_management: bool = None # False compaction_throughput_mb_per_sec: int = None # 0 compaction_large_partition_warning_threshold_mb: int = None # 1000 @@ -344,14 +348,14 @@ def set_authorizer(cls, authorizer: str): def dict( # pylint: disable=arguments-differ # noqa: PLR0913 self, *, - include: Union['MappingIntStrAny', 'AbstractSetIntStr'] = None, - exclude: Union['MappingIntStrAny', 'AbstractSetIntStr'] = None, + include: 'MappingIntStrAny | AbstractSetIntStr' = None, + exclude: 'MappingIntStrAny | AbstractSetIntStr' = None, by_alias: bool = False, skip_defaults: bool = None, exclude_defaults: bool = False, exclude_none: bool = False, exclude_unset: bool = False, - explicit: Union['AbstractSetIntStr', 'MappingIntStrAny'] = None, + explicit: 'AbstractSetIntStr | MappingIntStrAny' = None, ) -> 'DictStrAny': to_dict = super().dict( include=include, exclude=exclude, by_alias=by_alias, skip_defaults=skip_defaults, @@ -369,12 +373,12 @@ def _update_dict(self, obj: dict, fields_data: dict): if attr_info and hasattr(attr_info.type_, "__attrs_attrs__"): if attr_value is not None: if not isinstance(attr_value, dict): - raise ValueError("Unexpected data `%s` in attribute `%s`" % ( + raise ValueError("Unexpected data `{}` in attribute `{}`".format( type(attr_value), attr_name)) attr_value = attr_info.type(**attr_value) # noqa: PLW2901 setattr(self, attr_name, attr_value) - def update(self, *objects: Union['ScyllaYaml', dict]): + def update(self, *objects: 'ScyllaYaml | dict'): """ Do the same as dict.update, with one exception. It ignores whatever key if it's value equal to default diff --git a/sdcm/provision/user_data.py b/sdcm/provision/user_data.py index 108e1944beb..d7a98af0d75 100644 --- a/sdcm/provision/user_data.py +++ b/sdcm/provision/user_data.py @@ -14,7 +14,6 @@ import abc from dataclasses import dataclass, field from textwrap import dedent -from typing import List, Dict import yaml @@ -58,10 +57,10 @@ def scylla_machine_image_json(self) -> str: @dataclass class UserDataBuilder: """Generates content for cloud-init""" - user_data_objects: List[UserDataObject] = field(default_factory=list) + user_data_objects: list[UserDataObject] = field(default_factory=list) @property - def yum_repos(self) -> Dict: + def yum_repos(self) -> dict: return { "yum_repos": { @@ -77,7 +76,7 @@ def yum_repos(self) -> Dict: } @property - def apt_configuration(self) -> Dict: + def apt_configuration(self) -> dict: return yaml.safe_load(dedent(""" apt: conf: | diff --git a/sdcm/remote/__init__.py b/sdcm/remote/__init__.py index 752bd91880b..865726e3958 100644 --- a/sdcm/remote/__init__.py +++ b/sdcm/remote/__init__.py @@ -11,12 +11,16 @@ # # Copyright (c) 2020 ScyllaDB +from .base import ( + FailuresWatcher, + RetryableNetworkException, + SSHConnectTimeoutError, + shell_script_cmd, +) from .local_cmd_runner import LocalCmdRunner +from .remote_base import RemoteCmdRunnerBase from .remote_cmd_runner import RemoteCmdRunner from .remote_libssh_cmd_runner import RemoteLibSSH2CmdRunner -from .remote_base import RemoteCmdRunnerBase -from .base import FailuresWatcher, RetryableNetworkException, SSHConnectTimeoutError, shell_script_cmd - __all__ = ( 'LocalCmdRunner', 'RemoteLibSSH2CmdRunner', 'RemoteCmdRunner', 'NETWORK_EXCEPTIONS', 'LOCALRUNNER', diff --git a/sdcm/remote/base.py b/sdcm/remote/base.py index 33db37b2d03..e66172ffbc0 100644 --- a/sdcm/remote/base.py +++ b/sdcm/remote/base.py @@ -11,18 +11,18 @@ # # Copyright (c) 2020 ScyllaDB -from typing import Optional, List, Callable -from abc import abstractmethod, ABCMeta -import shlex import logging -import re import os +import re +import shlex import subprocess +from abc import ABCMeta, abstractmethod +from collections.abc import Callable from textwrap import dedent -from invoke.watchers import StreamWatcher, Responder -from invoke.runners import Result from fabric import Connection +from invoke.runners import Result +from invoke.watchers import Responder, StreamWatcher class OutputCheckError(Exception): @@ -73,15 +73,15 @@ def get_init_arguments(self) -> dict: return {'hostname': self.hostname, 'user': self.user, 'password': self.password} @abstractmethod - def is_up(self, timeout: Optional[float] = None) -> bool: + def is_up(self, timeout: float | None = None) -> bool: """ Return instance parameters required to rebuild instance """ def __str__(self): - return '{} [{}@{}]'.format(self.__class__.__name__, self.user, self.hostname) + return f'{self.__class__.__name__} [{self.user}@{self.hostname}]' - def _setup_watchers(self, verbose: bool, log_file: str, additional_watchers: list) -> List[StreamWatcher]: + def _setup_watchers(self, verbose: bool, log_file: str, additional_watchers: list) -> list[StreamWatcher]: watchers = additional_watchers if additional_watchers else [] if verbose: watchers.append(OutputWatcher(self.log)) @@ -93,13 +93,13 @@ def _setup_watchers(self, verbose: bool, log_file: str, additional_watchers: lis @abstractmethod def run(self, # noqa: PLR0913 cmd: str, - timeout: Optional[float] = None, + timeout: float | None = None, ignore_status: bool = False, verbose: bool = True, new_session: bool = False, - log_file: Optional[str] = None, + log_file: str | None = None, retry: int = 1, - watchers: Optional[List[StreamWatcher]] = None, + watchers: list[StreamWatcher] | None = None, change_context: bool = False ) -> Result: pass @@ -107,14 +107,14 @@ def run(self, # noqa: PLR0913 # pylint: disable=too-many-arguments def sudo(self, # noqa: PLR0913 cmd: str, - timeout: Optional[float] = None, + timeout: float | None = None, ignore_status: bool = False, verbose: bool = True, new_session: bool = False, - log_file: Optional[str] = None, + log_file: str | None = None, retry: int = 1, - watchers: Optional[List[StreamWatcher]] = None, - user: Optional[str] = 'root') -> Result: + watchers: list[StreamWatcher] | None = None, + user: str | None = 'root') -> Result: if user != self.user: if user == 'root': cmd = f"sudo {cmd}" @@ -182,7 +182,7 @@ def _scp_remote_escape(filename: str) -> str: new_name = [] for char in filename: if char in escape_chars: - new_name.append("\\%s" % (char,)) + new_name.append(f"\\{char}") else: new_name.append(char) @@ -277,7 +277,7 @@ def submit_line(self, line: str): self._process_line(line) def _process_line(self, line): - err = 'command failed found {!r} in \n{!r}'.format(self.sentinel, line) + err = f'command failed found {self.sentinel!r} in \n{line!r}' if callable(self.callback): self.callback(self.sentinel, line) if self.raise_exception: diff --git a/sdcm/remote/kubernetes_cmd_runner.py b/sdcm/remote/kubernetes_cmd_runner.py index 85423f12612..fe9af68131f 100644 --- a/sdcm/remote/kubernetes_cmd_runner.py +++ b/sdcm/remote/kubernetes_cmd_runner.py @@ -15,11 +15,11 @@ import logging import threading import time -from typing import Optional, Callable, Iterator, List -import yaml +from collections.abc import Callable, Iterator import kubernetes as k8s -from invoke import Runner, Context, Config +import yaml +from invoke import Config, Context, Runner from invoke.exceptions import ThreadException from urllib3.exceptions import ( MaxRetryError, @@ -27,15 +27,15 @@ ReadTimeoutError, ) -from sdcm.cluster import TestConfig from sdcm import sct_abs_path -from sdcm.utils.k8s import KubernetesOps +from sdcm.cluster import TestConfig from sdcm.utils.common import ( + KeyBasedLock, deprecation, generate_random_string, - KeyBasedLock, ) from sdcm.utils.decorators import retrying +from sdcm.utils.k8s import KubernetesOps from sdcm.wait import wait_for from .base import RetryableNetworkException @@ -46,7 +46,7 @@ def is_scylla_bench_command(command): - return all((str_part in command for str_part in ("scylla-bench", " -workload=", " -mode="))) + return all(str_part in command for str_part in ("scylla-bench", " -workload=", " -mode=")) class KubernetesRunner(Runner): @@ -105,7 +105,7 @@ def kill(self) -> None: def process_is_finished(self) -> bool: return not self.process.is_open() - def returncode(self) -> Optional[int]: + def returncode(self) -> int | None: try: return self.process.returncode except (TypeError, KeyError, ValueError): @@ -128,7 +128,7 @@ class KubernetesCmdRunner(RemoteCmdRunnerBase): default_run_retry = 8 def __init__(self, kluster, pod_image: str, # pylint: disable=too-many-arguments # noqa: PLR0913 - pod_name: str, container: Optional[str] = None, + pod_name: str, container: str | None = None, namespace: str = "default") -> None: self.kluster = kluster self.pod_image = pod_image @@ -187,9 +187,9 @@ def _create_connection(self): "k8s_namespace": self.namespace, }))) # pylint: disable=too-many-arguments - def _run_execute(self, cmd: str, timeout: Optional[float] = None, # pylint: disable=too-many-arguments # noqa: PLR0913 + def _run_execute(self, cmd: str, timeout: float | None = None, # pylint: disable=too-many-arguments # noqa: PLR0913 ignore_status: bool = False, verbose: bool = True, new_session: bool = False, - watchers: Optional[List[StreamWatcher]] = None): + watchers: list[StreamWatcher] | None = None): # TODO: This should be removed than sudo calls will be done in more organized way. tmp = cmd.split(maxsplit=3) if tmp[0] == 'sudo': @@ -375,7 +375,7 @@ def _get_pod_status(self) -> dict: namespace=self.context.config.k8s_namespace).stdout.strip() return yaml.safe_load(result_raw) or {} - def returncode(self, status: dict | None = None) -> Optional[int]: # pylint: disable=arguments-differ + def returncode(self, status: dict | None = None) -> int | None: # pylint: disable=arguments-differ if status is None: status = self._get_pod_status() # NOTE: logic is based on the following conditions: @@ -626,8 +626,8 @@ def send_files(self, src, dst, delete_dst=False, preserve_symlinks=False, verbos return None # Different src file, but the same 'dst' means we do something really wrong raise ValueError( - "Cannot mount '%s' src file to the '%s' dst. " - "It is already used by another src file -> '%s'." % (src, dst, existing_src)) + f"Cannot mount '{src}' src file to the '{dst}' dst. " + f"It is already used by another src file -> '{existing_src}'.") # Generate name for the configMap cm_name = f"cm--{self.pod_name_template}--{generate_random_string(5).lower()}" diff --git a/sdcm/remote/libssh2_client/__init__.py b/sdcm/remote/libssh2_client/__init__.py index c93b99cb2b6..525a9d4e5c6 100644 --- a/sdcm/remote/libssh2_client/__init__.py +++ b/sdcm/remote/libssh2_client/__init__.py @@ -11,28 +11,36 @@ # # Copyright (c) 2020 ScyllaDB -from typing import List, Optional, Dict -from time import perf_counter, sleep -from os.path import normpath, expanduser, exists -from sys import float_info +import ipaddress +from abc import ABC, abstractmethod from io import StringIO -from warnings import warn -from socket import socket, AF_INET, AF_INET6, SOCK_STREAM, gaierror, gethostbyname, error as sock_error -from threading import Thread, Lock, Event, BoundedSemaphore -from abc import abstractmethod, ABC +from os.path import exists, expanduser, normpath from queue import SimpleQueue as Queue -import ipaddress +from socket import AF_INET, AF_INET6, SOCK_STREAM, gaierror, gethostbyname, socket +from sys import float_info +from threading import BoundedSemaphore, Event, Lock, Thread +from time import perf_counter, sleep +from warnings import warn from ssh2.channel import Channel # pylint: disable=no-name-in-module -from ssh2.exceptions import AuthenticationError # pylint: disable=no-name-in-module from ssh2.error_codes import LIBSSH2_ERROR_EAGAIN # pylint: disable=no-name-in-module +from ssh2.exceptions import AuthenticationError # pylint: disable=no-name-in-module -from .exceptions import AuthenticationException, UnknownHostException, ConnectError, PKeyFileError, UnexpectedExit, \ - CommandTimedOut, FailedToReadCommandOutput, ConnectTimeout, FailedToRunCommand, OpenChannelTimeout +from .exceptions import ( + AuthenticationException, + CommandTimedOut, + ConnectError, + ConnectTimeout, + FailedToReadCommandOutput, + FailedToRunCommand, + OpenChannelTimeout, + PKeyFileError, + UnexpectedExit, + UnknownHostException, +) from .result import Result from .session import Session -from .timings import Timings, NullableTiming - +from .timings import NullableTiming, Timings __all__ = ['Session', 'Timings', 'Client', 'Channel', 'FailedToRunCommand'] @@ -309,8 +317,8 @@ def __init__(self, host: str, user: str, password: str = None, # pylint: disabl if flood_preventing is not None: self.flood_preventing = flood_preventing self.channel_lock = Lock() - self.session: Optional[Session] = None - self.sock: Optional[socket] = None + self.session: Session | None = None + self.sock: socket | None = None def __reduce__(self): return self.__class__, ( @@ -368,7 +376,7 @@ def _pkey_auth(self): self.password if self.password is not None else '')) @staticmethod - def _validate_pkey_path(pkey: str, host: str = None) -> Optional[str]: + def _validate_pkey_path(pkey: str, host: str = None) -> str | None: if pkey is None: return None pkey = normpath(expanduser(pkey)) @@ -416,14 +424,14 @@ def _init_socket(self, host: str, port: int): try: self.sock.connect((host, port)) except gaierror as ex: - raise UnknownHostException("Unknown host %s - %s" % (host, str(ex.args[1]))) from ex - except sock_error as ex: + raise UnknownHostException(f"Unknown host {host} - {str(ex.args[1])}") from ex + except OSError as ex: error_type = ex.args[1] if len(ex.args) > 1 else ex.args[0] - raise ConnectError("Error connecting to host '%s:%s' - %s" % (host, port, str(error_type))) from ex + raise ConnectError(f"Error connecting to host '{host}:{port}' - {str(error_type)}") from ex @staticmethod def _process_output( # pylint: disable=too-many-arguments, too-many-branches # noqa: PLR0913, PLR0912 - watchers: List[StreamWatcher], encoding: str, stdout_stream: StringIO, stderr_stream: StringIO, + watchers: list[StreamWatcher], encoding: str, stdout_stream: StringIO, stderr_stream: StringIO, reader: SSHReaderThread, timeout: NullableTiming, timeout_read_data_chunk: NullableTiming): """Separate different approach for the case when watchers are present, since watchers are slow, we can loose data due to the socket buffer limit, if endpoint sending it faster than watchers can read it. @@ -582,7 +590,7 @@ def run( # pylint: disable=unused-argument,too-many-arguments,too-many-locals stdout='', stderr='' ) - channel: Optional[Channel] = None + channel: Channel | None = None try: if self.session is None: self.connect() @@ -620,7 +628,7 @@ def run( # pylint: disable=unused-argument,too-many-arguments,too-many-locals return self._complete_run(channel, exception, timeout_reached, timeout, result, warn, stdout, stderr) @staticmethod - def _apply_env(channel: Channel, env: Dict[str, str]): + def _apply_env(channel: Channel, env: dict[str, str]): if env: for var, val in env.items(): channel.setenv(str(var), str(val)) diff --git a/sdcm/remote/libssh2_client/exceptions.py b/sdcm/remote/libssh2_client/exceptions.py index b6f843cfcd6..27a45dc1003 100644 --- a/sdcm/remote/libssh2_client/exceptions.py +++ b/sdcm/remote/libssh2_client/exceptions.py @@ -17,7 +17,6 @@ from .result import Result - __all__ = ( 'AuthenticationException', 'UnknownHostException', 'ConnectError', 'ConnectTimeout', 'PKeyFileError', 'UnexpectedExit', 'CommandTimedOut', 'FailedToReadCommandOutput', 'OpenChannelTimeout', 'Failure', @@ -121,7 +120,7 @@ def _repr(self, **kwargs) -> str: rest = "" if kwargs: rest = " " + " ".join( - "{}={}".format(key, value) for key, value in kwargs.items() + f"{key}={value}" for key, value in kwargs.items() ) return template.format( self.__class__.__name__, self.result.command, rest diff --git a/sdcm/remote/libssh2_client/result.py b/sdcm/remote/libssh2_client/result.py index feccc8068b2..0928d08b8db 100644 --- a/sdcm/remote/libssh2_client/result.py +++ b/sdcm/remote/libssh2_client/result.py @@ -42,20 +42,18 @@ def __bool__(self): def __str__(self) -> str: if self.exited is not None: - desc = "Command exited with status {}.".format(self.exited) + desc = f"Command exited with status {self.exited}." else: desc = "Command was not fully executed due to watcher error." ret = [desc] for stream in ("stdout", "stderr"): val = getattr(self, stream) ret.append( - """=== {} === -{} -""".format( - stream, val.rstrip() - ) + f"""=== {stream} === +{val.rstrip()} +""" if val - else "(no {})".format(stream) + else f"(no {stream})" ) return "\n".join(ret) diff --git a/sdcm/remote/libssh2_client/session.py b/sdcm/remote/libssh2_client/session.py index b4ae1302b74..bba7643ef13 100644 --- a/sdcm/remote/libssh2_client/session.py +++ b/sdcm/remote/libssh2_client/session.py @@ -15,10 +15,11 @@ from select import select from threading import Lock -from ssh2.session import Session as LibSSH2Session, LIBSSH2_SESSION_BLOCK_INBOUND, LIBSSH2_SESSION_BLOCK_OUTBOUND # pylint: disable=no-name-in-module -from ssh2.exceptions import SocketRecvError # pylint: disable=no-name-in-module -from ssh2.error_codes import LIBSSH2_ERROR_EAGAIN # pylint: disable=no-name-in-module from ssh2.channel import Channel # pylint: disable=no-name-in-module +from ssh2.error_codes import LIBSSH2_ERROR_EAGAIN # pylint: disable=no-name-in-module +from ssh2.exceptions import SocketRecvError # pylint: disable=no-name-in-module +from ssh2.session import LIBSSH2_SESSION_BLOCK_INBOUND, LIBSSH2_SESSION_BLOCK_OUTBOUND +from ssh2.session import Session as LibSSH2Session # pylint: disable=no-name-in-module from .timings import NullableTiming diff --git a/sdcm/remote/libssh2_client/timings.py b/sdcm/remote/libssh2_client/timings.py index bd4732d6876..716b34c34fe 100644 --- a/sdcm/remote/libssh2_client/timings.py +++ b/sdcm/remote/libssh2_client/timings.py @@ -11,11 +11,10 @@ # # Copyright (c) 2020 ScyllaDB -from typing import Iterable, Optional +from collections.abc import Iterable from dataclasses import dataclass, field - -NullableTiming = Optional[float] +NullableTiming = float | None Timing = float Delays = Iterable[float] diff --git a/sdcm/remote/local_cmd_runner.py b/sdcm/remote/local_cmd_runner.py index 9653160fc22..f7bce055c78 100644 --- a/sdcm/remote/local_cmd_runner.py +++ b/sdcm/remote/local_cmd_runner.py @@ -11,16 +11,18 @@ # # Copyright (c) 2020 ScyllaDB -from typing import Optional, List -import os -import time import getpass +import os import socket +import time + from fabric import Connection -from invoke.exceptions import UnexpectedExit, Failure +from invoke.exceptions import Failure, UnexpectedExit from invoke.runners import Result from invoke.watchers import StreamWatcher + from sdcm.utils.decorators import retrying + from .base import CommandRunner, RetryableNetworkException @@ -44,9 +46,9 @@ def _create_connection(self) -> Connection: def is_up(self, timeout: float = None) -> bool: # pylint: disable=no-self-use return True - def run(self, cmd: str, timeout: Optional[float] = None, ignore_status: bool = False, # pylint: disable=too-many-arguments # noqa: PLR0913 - verbose: bool = True, new_session: bool = False, log_file: Optional[str] = None, retry: int = 1, - watchers: Optional[List[StreamWatcher]] = None, change_context: bool = False) -> Result: + def run(self, cmd: str, timeout: float | None = None, ignore_status: bool = False, # pylint: disable=too-many-arguments # noqa: PLR0913 + verbose: bool = True, new_session: bool = False, log_file: str | None = None, retry: int = 1, + watchers: list[StreamWatcher] | None = None, change_context: bool = False) -> Result: watchers = self._setup_watchers(verbose=verbose, log_file=log_file, additional_watchers=watchers) diff --git a/sdcm/remote/remote_base.py b/sdcm/remote/remote_base.py index 8bce8d1c6bd..9da9800d6f6 100644 --- a/sdcm/remote/remote_base.py +++ b/sdcm/remote/remote_base.py @@ -11,22 +11,21 @@ # # Copyright (c) 2020 ScyllaDB -from abc import abstractmethod -from typing import Type, Tuple, List, Optional -from shlex import quote import glob import os import shutil import tempfile -import time import threading +import time +from abc import abstractmethod +from shlex import quote -from invoke.watchers import StreamWatcher from invoke.runners import Result +from invoke.watchers import StreamWatcher from sdcm.utils.decorators import retrying -from .base import RetryableNetworkException, CommandRunner +from .base import CommandRunner, RetryableNetworkException from .local_cmd_runner import LocalCmdRunner @@ -38,11 +37,11 @@ class RemoteCmdRunnerBase(CommandRunner): # pylint: disable=too-many-instance-a auth_sleep_time = 30 _use_rsync = None known_hosts_file = None - default_remoter_class: Type['RemoteCmdRunnerBase'] = None + default_remoter_class: type['RemoteCmdRunnerBase'] = None remoter_classes = {} - exception_unexpected: Type[Exception] = None - exception_failure: Type[Exception] = None - exception_retryable: Tuple[Type[Exception]] = None + exception_unexpected: type[Exception] = None + exception_failure: type[Exception] = None + exception_retryable: tuple[type[Exception]] = None connection_thread_map = threading.local() default_run_retry = 3 @@ -78,7 +77,7 @@ def connection(self): return connection @classmethod - def get_retryable_exceptions(cls) -> Tuple[Type[Exception]]: + def get_retryable_exceptions(cls) -> tuple[type[Exception]]: return cls.exception_retryable def get_init_arguments(self) -> dict: @@ -112,7 +111,7 @@ def set_default_ssh_transport(cls, ssh_transport: str): RemoteCmdRunnerBase.default_remoter_class = remoter_class @staticmethod - def set_default_remoter_class(remoter_class: Type['RemoteCmdRunnerBase']): + def set_default_remoter_class(remoter_class: type['RemoteCmdRunnerBase']): RemoteCmdRunnerBase.default_remoter_class = remoter_class @abstractmethod @@ -152,12 +151,9 @@ def _reconnect(self): def ssh_debug_cmd(self) -> str: if self.key_file: - return "SSH access -> 'ssh -i %s %s@%s'" % (self.key_file, - self.user, - self.hostname) + return f"SSH access -> 'ssh -i {self.key_file} {self.user}@{self.hostname}'" else: - return "SSH access -> 'ssh %s@%s'" % (self.user, - self.hostname) + return f"SSH access -> 'ssh {self.user}@{self.hostname}'" @abstractmethod def is_up(self, timeout: float = 30): @@ -333,7 +329,7 @@ def send_files(self, src: str, # pylint: disable=too-many-arguments,too-many-st dest_is_dir = True if dest_exists and dest_is_dir: - cmd = "rm -rf %s && mkdir %s" % (dst, dst) + cmd = f"rm -rf {dst} && mkdir {dst}" self.run(cmd, verbose=verbose) elif not dest_exists and dest_is_dir: @@ -372,14 +368,14 @@ def _check_rsync(self) -> bool: result = self.run("rsync --version", ignore_status=True) return result.ok - def _encode_remote_paths(self, paths: List[str], escape=True) -> str: + def _encode_remote_paths(self, paths: list[str], escape=True) -> str: """ Given a list of file paths, encodes it as a single remote path, in the style used by rsync and scp. """ if escape: paths = [self._scp_remote_escape(path) for path in paths] - return '%s@[%s]:"%s"' % (self.user, self.hostname, " ".join(paths)) + return '{}@[{}]:"{}"'.format(self.user, self.hostname, " ".join(paths)) def _make_scp_cmd(self, src: str, dst: str, connect_timeout: int = 300, alive_interval: int = 300) -> str: """ @@ -396,7 +392,7 @@ def _make_scp_cmd(self, src: str, dst: str, connect_timeout: int = 300, alive_in return command % (connect_timeout, alive_interval, self.known_hosts_file, self.port, key_option, " ".join(src), dst) - def _make_rsync_compatible_globs(self, pth: str, is_local: bool) -> List[str]: + def _make_rsync_compatible_globs(self, pth: str, is_local: bool) -> list[str]: """ Given an rsync-style path (pth), returns a list of globbed paths. @@ -419,7 +415,7 @@ def glob_matches_files(path, pattern): return glob.glob(path + pattern) else: def glob_matches_files(path, pattern): - match_cmd = "ls \"%s\"%s" % (quote(path), pattern) + match_cmd = f"ls \"{quote(path)}\"{pattern}" result = self.run(match_cmd, ignore_status=True) return result.exit_status == 0 @@ -429,13 +425,13 @@ def glob_matches_files(path, pattern): # convert them into a set of paths suitable for the commandline if is_local: - return ["\"%s\"%s" % (quote(pth), pattern) + return [f"\"{quote(pth)}\"{pattern}" for pattern in patterns] else: return [self._scp_remote_escape(pth) + pattern for pattern in patterns] - def _make_rsync_compatible_source(self, source: List[str], is_local: bool) -> List[str]: + def _make_rsync_compatible_source(self, source: list[str], is_local: bool) -> list[str]: """ Make an rsync compatible source string. @@ -514,9 +510,9 @@ def _make_rsync_cmd( # pylint: disable=too-many-arguments # noqa: PLR0913 return command % (symlink_flag, delete_flag, timeout, ssh_cmd, " ".join(src), dst) - def _run_execute(self, cmd: str, timeout: Optional[float] = None, # pylint: disable=too-many-arguments # noqa: PLR0913 + def _run_execute(self, cmd: str, timeout: float | None = None, # pylint: disable=too-many-arguments # noqa: PLR0913 ignore_status: bool = False, verbose: bool = True, new_session: bool = False, - watchers: Optional[List[StreamWatcher]] = None): + watchers: list[StreamWatcher] | None = None): if verbose: self.log.debug('Running command "%s"...', cmd) start_time = time.perf_counter() @@ -540,9 +536,9 @@ def _run_execute(self, cmd: str, timeout: Optional[float] = None, # pylint: dis result.exit_status = result.exited return result - def _run_pre_run(self, cmd: str, timeout: Optional[float] = None, # pylint: disable=too-many-arguments # noqa: PLR0913 + def _run_pre_run(self, cmd: str, timeout: float | None = None, # pylint: disable=too-many-arguments # noqa: PLR0913 ignore_status: bool = False, verbose: bool = True, new_session: bool = False, - log_file: Optional[str] = None, retry: int = 1, watchers: Optional[List[StreamWatcher]] = None): + log_file: str | None = None, retry: int = 1, watchers: list[StreamWatcher] | None = None): pass @abstractmethod @@ -577,7 +573,7 @@ def run(self, # noqa: PLR0913 new_session: bool = False, log_file: str | None = None, retry: int = 1, - watchers: List[StreamWatcher] | None = None, + watchers: list[StreamWatcher] | None = None, change_context: bool = False ) -> Result: """ diff --git a/sdcm/remote/remote_cmd_runner.py b/sdcm/remote/remote_cmd_runner.py index bd78fe6c2dc..cf45aae5d3d 100644 --- a/sdcm/remote/remote_cmd_runner.py +++ b/sdcm/remote/remote_cmd_runner.py @@ -11,15 +11,14 @@ # # Copyright (c) 2020 ScyllaDB -from typing import Optional, List import os import threading -from fabric import Connection, Config -from paramiko import SSHException -from paramiko.ssh_exception import NoValidConnectionsError, AuthenticationException +from fabric import Config, Connection +from invoke.exceptions import Failure, UnexpectedExit from invoke.watchers import StreamWatcher -from invoke.exceptions import UnexpectedExit, Failure +from paramiko import SSHException +from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError from .base import RetryableNetworkException, SSHConnectTimeoutError from .remote_base import RemoteCmdRunnerBase @@ -29,7 +28,7 @@ class RemoteCmdRunner(RemoteCmdRunnerBase, ssh_transport='fabric', default=True) connection: Connection ssh_config: Config = None ssh_is_up: threading.Event = None - ssh_up_thread: Optional[threading.Thread] = None + ssh_up_thread: threading.Thread | None = None ssh_up_thread_termination: threading.Event = None exception_unexpected = UnexpectedExit exception_failure = Failure @@ -97,13 +96,12 @@ def stop(self): self.stop_ssh_up_thread() super().stop() - def _run_pre_run(self, cmd: str, timeout: Optional[float] = None, # pylint: disable=too-many-arguments # noqa: PLR0913 + def _run_pre_run(self, cmd: str, timeout: float | None = None, # pylint: disable=too-many-arguments # noqa: PLR0913 ignore_status: bool = False, verbose: bool = True, new_session: bool = False, - log_file: Optional[str] = None, retry: int = 1, watchers: Optional[List[StreamWatcher]] = None): + log_file: str | None = None, retry: int = 1, watchers: list[StreamWatcher] | None = None): if not self.is_up(timeout=self.connect_timeout): raise SSHConnectTimeoutError( - 'Unable to run "%s": failed connecting to "%s" during %ss' % - (cmd, self.hostname, self.connect_timeout) + f'Unable to run "{cmd}": failed connecting to "{self.hostname}" during {self.connect_timeout}s' ) def _run_on_retryable_exception(self, exc: Exception, new_session: bool) -> bool: diff --git a/sdcm/remote/remote_file.py b/sdcm/remote/remote_file.py index 588ad04fdbd..5e37dab1611 100644 --- a/sdcm/remote/remote_file.py +++ b/sdcm/remote/remote_file.py @@ -11,10 +11,10 @@ # # Copyright (c) 2020 ScyllaDB -import os +import contextlib import logging +import os import tempfile -import contextlib from io import StringIO import yaml @@ -22,7 +22,6 @@ from sdcm import wait from sdcm.remote import shell_script_cmd - LOGGER = logging.getLogger(__name__) diff --git a/sdcm/remote/remote_libssh_cmd_runner.py b/sdcm/remote/remote_libssh_cmd_runner.py index 60ca8c107c6..caef53a7dd4 100644 --- a/sdcm/remote/remote_libssh_cmd_runner.py +++ b/sdcm/remote/remote_libssh_cmd_runner.py @@ -12,14 +12,24 @@ # Copyright (c) 2020 ScyllaDB import os -import time import socket +import time -from .libssh2_client import Client as LibSSH2Client, Timings -from .libssh2_client.exceptions import AuthenticationException, UnknownHostException, ConnectError, \ - FailedToReadCommandOutput, CommandTimedOut, FailedToRunCommand, OpenChannelTimeout, SocketRecvError, \ - UnexpectedExit, Failure from .base import RetryableNetworkException +from .libssh2_client import Client as LibSSH2Client +from .libssh2_client import Timings +from .libssh2_client.exceptions import ( + AuthenticationException, + CommandTimedOut, + ConnectError, + FailedToReadCommandOutput, + FailedToRunCommand, + Failure, + OpenChannelTimeout, + SocketRecvError, + UnexpectedExit, + UnknownHostException, +) from .remote_base import RemoteCmdRunnerBase diff --git a/sdcm/rest/rest_client.py b/sdcm/rest/rest_client.py index adf6d75317a..390c5d33e25 100644 --- a/sdcm/rest/rest_client.py +++ b/sdcm/rest/rest_client.py @@ -13,13 +13,12 @@ import logging from functools import cached_property -from typing import Dict, Literal +from typing import Literal from urllib.parse import urljoin import requests from requests import Response - LOGGER = logging.getLogger(__name__) @@ -35,13 +34,13 @@ def __init__(self, def _base_url(self) -> str: return urljoin(f"{self._url_prefix}{self._host}", self._endpoint) - def get(self, path: str, params: Dict[str, str] = None) -> Response: + def get(self, path: str, params: dict[str, str] = None) -> Response: url = f"{self._base_url}/{path}" LOGGER.info("Sending a GET request for: %s", url) return requests.get(url=url, params=params) - def post(self, path: str, params: Dict[str, str] = None) -> Response: + def post(self, path: str, params: dict[str, str] = None) -> Response: url = f"{self._base_url}/{path}" LOGGER.info("Sending a POST request for: %s", url) return requests.post(url=url, params=params) diff --git a/sdcm/rest/storage_service_client.py b/sdcm/rest/storage_service_client.py index 43411566573..2ef930bfdf2 100644 --- a/sdcm/rest/storage_service_client.py +++ b/sdcm/rest/storage_service_client.py @@ -11,8 +11,6 @@ # # Copyright (c) 2022 ScyllaDB -from typing import Optional - from fabric.runners import Result from sdcm.cluster import BaseNode @@ -23,19 +21,19 @@ class StorageServiceClient(RemoteCurlClient): def __init__(self, node: BaseNode): super().__init__(host="localhost:10000", endpoint="storage_service", node=node) - def compact_ks_cf(self, keyspace: str, cf: Optional[str] = None) -> Result: + def compact_ks_cf(self, keyspace: str, cf: str | None = None) -> Result: params = {"cf": cf} if cf else {} path = f"keyspace_compaction/{keyspace}" return self.run_remoter_curl(method="POST", path=path, params=params, timeout=360) - def cleanup_ks_cf(self, keyspace: str, cf: Optional[str] = None, timeout: int = 600) -> Result: + def cleanup_ks_cf(self, keyspace: str, cf: str | None = None, timeout: int = 600) -> Result: params = {"cf": cf} if cf else {} path = f"keyspace_cleanup/{keyspace}" return self.run_remoter_curl(method="POST", path=path, params=params, timeout=timeout) - def scrub_ks_cf(self, keyspace: str, cf: Optional[str] = None, scrub_mode: Optional[str] = None) -> Result: + def scrub_ks_cf(self, keyspace: str, cf: str | None = None, scrub_mode: str | None = None) -> Result: params = {"cf": cf} if cf else {} if scrub_mode: @@ -45,7 +43,7 @@ def scrub_ks_cf(self, keyspace: str, cf: Optional[str] = None, scrub_mode: Optio return self.run_remoter_curl(method="GET", path=path, params=params) - def upgrade_sstables(self, keyspace: str = "ks", cf: Optional[str] = None): + def upgrade_sstables(self, keyspace: str = "ks", cf: str | None = None): params = {"cf": cf} if cf else {} path = f"keyspace_upgrade_sstables/{keyspace}" diff --git a/sdcm/results_analyze/__init__.py b/sdcm/results_analyze/__init__.py index cad4cd32170..ffdc06259a1 100644 --- a/sdcm/results_analyze/__init__.py +++ b/sdcm/results_analyze/__init__.py @@ -12,31 +12,36 @@ # Copyright (c) 2021 ScyllaDB # pylint: disable=too-many-lines +import collections import json -import os +import logging import math +import os import pprint -import logging -import collections import re - from datetime import datetime, timedelta from typing import Any -from sortedcontainers import SortedDict import jinja2 +from sortedcontainers import SortedDict -from sdcm.es import ES -from sdcm.test_config import TestConfig from sdcm.db_stats import TestStatsMixin -from sdcm.send_email import Email, BaseEmailReporter +from sdcm.es import ES from sdcm.sct_events import Severity +from sdcm.send_email import BaseEmailReporter, Email +from sdcm.test_config import TestConfig from sdcm.utils.common import format_timestamp -from sdcm.utils.es_queries import QueryFilter, PerformanceFilterYCSB, PerformanceFilterScyllaBench, \ - PerformanceFilterCS, CDCQueryFilterCS, LatencyWithNemesisQueryFilter +from sdcm.utils.es_queries import ( + CDCQueryFilterCS, + LatencyWithNemesisQueryFilter, + PerformanceFilterCS, + PerformanceFilterScyllaBench, + PerformanceFilterYCSB, + QueryFilter, +) from test_lib.utils import MagicList, get_data_by_path -from .test import TestResultClass +from .test import TestResultClass LOGGER = logging.getLogger(__name__) PP = pprint.PrettyPrinter(indent=2) @@ -69,7 +74,7 @@ def get_test_by_id(self, test_id): :return: test results in json format """ if not self._es.exists(index=self._es_index, doc_type=self._es_doc_type, id=test_id): - self.log.error('Test results not found: {}'.format(test_id)) + self.log.error(f'Test results not found: {test_id}') return None return self._es.get(index=self._es_index, doc_type=self._es_doc_type, id=test_id) @@ -179,7 +184,7 @@ def render_to_html(self, results, html_file_path="", template=None): def send_email(self, subject, content, html=True, files=()): if self._email_recipients: - self.log.debug('Send email to {}'.format(self._email_recipients)) + self.log.debug(f'Send email to {self._email_recipients}') email = Email() email.send(subject, content, html=html, recipients=self._email_recipients, files=files) else: @@ -194,7 +199,7 @@ def save_html_to_file(self, results, file_name, template_file): return report_file def gen_kibana_dashboard_url(self, dashboard_path=""): - return "%s/%s" % (self._conf.get('kibana_url'), dashboard_path) + return "{}/{}".format(self._conf.get('kibana_url'), dashboard_path) def save_email_data_file(self, subject, email_data, file_path='email_data.json'): file_path = os.path.join(TestConfig.logdir(), file_path) @@ -203,7 +208,7 @@ def save_email_data_file(self, subject, email_data, file_path='email_data.json') with open(file_path, encoding="utf-8") as file: data = file.read().strip() file_content = json.loads(data or '{}') - except EnvironmentError as err: + except OSError as err: self.log.error('Failed to read file %s with error %s', file_path, err) else: file_content = {} @@ -211,7 +216,7 @@ def save_email_data_file(self, subject, email_data, file_path='email_data.json') try: with open(file_path, 'w', encoding="utf-8") as file: json.dump(file_content, file) - except EnvironmentError as err: + except OSError as err: self.log.error('Failed to write %s to file %s with error %s', file_content, file_path, err) else: self.log.debug('Successfully wrote %s to file %s', file_content, file_path) @@ -521,7 +526,7 @@ def check_regression(self, test_id, stats): # pylint: disable=too-many-locals, # get test res doc = self.get_test_by_id(test_id) if not doc: - self.log.error('Cannot find test by id: {}!'.format(test_id)) + self.log.error(f'Cannot find test by id: {test_id}!') return False test_stats = self._test_stats(doc) @@ -543,10 +548,10 @@ def check_regression(self, test_id, stats): # pylint: disable=too-many-locals, size=self._limit, filter_path=filter_path, ) - self.log.debug("Filtered tests found are: {}".format(tests_filtered)) + self.log.debug(f"Filtered tests found are: {tests_filtered}") if not tests_filtered: - self.log.info('Cannot find tests with the same parameters as {}'.format(test_id)) + self.log.info(f'Cannot find tests with the same parameters as {test_id}') return False cur_test_version = None tested_params = stats.keys() @@ -572,7 +577,7 @@ def check_regression(self, test_id, stats): # pylint: disable=too-many-locals, continue version_info = tag_row['_source']['versions']['scylla-server'] version = version_info['version'] - self.log.debug("version_info={} version={}".format(version_info, version)) + self.log.debug(f"version_info={version_info} version={version}") if tag_row['_id'] == test_id: # save the current test values cur_test_version = version @@ -599,7 +604,7 @@ def check_regression(self, test_id, stats): # pylint: disable=too-many-locals, else: group_by_version[version][param].append(tag_row['_source'][param]) - self.log.debug("group_by_version={}".format(group_by_version)) + self.log.debug(f"group_by_version={group_by_version}") if not cur_test_version: raise ValueError("Could not retrieve current test details from database") @@ -725,7 +730,7 @@ def check_regression(self, test_id, is_gce=False, email_subject_postfix=None, # # get test res doc = self.get_test_by_id(test_id) if not doc: - self.log.error('Cannot find test by id: {}!'.format(test_id)) + self.log.error(f'Cannot find test by id: {test_id}!') return False self.log.debug(PP.pformat(doc)) @@ -747,7 +752,7 @@ def check_regression(self, test_id, is_gce=False, email_subject_postfix=None, # size=self._limit, request_timeout=30) if not tests_filtered: - self.log.info('Cannot find tests with the same parameters as {}'.format(test_id)) + self.log.info(f'Cannot find tests with the same parameters as {test_id}') return False # get the best res for all versions of this job group_by_version = {} @@ -816,7 +821,7 @@ def check_regression(self, test_id, is_gce=False, email_subject_postfix=None, # for version, group in group_by_version.items(): if version == test_version and not group_by_version[test_version]['tests']: - self.log.info('No previous tests in the current version {} to compare'.format(test_version)) + self.log.info(f'No previous tests in the current version {test_version} to compare') continue cmp_res = self.cmp(test_stats, group['stats_best'], version, group['best_test_id']) latest_version_test = group["tests"].peekitem(index=-1)[1] @@ -904,7 +909,7 @@ def check_regression_with_subtest_baseline(self, test_id, base_test_id, subtest_ doc = self.get_test_by_id(test_id) if not doc: - self.log.error('Cannot find test by id: {}!'.format(test_id)) + self.log.error(f'Cannot find test by id: {test_id}!') return False self.log.debug(PP.pformat(doc)) @@ -933,7 +938,7 @@ def check_regression_with_subtest_baseline(self, test_id, base_test_id, subtest_ ) if not tests_filtered: - self.log.info('Cannot find tests with the same parameters as {}'.format(test_id)) + self.log.info(f'Cannot find tests with the same parameters as {test_id}') return False # get the best res for all versions of this job group_by_version_sub_type = SortedDict() @@ -1047,7 +1052,7 @@ def check_regression_with_subtest_baseline(self, test_id, base_test_id, subtest_ cmp_res = {} for sub_type, tests in group.items(): if not tests['tests']: - self.log.info('No previous tests in the current version {} to compare'.format(test_version)) + self.log.info(f'No previous tests in the current version {test_version} to compare') continue if sub_type not in current_tests: continue @@ -1107,7 +1112,7 @@ def get_test_instance_by_id(self, test_id): if rp_main_test: rp_main_test = rp_main_test[0] if not rp_main_test or not rp_main_test.is_valid(): - self.log.error('Cannot find main_test by id: {}!'.format(test_id)) + self.log.error(f'Cannot find main_test by id: {test_id}!') return None return rp_main_test @@ -1275,12 +1280,12 @@ def check_regression_multi_baseline( # noqa: PLR0912, PLR0915 rp_main_test = self.get_test_instance_by_id(test_id) if not rp_main_test: - self.log.error('Cannot find test with id: {}!'.format(test_id)) + self.log.error(f'Cannot find test with id: {test_id}!') return False if subject is None: - subject = 'Performance Regression Compare Results - {test.test_name} - ' \ - '{test.software.scylla_server_any.version.as_string}'.format(test=rp_main_test) + subject = f'Performance Regression Compare Results - {rp_main_test.test_name} - ' \ + f'{rp_main_test.software.scylla_server_any.version.as_string}' else: subject = subject.format(test=rp_main_test) @@ -1298,7 +1303,7 @@ def check_regression_multi_baseline( # noqa: PLR0912, PLR0915 ]) if not prior_main_tests: - self.log.error('Cannot find prior runs for test with id: {}!'.format(test_id)) + self.log.error(f'Cannot find prior runs for test with id: {test_id}!') return False # Get all subtests of the current main test and sort them by subtest name @@ -1308,7 +1313,7 @@ def check_regression_multi_baseline( # noqa: PLR0912, PLR0915 ]).sort_by('subtest_name') if not rp_subtests_of_current_test: - self.log.error('Cannot find subtests for test id: {}!'.format(test_id)) + self.log.error(f'Cannot find subtests for test id: {test_id}!') return False if not subtests_info: diff --git a/sdcm/results_analyze/test.py b/sdcm/results_analyze/test.py index 0680ed5ba91..e61c2291c6d 100644 --- a/sdcm/results_analyze/test.py +++ b/sdcm/results_analyze/test.py @@ -11,16 +11,16 @@ # # Copyright (c) 2021 ScyllaDB +import logging import re import typing from datetime import datetime -import logging from sdcm.es import ES from test_lib.utils import get_class_by_path -from .base import ClassBase, __DEFAULT__ -from .metrics import ScyllaTestMetrics +from .base import __DEFAULT__, ClassBase +from .metrics import ScyllaTestMetrics LOGGER = logging.getLogger(__name__) @@ -239,7 +239,7 @@ class SoftwareVersions(ClassBase): scylla_enterprise_server: ScyllaEnterpriseVersionInfo = None @property - def scylla_server_any(self) -> typing.Union[ScyllaVersionInfo, ScyllaEnterpriseVersionInfo]: + def scylla_server_any(self) -> ScyllaVersionInfo | ScyllaEnterpriseVersionInfo: return self.scylla_server if self.scylla_server else self.scylla_enterprise_server def is_valid(self): @@ -520,7 +520,7 @@ def get_metric_class(cls, metric_path, default=__DEFAULT__): def gen_kibana_dashboard_url( dashboard_path="app/kibana#/dashboard/03414b70-0e89-11e9-a976-2fe0f5890cd0?_g=()" ): - return "%s/%s" % (ES()._conf.get('kibana_url'), dashboard_path) # pylint: disable=protected-access + return "{}/{}".format(ES()._conf.get('kibana_url'), dashboard_path) # pylint: disable=protected-access def get_subtests(self): return self.get_by_params(es_index=self.es_index, main_test_id=self.test_id, subtest_name='*') @@ -555,7 +555,7 @@ def get_same_tests_query(self): ]) return self._get_es_query_from_self(list_of_attributes) - def get_prior_tests(self, filter_path=None) -> typing.List['TestResultClass']: + def get_prior_tests(self, filter_path=None) -> list['TestResultClass']: output = [] try: es_query = self.get_same_tests_query() diff --git a/sdcm/scan_operation_thread.py b/sdcm/scan_operation_thread.py index 9eeb93840c9..1d1cf25c4f9 100644 --- a/sdcm/scan_operation_thread.py +++ b/sdcm/scan_operation_thread.py @@ -8,19 +8,36 @@ import traceback from abc import abstractmethod from string import Template -from typing import Optional, Type, NamedTuple, TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple + from cassandra import ConsistencyLevel, OperationTimedOut, ReadTimeout -from cassandra.cluster import ResponseFuture, ResultSet # pylint: disable=no-name-in-module +from cassandra.cluster import ( # pylint: disable=no-name-in-module + ResponseFuture, + ResultSet, +) from cassandra.query import SimpleStatement # pylint: disable=no-name-in-module + +from sdcm.db_stats import PrometheusDBStats from sdcm.remote import LocalCmdRunner from sdcm.sct_events import Severity -from sdcm.sct_events.database import FullScanEvent, FullPartitionScanReversedOrderEvent, FullPartitionScanEvent, \ - FullScanAggregateEvent -from sdcm.utils.database_query_utils import get_table_clustering_order, get_partition_keys -from sdcm.utils.operations_thread import OperationThreadStats, OneOperationStat, OperationThread, ThreadParams -from sdcm.db_stats import PrometheusDBStats +from sdcm.sct_events.database import ( + FullPartitionScanEvent, + FullPartitionScanReversedOrderEvent, + FullScanAggregateEvent, + FullScanEvent, +) from sdcm.test_config import TestConfig -from sdcm.utils.decorators import retrying, Retry +from sdcm.utils.database_query_utils import ( + get_partition_keys, + get_table_clustering_order, +) +from sdcm.utils.decorators import Retry, retrying +from sdcm.utils.operations_thread import ( + OneOperationStat, + OperationThread, + OperationThreadStats, + ThreadParams, +) if TYPE_CHECKING: from sdcm.cluster import BaseNode @@ -86,8 +103,8 @@ def __init__(self, thread_params: ThreadParams, thread_name: str = ""): class FullscanOperationBase: def __init__(self, generator: random.Random, thread_params: ThreadParams, thread_stats: OperationThreadStats, - scan_event: Type[FullScanEvent] | Type[FullPartitionScanEvent] - | Type[FullPartitionScanReversedOrderEvent] | Type[FullScanAggregateEvent]): + scan_event: type[FullScanEvent] | type[FullPartitionScanEvent] + | type[FullPartitionScanReversedOrderEvent] | type[FullScanAggregateEvent]): """ Base class for performing fullscan operations. """ @@ -112,7 +129,7 @@ def randomly_form_cql_statement(self) -> str: def execute_query( self, session, cmd: str, - event: Type[FullScanEvent | FullPartitionScanEvent + event: type[FullScanEvent | FullPartitionScanEvent | FullPartitionScanReversedOrderEvent]) -> ResultSet: # pylint: disable=unused-argument self.log.debug('Will run command %s', cmd) @@ -123,7 +140,7 @@ def execute_query( ) def run_scan_event(self, cmd: str, - scan_event: Type[FullScanEvent | FullPartitionScanEvent + scan_event: type[FullScanEvent | FullPartitionScanEvent | FullPartitionScanReversedOrderEvent] = None) -> OneOperationStat: scan_event = scan_event or self.scan_event cmd = cmd or self.randomly_form_cql_statement() @@ -246,7 +263,7 @@ def get_table_clustering_order(self) -> str: error) raise Exception('Failed getting table clustering order from all db nodes') - def randomly_form_cql_statement(self) -> Optional[tuple[str, str]]: # pylint: disable=too-many-branches + def randomly_form_cql_statement(self) -> tuple[str, str] | None: # pylint: disable=too-many-branches """ The purpose of this method is to form a random reversed-query out of all options, like: select * from scylla_bench.test where pk = 1234 and ck < 4721 and ck > 2549 order by ck desc @@ -341,7 +358,7 @@ def fetch_result_pages(self, result: ResponseFuture, read_pages): def execute_query( self, session, cmd: str, - event: Type[FullScanEvent | FullPartitionScanEvent + event: type[FullScanEvent | FullPartitionScanEvent | FullPartitionScanReversedOrderEvent]) -> ResponseFuture: self.log.debug('Will run command "%s"', cmd) session.default_fetch_size = self.fullscan_params.page_size @@ -448,7 +465,7 @@ def randomly_form_cql_statement(self) -> str: return cmd def execute_query(self, session, cmd: str, - event: Type[FullScanEvent | FullPartitionScanEvent + event: type[FullScanEvent | FullPartitionScanEvent | FullPartitionScanReversedOrderEvent]) -> None: self.log.debug('Will run command %s', cmd) validate_forward_service_requests_start_time = time.time() diff --git a/sdcm/sct_config.py b/sdcm/sct_config.py index 860c2303694..49f35818f9a 100644 --- a/sdcm/sct_config.py +++ b/sdcm/sct_config.py @@ -14,48 +14,47 @@ """ Handling Scylla-cluster-test configuration loading """ +import ast +import getpass import json +import logging + # pylint: disable=too-many-lines import os -import re -import ast -import logging -import getpass import pathlib +import re import tempfile -from typing import List, Union, Set - from distutils.util import strtobool import anyconfig -from sdcm import sct_abs_path import sdcm.provision.azure.utils as azure_utils +from sdcm import sct_abs_path +from sdcm.remote import LOCALRUNNER, shell_script_cmd +from sdcm.sct_events.base import add_severity_limit_rules, print_critical_events from sdcm.utils import alternator from sdcm.utils.aws_utils import get_arch_from_instance_type from sdcm.utils.common import ( ami_built_by_scylla, + convert_name_to_ami_if_needed, get_ami_tags, get_branched_ami, get_branched_gce_images, + get_sct_root_path, get_scylla_ami_versions, get_scylla_gce_images_versions, - convert_name_to_ami_if_needed, - get_sct_root_path, ) +from sdcm.utils.gce_utils import get_gce_image_tags from sdcm.utils.operations_thread import ConfigParams from sdcm.utils.version_utils import ( + find_scylla_repo, get_branch_version, get_branch_version_for_multiple_repositories, get_scylla_docker_repo_from_version, - resolve_latest_repo_symlink, get_specific_tag_of_docker_image, - find_scylla_repo, is_enterprise, + resolve_latest_repo_symlink, ) -from sdcm.sct_events.base import add_severity_limit_rules, print_critical_events -from sdcm.utils.gce_utils import get_gce_image_tags -from sdcm.remote import LOCALRUNNER, shell_script_cmd def _str(value: str) -> str: @@ -71,7 +70,7 @@ def _file(value: str) -> str: raise ValueError(f"{value} isn't an existing file") -def str_or_list(value: Union[str, List[str], List[List[str]]]) -> List[str]: +def str_or_list(value: str | list[str] | list[list[str]]) -> list[str]: if isinstance(value, str): return [value] if isinstance(value, list): @@ -84,7 +83,7 @@ def str_or_list(value: Union[str, List[str], List[List[str]]]) -> List[str]: raise ValueError(f"{value} isn't a string or a list of strings.") -def str_or_list_or_eval(value: Union[str, List[str]]) -> List[str]: +def str_or_list_or_eval(value: str | list[str]) -> list[str]: """Convert an environment variable into a Python's list.""" if isinstance(value, str): @@ -125,7 +124,7 @@ def int_or_list(value): except Exception: # pylint: disable=broad-except # noqa: BLE001 pass - raise ValueError("{} isn't int or list".format(value)) + raise ValueError(f"{value} isn't int or list") def dict_or_str(value): @@ -137,7 +136,7 @@ def dict_or_str(value): if isinstance(value, dict): return value - raise ValueError('"{}" isn\'t a dict'.format(value)) + raise ValueError(f'"{value}" isn\'t a dict') def boolean(value): @@ -146,7 +145,7 @@ def boolean(value): elif isinstance(value, str): return bool(strtobool(value)) else: - raise ValueError("{} isn't a boolean".format(type(value))) + raise ValueError(f"{type(value)} isn't a boolean") class SCTConfiguration(dict): @@ -1676,7 +1675,7 @@ def __init__(self): # noqa: PLR0912, PLR0915 if key not in self.keys(): self[key] = value elif len(self[key].split()) < len(region_names): - self[key] += " {}".format(value) + self[key] += f" {value}" else: for region in region_names: if region not in self.aws_supported_regions: @@ -1817,7 +1816,7 @@ def __init__(self): # noqa: PLR0912, PLR0915 version_tag = self.get('ami_id_db_scylla_desc') or getpass.getuser() user_prefix = self.get('user_prefix') or getpass.getuser() if version_tag != user_prefix: - self['user_prefix'] = "{}-{}".format(user_prefix, version_tag)[:35] + self['user_prefix'] = f"{user_prefix}-{version_tag}"[:35] else: self['user_prefix'] = user_prefix[:35] @@ -1878,7 +1877,7 @@ def log_config(self): self.log.info(self.dump_config()) @property - def region_names(self) -> List[str]: + def region_names(self) -> list[str]: region_names = self.environment.get('region_name') if region_names is None: region_names = self.get('region_name') @@ -1892,7 +1891,7 @@ def region_names(self) -> List[str]: return output @property - def gce_datacenters(self) -> List[str]: + def gce_datacenters(self) -> list[str]: gce_datacenters = self.environment.get('gce_datacenter') if gce_datacenters is None: gce_datacenters = self.get('gce_datacenter') @@ -2007,11 +2006,11 @@ def _validate_value(self, opt): opt['name'], cur_val_element, choices) @property - def list_of_stress_tools(self) -> Set[str]: + def list_of_stress_tools(self) -> set[str]: stress_tools = set() for param_name in self.stress_cmd_params: stress_cmds = self.get(param_name) - if not (isinstance(stress_cmds, (list, str)) and stress_cmds): + if not (isinstance(stress_cmds, list | str) and stress_cmds): continue if isinstance(stress_cmds, str): stress_cmds = [stress_cmds] @@ -2126,7 +2125,7 @@ def _check_unexpected_sct_variables(self): env_keys = {o.split('.')[0] for o in os.environ if o.startswith('SCT_')} unknown_env_keys = env_keys.difference(config_keys) if unknown_env_keys: - output = ["{}={}".format(key, os.environ.get(key)) for key in unknown_env_keys] + output = [f"{key}={os.environ.get(key)}" for key in unknown_env_keys] raise ValueError("Unsupported environment variables were used:\n\t - {}".format("\n\t - ".join(output))) # check for unsupported configuration @@ -2136,7 +2135,7 @@ def _check_unexpected_sct_variables(self): if unsupported_option: res = "Unsupported config option/s found:\n" for option in unsupported_option: - res += "\t * '{}: {}'\n".format(option, self[option]) + res += f"\t * '{option}: {self[option]}'\n" raise ValueError(res) def _validate_sct_variable_values(self): @@ -2157,7 +2156,7 @@ def _check_multi_region_params(self, backend): else: region_count[opt] = 1 if not all(region_count[current_region_param_name] == x for x in region_count.values()): - raise ValueError("not all multi region values are equal: \n\t{}".format(region_count)) + raise ValueError(f"not all multi region values are equal: \n\t{region_count}") def _validate_seeds_number(self): seeds_num = self.get('seeds_num') @@ -2203,7 +2202,7 @@ def _check_per_backend_required_values(self, backend: str): backend += "-siren" self._check_backend_defaults(backend, self.backend_required_params[backend]) else: - raise ValueError("Unsupported backend [{}]".format(backend)) + raise ValueError(f"Unsupported backend [{backend}]") def _check_backend_defaults(self, backend, required_params): opts = [o for o in self.config_options if o['name'] in required_params] @@ -2427,7 +2426,7 @@ def dump_help_config_yaml(self): ret = "" for opt in self.config_options: if opt['help']: - help_text = '\n'.join(["# {}".format(l.strip()) for l in opt['help'].splitlines() if l.strip()]) + '\n' + help_text = '\n'.join([f"# {l.strip()}" for l in opt['help'].splitlines() if l.strip()]) + '\n' else: help_text = '' default = self.get_default_value(opt['name']) diff --git a/sdcm/sct_events/__init__.py b/sdcm/sct_events/__init__.py index 5740f7cb19d..a27cccdbb62 100644 --- a/sdcm/sct_events/__init__.py +++ b/sdcm/sct_events/__init__.py @@ -14,7 +14,7 @@ import enum import json import logging -from typing import Protocol, Optional, Type, runtime_checkable +from typing import Protocol, runtime_checkable class Severity(enum.Enum): @@ -29,9 +29,9 @@ class Severity(enum.Enum): @runtime_checkable class SctEventProtocol(Protocol): base: str - type: Optional[str] - subtype: Optional[str] - event_timestamp: Optional[float] + type: str | None + subtype: str | None + event_timestamp: float | None severity: Severity # pylint: disable=super-init-not-called @@ -47,7 +47,7 @@ def add_subevent_type(cls, name: str, /, *, abstract: bool = False, - mixin: Optional[Type] = None, + mixin: type | None = None, **kwargs) -> None: ... @@ -62,7 +62,7 @@ def timestamp(self) -> float: def publish(self, warn_not_ready: bool = True) -> None: ... - def publish_or_dump(self, default_logger: Optional[logging.Logger] = None, warn_not_ready: bool = True) -> None: + def publish_or_dump(self, default_logger: logging.Logger | None = None, warn_not_ready: bool = True) -> None: ... def to_json(self) -> str: diff --git a/sdcm/sct_events/base.py b/sdcm/sct_events/base.py index 847f6eecb25..c75b782d3d5 100644 --- a/sdcm/sct_events/base.py +++ b/sdcm/sct_events/base.py @@ -15,27 +15,33 @@ from __future__ import annotations +import fnmatch import json +import logging +import pickle import time import uuid -import pickle -import fnmatch -import logging +from collections.abc import Callable +from datetime import UTC, datetime, timezone from enum import Enum +from functools import cached_property, partialmethod from json import JSONEncoder -from types import new_class # pylint: disable=no-name-in-module -from typing import \ - Any, Optional, Type, Dict, List, Tuple, Callable, Generic, TypeVar, Protocol, runtime_checkable from keyword import iskeyword +from types import new_class # pylint: disable=no-name-in-module +from typing import ( + Any, + Generic, + Protocol, + TypeVar, + runtime_checkable, +) from weakref import proxy as weakproxy -from datetime import datetime, timezone -from functools import partialmethod, cached_property -import yaml import dateutil.parser +import yaml from sdcm import sct_abs_path -from sdcm.sct_events import Severity, SctEventProtocol +from sdcm.sct_events import SctEventProtocol, Severity from sdcm.sct_events.events_processes import EventsProcessesRegistry DEFAULT_SEVERITIES = sct_abs_path("defaults/severities.yaml") @@ -43,19 +49,19 @@ LOGGER = logging.getLogger(__name__) -class SctEventTypesRegistry(Dict[str, Type["SctEvent"]]): # pylint: disable=too-few-public-methods +class SctEventTypesRegistry(dict[str, type["SctEvent"]]): # pylint: disable=too-few-public-methods def __init__(self, severities_conf: str = DEFAULT_SEVERITIES): super().__init__() with open(severities_conf, encoding="utf-8") as fobj: self.max_severities = {event_t: Severity[sev] for event_t, sev in yaml.safe_load(fobj).items()} self.limit_rules = [] - def __setitem__(self, key: str, value: Type[SctEvent]): + def __setitem__(self, key: str, value: type[SctEvent]): if not value.is_abstract() and key not in self.max_severities: raise ValueError(f"There is no max severity configured for {key}") super().__setitem__(key, weakproxy(value)) # pylint: disable=no-member; pylint doesn't know about Dict - def __set_name__(self, owner: Type[SctEvent], name: str) -> None: + def __set_name__(self, owner: type[SctEvent], name: str) -> None: self[owner.__name__] = owner # add owner class to the registry. @@ -79,20 +85,20 @@ class EventPeriod(Enum): class SctEvent: _sct_event_types_registry: SctEventTypesRegistry = SctEventTypesRegistry() - _events_processes_registry: Optional[EventsProcessesRegistry] = None + _events_processes_registry: EventsProcessesRegistry | None = None _abstract: bool = True # this attribute set by __init_subclass__() base: str = "SctEvent" # this attribute set by __init_subclass__() - type: Optional[str] = None # this attribute set by add_subevent_type() - subtype: Optional[str] = None # this attribute set by add_subevent_type() - subcontext: Optional[List[dict]] = None # this attribute keeps context of event + type: str | None = None # this attribute set by add_subevent_type() + subtype: str | None = None # this attribute set by add_subevent_type() + subcontext: list[dict] | None = None # this attribute keeps context of event period_type: str = EventPeriod.NOT_DEFINED.value # attribute possible values are from EventTypes enum formatter: Callable[[str, SctEvent], str] = staticmethod(str.format) - event_timestamp: Optional[float] = None # actual value should be set using __init__() - source_timestamp: Optional[float] = None + event_timestamp: float | None = None # actual value should be set using __init__() + source_timestamp: float | None = None severity: Severity = Severity.UNKNOWN # actual value should be set using __init__() _ready_to_publish: bool = False # set it to True in __init__() and to False in publish() to prevent double-publish @@ -115,7 +121,7 @@ def __new__(cls, *_, **__): def __init__(self, severity: Severity = Severity.UNKNOWN): self.event_timestamp = time.time() - self.source_timestamp: Optional[float] = None + self.source_timestamp: float | None = None self.severity = severity self._ready_to_publish = True self.event_id = str(uuid.uuid4()) @@ -131,7 +137,7 @@ def add_subevent_type(cls, name: str, /, *, abstract: bool = False, - mixin: Optional[Type] = None, + mixin: type | None = None, **kwargs) -> None: # Check if we can add a new sub-event type: @@ -169,7 +175,7 @@ def add_subevent_type(cls, @staticmethod def _formatted_timestamp(timestamp: float) -> str: try: - return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + return datetime.fromtimestamp(timestamp, tz=UTC).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] except (TypeError, OverflowError, OSError,): LOGGER.exception("Failed to format a timestamp: %r", timestamp) return "0000-00-00 " @@ -253,7 +259,7 @@ def publish(self, warn_not_ready: bool = True) -> None: get_events_main_device(_registry=self._events_processes_registry).publish_event(self) self._ready_to_publish = False - def publish_or_dump(self, default_logger: Optional[logging.Logger] = None, warn_not_ready: bool = True) -> None: + def publish_or_dump(self, default_logger: logging.Logger | None = None, warn_not_ready: bool = True) -> None: # pylint: disable=import-outside-toplevel; to avoid cyclic imports from sdcm.sct_events.events_device import get_events_main_device @@ -290,7 +296,7 @@ def subcontext_fields(self) -> list: """ return [] - def to_json(self, encoder: Type[JSONEncoder] = JSONEncoder) -> str: + def to_json(self, encoder: type[JSONEncoder] = JSONEncoder) -> str: return json.dumps({ **self.attribute_with_value_for_json(attributes_list=["base", "type", "subtype"]), **self.__getstate__(), @@ -335,7 +341,7 @@ def __init__(self, severity: Severity = Severity.UNKNOWN): self.add_subcontext() -def add_severity_limit_rules(rules: List[str]) -> None: +def add_severity_limit_rules(rules: list[str]) -> None: for rule in rules: if not rule: continue @@ -347,7 +353,7 @@ def add_severity_limit_rules(rules: List[str]) -> None: LOGGER.exception("Unable to add a max severity limit rule `%s'", rule) -def _max_severity(keys: Tuple[str, ...], name: str) -> Severity: +def _max_severity(keys: tuple[str, ...], name: str) -> Severity: for pattern, severity in SctEvent._sct_event_types_registry.limit_rules: if fnmatch.filter(keys, pattern): return severity @@ -419,10 +425,10 @@ def eval_filter(self, event: SctEventProtocol) -> bool: class LogEventProtocol(SctEventProtocol, Protocol[T_log_event]): regex: str node: Any - line: Optional[str] + line: str | None line_number: int - backtrace: Optional[str] - raw_backtrace: Optional[str] + backtrace: str | None + raw_backtrace: str | None def add_info(self: T_log_event, node, line: str, line_number: int) -> T_log_event: ... diff --git a/sdcm/sct_events/continuous_event.py b/sdcm/sct_events/continuous_event.py index f2c77c087a5..119c3ece6b5 100644 --- a/sdcm/sct_events/continuous_event.py +++ b/sdcm/sct_events/continuous_event.py @@ -15,12 +15,13 @@ import logging import time import traceback -from typing import Optional, List, Iterable, Any +from collections.abc import Iterable +from typing import Any from dateutil.relativedelta import relativedelta from sdcm.sct_events import Severity -from sdcm.sct_events.base import SctEvent, EventPeriod +from sdcm.sct_events.base import EventPeriod, SctEvent from sdcm.utils.metaclasses import Singleton LOGGER = logging.getLogger(__name__) @@ -37,8 +38,7 @@ def __init__(self): @property def continuous_events(self) -> Iterable[ContinuousEvent]: for hash_bucket in self.hashed_continuous_events.values(): - for event in hash_bucket: - yield event + yield from hash_bucket def add_event(self, event: ContinuousEvent): if not issubclass(type(event), ContinuousEvent): @@ -80,7 +80,7 @@ def find_running_disruption_events(self): class ContinuousEvent(SctEvent, abstract=True): # Event filter does not create object of the class (not initialize it), so "_duration" attribute should # exist without initialization - _duration: Optional[int] = None + _duration: int | None = None continuous_hash_fields: tuple[str] = ('event_id',) log_file_name: str = None @@ -203,7 +203,7 @@ def end_event(self, publish: bool = True) -> None: self._ready_to_publish = True self.publish() - def add_error(self, errors: Optional[List[str]]) -> None: + def add_error(self, errors: list[str] | None) -> None: if not isinstance(self.errors, list): self.errors = [] @@ -221,5 +221,5 @@ def event_error(self): def publish(self, warn_not_ready: bool = True) -> None: super().publish(warn_not_ready=warn_not_ready) - def publish_or_dump(self, default_logger: Optional[logging.Logger] = None, warn_not_ready: bool = True) -> None: + def publish_or_dump(self, default_logger: logging.Logger | None = None, warn_not_ready: bool = True) -> None: super().publish_or_dump(default_logger=default_logger, warn_not_ready=warn_not_ready) diff --git a/sdcm/sct_events/database.py b/sdcm/sct_events/database.py index 53322fc44d3..cfdf081fbbb 100644 --- a/sdcm/sct_events/database.py +++ b/sdcm/sct_events/database.py @@ -11,16 +11,26 @@ # # Copyright (c) 2020 ScyllaDB -import re import logging +import re +from collections.abc import Callable from functools import partial -from typing import Type, List, Tuple, Generic, Optional, NamedTuple, Pattern, Callable, Match - -from sdcm.sct_events import Severity, SctEventProtocol -from sdcm.sct_events.base import SctEvent, LogEvent, LogEventProtocol, T_log_event, InformationalEvent, \ - EventPeriod +from re import Match, Pattern +from typing import ( + Generic, + NamedTuple, +) -from sdcm.sct_events.continuous_event import ContinuousEventsRegistry, ContinuousEvent +from sdcm.sct_events import SctEventProtocol, Severity +from sdcm.sct_events.base import ( + EventPeriod, + InformationalEvent, + LogEvent, + LogEventProtocol, + SctEvent, + T_log_event, +) +from sdcm.sct_events.continuous_event import ContinuousEvent, ContinuousEventsRegistry from sdcm.sct_events.system import TestFrameworkEvent TOLERABLE_REACTOR_STALL: int = 500 # ms @@ -29,39 +39,39 @@ class DatabaseLogEvent(LogEvent, abstract=True): - WARNING: Type[LogEventProtocol] - NO_SPACE_ERROR: Type[LogEventProtocol] - UNKNOWN_VERB: Type[LogEventProtocol] - CLIENT_DISCONNECT: Type[LogEventProtocol] - SEMAPHORE_TIME_OUT: Type[LogEventProtocol] - LDAP_CONNECTION_RESET: Type[LogEventProtocol] - SYSTEM_PAXOS_TIMEOUT: Type[LogEventProtocol] - SERVICE_LEVEL_CONTROLLER: Type[LogEventProtocol] - GATE_CLOSED: Type[LogEventProtocol] - RESTARTED_DUE_TO_TIME_OUT: Type[LogEventProtocol] - EMPTY_NESTED_EXCEPTION: Type[LogEventProtocol] - BAD_ALLOC: Type[LogEventProtocol] - SCHEMA_FAILURE: Type[LogEventProtocol] - RUNTIME_ERROR: Type[LogEventProtocol] - DIRECTORY_NOT_EMPTY: Type[LogEventProtocol] - FILESYSTEM_ERROR: Type[LogEventProtocol] - STACKTRACE: Type[LogEventProtocol] - RAFT_TRANSFER_SNAPSHOT_ERROR: Type[LogEventProtocol] - DISK_ERROR: Type[LogEventProtocol] - COMPACTION_STOPPED: Type[LogEventProtocol] + WARNING: type[LogEventProtocol] + NO_SPACE_ERROR: type[LogEventProtocol] + UNKNOWN_VERB: type[LogEventProtocol] + CLIENT_DISCONNECT: type[LogEventProtocol] + SEMAPHORE_TIME_OUT: type[LogEventProtocol] + LDAP_CONNECTION_RESET: type[LogEventProtocol] + SYSTEM_PAXOS_TIMEOUT: type[LogEventProtocol] + SERVICE_LEVEL_CONTROLLER: type[LogEventProtocol] + GATE_CLOSED: type[LogEventProtocol] + RESTARTED_DUE_TO_TIME_OUT: type[LogEventProtocol] + EMPTY_NESTED_EXCEPTION: type[LogEventProtocol] + BAD_ALLOC: type[LogEventProtocol] + SCHEMA_FAILURE: type[LogEventProtocol] + RUNTIME_ERROR: type[LogEventProtocol] + DIRECTORY_NOT_EMPTY: type[LogEventProtocol] + FILESYSTEM_ERROR: type[LogEventProtocol] + STACKTRACE: type[LogEventProtocol] + RAFT_TRANSFER_SNAPSHOT_ERROR: type[LogEventProtocol] + DISK_ERROR: type[LogEventProtocol] + COMPACTION_STOPPED: type[LogEventProtocol] # REACTOR_STALLED must be above BACKTRACE as it has "Backtrace" in its message - REACTOR_STALLED: Type[LogEventProtocol] - KERNEL_CALLSTACK: Type[LogEventProtocol] - ABORTING_ON_SHARD: Type[LogEventProtocol] - SEGMENTATION: Type[LogEventProtocol] - CORRUPTED_SSTABLE: Type[LogEventProtocol] - INTEGRITY_CHECK: Type[LogEventProtocol] - SUPPRESSED_MESSAGES: Type[LogEventProtocol] - stream_exception: Type[LogEventProtocol] - RPC_CONNECTION: Type[LogEventProtocol] - DATABASE_ERROR: Type[LogEventProtocol] - BACKTRACE: Type[LogEventProtocol] + REACTOR_STALLED: type[LogEventProtocol] + KERNEL_CALLSTACK: type[LogEventProtocol] + ABORTING_ON_SHARD: type[LogEventProtocol] + SEGMENTATION: type[LogEventProtocol] + CORRUPTED_SSTABLE: type[LogEventProtocol] + INTEGRITY_CHECK: type[LogEventProtocol] + SUPPRESSED_MESSAGES: type[LogEventProtocol] + stream_exception: type[LogEventProtocol] + RPC_CONNECTION: type[LogEventProtocol] + DATABASE_ERROR: type[LogEventProtocol] + BACKTRACE: type[LogEventProtocol] MILLI_RE = re.compile(r"(\d+) ms") @@ -192,7 +202,7 @@ def add_info(self: T_log_event, node, line: str, line_number: int) -> T_log_even DatabaseLogEvent.DATABASE_ERROR(), DatabaseLogEvent.BACKTRACE(), ) -SYSTEM_ERROR_EVENTS_PATTERNS: List[Tuple[re.Pattern, LogEventProtocol]] = \ +SYSTEM_ERROR_EVENTS_PATTERNS: list[tuple[re.Pattern, LogEventProtocol]] = \ [(re.compile(event.regex, re.IGNORECASE), event) for event in SYSTEM_ERROR_EVENTS] # BACKTRACE_RE should match those: @@ -203,11 +213,11 @@ def add_info(self: T_log_event, node, line: str, line_number: int) -> T_log_even class ScyllaHelpErrorEvent(SctEvent, abstract=True): - duplicate: Type[SctEventProtocol] - filtered: Type[SctEventProtocol] + duplicate: type[SctEventProtocol] + filtered: type[SctEventProtocol] message: str - def __init__(self, message: Optional[str] = None, severity=Severity.WARNING): + def __init__(self, message: str | None = None, severity=Severity.WARNING): super().__init__(severity=severity) # Don't include `message' to the state if it's None. @@ -255,7 +265,7 @@ class ScyllaServerEventPattern(NamedTuple): class ScyllaServerEventPatternFuncs(NamedTuple): pattern: Pattern - event_class: Type[ContinuousEvent] + event_class: type[ContinuousEvent] period_func: Callable @@ -295,7 +305,7 @@ def __init__(self, node: str, severity=Severity.NORMAL, **__): class FullScanEvent(ScyllaDatabaseContinuousEvent): - def __init__(self, node: str, ks_cf: str, message: Optional[str] = None, severity=Severity.NORMAL, + def __init__(self, node: str, ks_cf: str, message: str | None = None, severity=Severity.NORMAL, **kwargs): self.ks_cf = ks_cf self.message = message @@ -316,7 +326,7 @@ def msgfmt(self): class FullPartitionScanReversedOrderEvent(ScyllaDatabaseContinuousEvent): - def __init__(self, node: str, ks_cf: str, message: Optional[str] = None, severity=Severity.NORMAL, **__): + def __init__(self, node: str, ks_cf: str, message: str | None = None, severity=Severity.NORMAL, **__): super().__init__(node=node, severity=severity) self.ks_cf = ks_cf self.message = message @@ -330,7 +340,7 @@ def msgfmt(self): class FullPartitionScanEvent(ScyllaDatabaseContinuousEvent): - def __init__(self, node: str, ks_cf: str, message: Optional[str] = None, severity=Severity.NORMAL, **__): + def __init__(self, node: str, ks_cf: str, message: str | None = None, severity=Severity.NORMAL, **__): super().__init__(node=node, severity=severity) self.ks_cf = ks_cf self.message = message @@ -344,7 +354,7 @@ def msgfmt(self): class TombstoneGcVerificationEvent(ScyllaDatabaseContinuousEvent): - def __init__(self, node: str, ks_cf: str, message: Optional[str] = None, severity=Severity.NORMAL, **__): + def __init__(self, node: str, ks_cf: str, message: str | None = None, severity=Severity.NORMAL, **__): super().__init__(node=node, severity=severity) self.ks_cf = ks_cf self.message = message @@ -427,7 +437,7 @@ def __init__(self, node: str, severity=Severity.NORMAL, **__): class ScyllaYamlUpdateEvent(InformationalEvent): - def __init__(self, node_name: str, message: Optional[str] = None, diff: dict | None = None, + def __init__(self, node_name: str, message: str | None = None, diff: dict | None = None, severity=Severity.NORMAL, **__): super().__init__(severity=severity) self.message = message or f"Updating scylla.yaml contents on node: {node_name}. Diff: {diff}" @@ -446,7 +456,7 @@ def msgfmt(self) -> str: def get_pattern_to_event_to_func_mapping(node: str) \ - -> List[ScyllaServerEventPatternFuncs]: + -> list[ScyllaServerEventPatternFuncs]: """ This function maps regex patterns, event classes and begin / end functions into ScyllaServerEventPatternFuncs object. Helper @@ -456,13 +466,13 @@ def get_pattern_to_event_to_func_mapping(node: str) \ mapping = [] event_registry = ContinuousEventsRegistry() - def _add_event(event_type: Type[ScyllaDatabaseContinuousEvent], match: Match): + def _add_event(event_type: type[ScyllaDatabaseContinuousEvent], match: Match): kwargs = match.groupdict() if "shard" in kwargs: kwargs["shard"] = int(kwargs["shard"]) event_type(node=node, **kwargs).begin_event() - def _end_event(event_type: Type[ScyllaDatabaseContinuousEvent], match: Match): + def _end_event(event_type: type[ScyllaDatabaseContinuousEvent], match: Match): kwargs = match.groupdict() continuous_hash = event_type.get_continuous_hash_from_dict({'node': node, **kwargs}) if begin_event := event_registry.find_continuous_events_by_hash(continuous_hash): diff --git a/sdcm/sct_events/event_counter.py b/sdcm/sct_events/event_counter.py index 7c2d9c33ec0..f8cae2ae8c9 100644 --- a/sdcm/sct_events/event_counter.py +++ b/sdcm/sct_events/event_counter.py @@ -11,23 +11,26 @@ # # Copyright (c) 2023 ScyllaDB -import re import logging import multiprocessing - +import re +from collections.abc import Callable from dataclasses import dataclass +from functools import partial from pathlib import Path +from typing import Any, cast from uuid import uuid4 -from typing import Tuple, Callable, Any, cast -from functools import partial from sdcm.sct_events.base import T_log_event - from sdcm.sct_events.events_device import get_events_main_device -from sdcm.sct_events.events_processes import \ - EVENTS_COUNTER_ID, EventsProcessesRegistry, BaseEventsProcess, \ - start_events_process, get_events_process, verbose_suppress - +from sdcm.sct_events.events_processes import ( + EVENTS_COUNTER_ID, + BaseEventsProcess, + EventsProcessesRegistry, + get_events_process, + start_events_process, + verbose_suppress, +) LOGGER = logging.getLogger(__name__) REACTOR_MS_REGEX = re.compile(r'Reactor stalled for\s(\d+)\sms') @@ -98,7 +101,7 @@ def update_stat(self, stat: dict): } -class EventsCounter(BaseEventsProcess[Tuple[str, Any], None], multiprocessing.Process): +class EventsCounter(BaseEventsProcess[tuple[str, Any], None], multiprocessing.Process): def __init__(self, _registry: EventsProcessesRegistry): base_dir: Path = get_events_main_device(_registry=_registry).events_log_base_dir diff --git a/sdcm/sct_events/event_handler.py b/sdcm/sct_events/event_handler.py index 2642a400547..dd52d47e0a8 100644 --- a/sdcm/sct_events/event_handler.py +++ b/sdcm/sct_events/event_handler.py @@ -14,12 +14,15 @@ import logging import threading from functools import partial -from typing import Tuple, Any +from typing import Any from sdcm.cluster import TestConfig -from sdcm.sct_events.events_processes import \ - EVENTS_HANDLER_ID, BaseEventsProcess, \ - start_events_process, verbose_suppress +from sdcm.sct_events.events_processes import ( + EVENTS_HANDLER_ID, + BaseEventsProcess, + start_events_process, + verbose_suppress, +) from sdcm.sct_events.handlers.schema_disagreement import SchemaDisagreementHandler LOGGER = logging.getLogger(__name__) @@ -29,7 +32,7 @@ class TestFailure(Exception): pass -class EventsHandler(BaseEventsProcess[Tuple[str, Any], None], threading.Thread): +class EventsHandler(BaseEventsProcess[tuple[str, Any], None], threading.Thread): """Runs handlers for events (according to EventsHandler.handlers mapping dict). Handlers are created to gather additional information for issue investigation purposes.""" diff --git a/sdcm/sct_events/events_analyzer.py b/sdcm/sct_events/events_analyzer.py index 728fcac8e43..ca635992441 100644 --- a/sdcm/sct_events/events_analyzer.py +++ b/sdcm/sct_events/events_analyzer.py @@ -11,18 +11,22 @@ # # Copyright (c) 2020 ScyllaDB -import sys import logging +import sys import threading -from typing import Tuple, Any, Optional from functools import partial +from typing import Any from sdcm.cluster import TestConfig from sdcm.sct_events import Severity -from sdcm.sct_events.events_processes import \ - EVENTS_ANALYZER_ID, EventsProcessesRegistry, BaseEventsProcess, \ - start_events_process, get_events_process, verbose_suppress - +from sdcm.sct_events.events_processes import ( + EVENTS_ANALYZER_ID, + BaseEventsProcess, + EventsProcessesRegistry, + get_events_process, + start_events_process, + verbose_suppress, +) LOADERS_EVENTS = \ {"CassandraStressEvent", "ScyllaBenchEvent", "YcsbStressEvent", "NdbenchStressEvent", "CDCReaderStressEvent"} @@ -34,7 +38,7 @@ class TestFailure(Exception): pass -class EventsAnalyzer(BaseEventsProcess[Tuple[str, Any], None], threading.Thread): +class EventsAnalyzer(BaseEventsProcess[tuple[str, Any], None], threading.Thread): def run(self) -> None: for event_tuple in self.inbound_events(): with verbose_suppress("EventsAnalyzer failed to process %s", event_tuple): @@ -71,7 +75,7 @@ def kill_test(self, backtrace_with_reason, memo={}) -> None: # pylint: disable= start_events_analyzer = partial(start_events_process, EVENTS_ANALYZER_ID, EventsAnalyzer) -def stop_events_analyzer(_registry: Optional[EventsProcessesRegistry] = None) -> None: +def stop_events_analyzer(_registry: EventsProcessesRegistry | None = None) -> None: if analyzer := get_events_process(EVENTS_ANALYZER_ID, _registry=_registry): analyzer.stop(timeout=60) diff --git a/sdcm/sct_events/events_device.py b/sdcm/sct_events/events_device.py index e4a61506ccd..25093cf8556 100644 --- a/sdcm/sct_events/events_device.py +++ b/sdcm/sct_events/events_device.py @@ -13,23 +13,29 @@ # pylint: disable=no-member; disable `no-member' messages because of zmq. -import time -import queue import ctypes -import pickle import logging import multiprocessing -from typing import Optional, Generator, Any, Tuple, Callable, cast, Dict -from pathlib import Path +import pickle +import queue +import time +from collections.abc import Callable, Generator from functools import cached_property, partial +from pathlib import Path +from typing import Any, cast from uuid import UUID import zmq -from sdcm.sct_events.events_processes import \ - EVENTS_MAIN_DEVICE_ID, StopEvent, EventsProcessesRegistry, \ - start_events_process, get_events_process, verbose_suppress, suppress_interrupt - +from sdcm.sct_events.events_processes import ( + EVENTS_MAIN_DEVICE_ID, + EventsProcessesRegistry, + StopEvent, + get_events_process, + start_events_process, + suppress_interrupt, + verbose_suppress, +) EVENTS_DEVICE_START_DELAY: float = 0 # seconds EVENTS_DEVICE_START_TIMEOUT: float = 30 # seconds @@ -76,7 +82,7 @@ def events_log_base_dir(self) -> Path: def raw_events_log(self) -> Path: return self.events_log_base_dir / RAW_EVENTS_LOG - def stop(self, timeout: Optional[float] = None) -> None: + def stop(self, timeout: float | None = None) -> None: self._running.clear() self.join(timeout) @@ -143,12 +149,12 @@ def inbound_events(self, stop_event: StopEvent) -> Generator[Any, None, None]: # pylint: disable=import-outside-toplevel def outbound_events(self, stop_event: StopEvent, - events_counter: multiprocessing.Value) -> Generator[Tuple[str, Any], None, None]: + events_counter: multiprocessing.Value) -> Generator[tuple[str, Any], None, None]: from sdcm.sct_events.base import max_severity - from sdcm.sct_events.system import SystemEvent from sdcm.sct_events.filters import BaseFilter + from sdcm.sct_events.system import SystemEvent - filters: Dict[UUID, BaseFilter] = {} + filters: dict[UUID, BaseFilter] = {} filters_gc_next_hit = time.perf_counter() + FILTERS_GC_PERIOD with suppress_interrupt(): diff --git a/sdcm/sct_events/events_processes.py b/sdcm/sct_events/events_processes.py index 9ef0cd3137e..3e0257b748b 100644 --- a/sdcm/sct_events/events_processes.py +++ b/sdcm/sct_events/events_processes.py @@ -14,18 +14,17 @@ from __future__ import annotations import abc -import queue import ctypes import logging -import threading import multiprocessing -from typing import Union, Generator, Protocol, TypeVar, Generic, Type, Optional, cast -from pathlib import Path +import queue +import threading +from collections.abc import Generator from contextlib import contextmanager - +from pathlib import Path +from typing import Generic, Protocol, TypeVar, cast from weakref import proxy as weakproxy - EVENTS_MAIN_DEVICE_ID = "MainDevice" EVENTS_FILE_LOGGER_ID = "EVENTS_FILE_LOGGER" EVENTS_GRAFANA_ANNOTATOR_ID = "EVENTS_GRAFANA_ANNOTATOR" @@ -41,7 +40,7 @@ LOGGER = logging.getLogger(__name__) -StopEvent = Union[multiprocessing.Event, threading.Event] +StopEvent = multiprocessing.Event | threading.Event T_inbound_event = TypeVar("T_inbound_event") # pylint: disable=invalid-name T_outbound_event = TypeVar("T_outbound_event") # pylint: disable=invalid-name @@ -134,18 +133,18 @@ class EventsProcessThread(BaseEventsProcess[T_inbound_event, T_outbound_event], ... -EventsProcess = Union[EventsProcessProcess, EventsProcessThread] +EventsProcess = EventsProcessProcess | EventsProcessThread class EventsProcessesRegistry: - def __init__(self, log_dir: Union[str, Path], _default: bool = False): + def __init__(self, log_dir: str | Path, _default: bool = False): self.log_dir = Path(log_dir) self.default = _default self._registry_dict_lock = threading.RLock() self._registry_dict = {} LOGGER.debug("New events processes registry created: %s", self) - def start_events_process(self, name: str, klass: Type[EventsProcess]) -> None: + def start_events_process(self, name: str, klass: type[EventsProcess]) -> None: with self._registry_dict_lock: self._registry_dict[name] = process = klass(_registry=weakproxy(self)) process.start() @@ -162,10 +161,10 @@ def __str__(self): _EVENTS_PROCESSES_LOCK = threading.RLock() -_EVENTS_PROCESSES: Optional[EventsProcessesRegistry] = None +_EVENTS_PROCESSES: EventsProcessesRegistry | None = None -def create_default_events_process_registry(log_dir: Union[str, Path]): +def create_default_events_process_registry(log_dir: str | Path): global _EVENTS_PROCESSES # pylint: disable=global-statement # noqa: PLW0603 with _EVENTS_PROCESSES_LOCK: @@ -176,7 +175,7 @@ def create_default_events_process_registry(log_dir: Union[str, Path]): raise RuntimeError("Try to create default EventsProcessRegistry second time") -def get_default_events_process_registry(_registry: Optional[EventsProcessesRegistry] = None) -> EventsProcessesRegistry: +def get_default_events_process_registry(_registry: EventsProcessesRegistry | None = None) -> EventsProcessesRegistry: if _registry is not None: return _registry with _EVENTS_PROCESSES_LOCK: @@ -185,11 +184,11 @@ def get_default_events_process_registry(_registry: Optional[EventsProcessesRegis return _EVENTS_PROCESSES -def start_events_process(name, klass, _registry: Optional[EventsProcessesRegistry] = None) -> None: +def start_events_process(name, klass, _registry: EventsProcessesRegistry | None = None) -> None: get_default_events_process_registry(_registry=_registry).start_events_process(name=name, klass=klass) -def get_events_process(name: str, _registry: Optional[EventsProcessesRegistry] = None) -> EventsProcess: +def get_events_process(name: str, _registry: EventsProcessesRegistry | None = None) -> EventsProcess: return get_default_events_process_registry(_registry=_registry).get_events_process(name=name) diff --git a/sdcm/sct_events/file_logger.py b/sdcm/sct_events/file_logger.py index 69790ba8795..44f69e062de 100644 --- a/sdcm/sct_events/file_logger.py +++ b/sdcm/sct_events/file_logger.py @@ -11,25 +11,30 @@ # # Copyright (c) 2020 ScyllaDB -import re +import collections import json import logging -import collections import multiprocessing -from typing import Tuple, Optional, Callable, Any, Dict, List, cast -from pathlib import Path +import re +from collections import deque as tail +from collections.abc import Callable from functools import partial from itertools import chain -from collections import deque as tail +from pathlib import Path +from typing import Any, cast from sdcm.sct_events import Severity from sdcm.sct_events.base import SctEvent -from sdcm.sct_events.system import TestResultEvent from sdcm.sct_events.events_device import get_events_main_device -from sdcm.sct_events.events_processes import \ - EVENTS_FILE_LOGGER_ID, EventsProcessesRegistry, BaseEventsProcess, \ - start_events_process, get_events_process, verbose_suppress - +from sdcm.sct_events.events_processes import ( + EVENTS_FILE_LOGGER_ID, + BaseEventsProcess, + EventsProcessesRegistry, + get_events_process, + start_events_process, + verbose_suppress, +) +from sdcm.sct_events.system import TestResultEvent EVENTS_LOG: str = "events.log" SUMMARY_LOG: str = "summary.log" @@ -45,7 +50,7 @@ class head(list): # pylint: disable=invalid-name - def __init__(self, maxlen: Optional[int] = None): + def __init__(self, maxlen: int | None = None): super().__init__() if maxlen is None: self.append = super().append @@ -56,7 +61,7 @@ def append(self, item: Any) -> None: super().append(item) -class EventsFileLogger(BaseEventsProcess[Tuple[str, Any], None], multiprocessing.Process): +class EventsFileLogger(BaseEventsProcess[tuple[str, Any], None], multiprocessing.Process): def __init__(self, _registry: EventsProcessesRegistry): base_dir: Path = get_events_main_device(_registry=_registry).events_log_base_dir @@ -117,7 +122,7 @@ def write_event(self, event: SctEvent) -> None: with self.events_summary_log.open("wb", buffering=0) as fobj: fobj.write(json.dumps(dict(self.events_summary), indent=4).encode("utf-8")) - def get_events_by_category(self, limit: Optional[int] = None) -> Dict[str, List[str]]: + def get_events_by_category(self, limit: int | None = None) -> dict[str, list[str]]: output = {} for severity, log_file in self.events_logs_by_severity.items(): # Get first `limit' events with CRITICAL severity and last `limit' for other severities. @@ -147,12 +152,12 @@ def get_events_by_category(self, limit: Optional[int] = None) -> Dict[str, List[ get_events_logger = cast(Callable[..., EventsFileLogger], partial(get_events_process, EVENTS_FILE_LOGGER_ID)) -def get_events_grouped_by_category(limit: Optional[int] = None, - _registry: Optional[EventsProcessesRegistry] = None) -> Dict[str, List[str]]: +def get_events_grouped_by_category(limit: int | None = None, + _registry: EventsProcessesRegistry | None = None) -> dict[str, list[str]]: return get_events_logger(_registry=_registry).get_events_by_category(limit=limit) -def get_logger_event_summary(_registry: Optional[EventsProcessesRegistry] = None) -> dict: +def get_logger_event_summary(_registry: EventsProcessesRegistry | None = None) -> dict: events_summary_log = get_events_logger(_registry=_registry).events_summary_log with verbose_suppress("Failed to read %s", events_summary_log): with events_summary_log.open() as fobj: diff --git a/sdcm/sct_events/filters.py b/sdcm/sct_events/filters.py index 6597a001069..42f84e25b2d 100644 --- a/sdcm/sct_events/filters.py +++ b/sdcm/sct_events/filters.py @@ -13,18 +13,23 @@ import re import time -from typing import Optional, Type, Union from functools import cached_property +from typing import Any from sdcm.sct_events import Severity -from sdcm.sct_events.base import SctEvent, SctEventProtocol, BaseFilter, LogEventProtocol +from sdcm.sct_events.base import ( + BaseFilter, + LogEventProtocol, + SctEvent, + SctEventProtocol, +) class DbEventsFilter(BaseFilter): def __init__(self, - db_event: Union[LogEventProtocol, Type[LogEventProtocol]], - line: Optional[str] = None, - node: Optional = None): + db_event: LogEventProtocol | type[LogEventProtocol], + line: str | None = None, + node: Any | None = None): super().__init__() self.filter_type = db_event.type @@ -62,9 +67,9 @@ def msgfmt(self) -> str: class EventsFilter(BaseFilter): def __init__(self, - event_class: Optional[Type[SctEventProtocol] | Type[SctEvent]] = None, - regex: Optional[Union[str, re.Pattern]] = None, - extra_time_to_expiration: Optional[int] = 0): + event_class: type[SctEventProtocol] | type[SctEvent] | None = None, + regex: str | re.Pattern | None = None, + extra_time_to_expiration: int | None = 0): assert event_class or regex, \ "Should call with event_class or regex, or both" @@ -118,9 +123,9 @@ def msgfmt(self) -> str: class EventsSeverityChangerFilter(EventsFilter): def __init__(self, new_severity: Severity, - event_class: Optional[Type[SctEvent]] = None, - regex: Optional[str] = None, - extra_time_to_expiration: Optional[int] = None): + event_class: type[SctEvent] | None = None, + regex: str | None = None, + extra_time_to_expiration: int | None = None): super().__init__(event_class=event_class, regex=regex, extra_time_to_expiration=extra_time_to_expiration) self.new_severity = new_severity diff --git a/sdcm/sct_events/gce_events.py b/sdcm/sct_events/gce_events.py index 1f184afa277..9ce797837d6 100644 --- a/sdcm/sct_events/gce_events.py +++ b/sdcm/sct_events/gce_events.py @@ -1,15 +1,13 @@ -from typing import Dict - from dateutil import parser from sdcm.sct_events import Severity -from sdcm.sct_events.base import SctEvent, EventPeriod +from sdcm.sct_events.base import EventPeriod, SctEvent class GceInstanceEvent(SctEvent): def __init__(self, - gce_log_entry: Dict, + gce_log_entry: dict, severity=Severity.ERROR): self.date = str(parser.parse(gce_log_entry["timestamp"]).astimezone()) self.node = gce_log_entry["protoPayload"]["resourceName"].split("/")[-1] diff --git a/sdcm/sct_events/grafana.py b/sdcm/sct_events/grafana.py index 23928fe4f04..953c9f86da7 100644 --- a/sdcm/sct_events/grafana.py +++ b/sdcm/sct_events/grafana.py @@ -11,35 +11,42 @@ # # Copyright (c) 2020 ScyllaDB -import time import logging import threading -from typing import NewType, Dict, Any, Tuple, Optional, Callable, cast -from functools import partial +import time from collections import defaultdict +from collections.abc import Callable +from functools import partial +from typing import Any, NewType, cast import requests -from sdcm.sct_events.events_processes import \ - EVENTS_GRAFANA_ANNOTATOR_ID, EVENTS_GRAFANA_AGGREGATOR_ID, EVENTS_GRAFANA_POSTMAN_ID, \ - EventsProcessesRegistry, BaseEventsProcess, EventsProcessPipe, \ - start_events_process, get_events_process, verbose_suppress - +from sdcm.sct_events.events_processes import ( + EVENTS_GRAFANA_AGGREGATOR_ID, + EVENTS_GRAFANA_ANNOTATOR_ID, + EVENTS_GRAFANA_POSTMAN_ID, + BaseEventsProcess, + EventsProcessesRegistry, + EventsProcessPipe, + get_events_process, + start_events_process, + verbose_suppress, +) GRAFANA_EVENT_AGGREGATOR_TIME_WINDOW: float = 90 # seconds GRAFANA_EVENT_AGGREGATOR_MAX_DUPLICATES: int = 5 GRAFANA_EVENT_AGGREGATOR_QUEUE_WAIT_TIMEOUT: float = 1 # seconds GRAFANA_ANNOTATIONS_API_ENDPOINT: str = "/api/annotations" -GRAFANA_ANNOTATIONS_API_AUTH: Tuple[str, str] = ("admin", "admin", ) +GRAFANA_ANNOTATIONS_API_AUTH: tuple[str, str] = ("admin", "admin", ) LOGGER = logging.getLogger(__name__) -Annotation = NewType("Annotation", Dict[str, Any]) -AnnotationKey = NewType("AnnotationKey", Tuple[str, ...]) +Annotation = NewType("Annotation", dict[str, Any]) +AnnotationKey = NewType("AnnotationKey", tuple[str, ...]) -class GrafanaAnnotator(EventsProcessPipe[Tuple[str, Any], Annotation]): +class GrafanaAnnotator(EventsProcessPipe[tuple[str, Any], Annotation]): def run(self) -> None: for event_tuple in self.inbound_events(): # pylint: disable=no-member; pylint doesn't understand generics with verbose_suppress("GrafanaAnnotator failed to process %s", event_tuple): @@ -67,7 +74,7 @@ class GrafanaEventAggregator(EventsProcessPipe[Annotation, Annotation]): max_duplicates = GRAFANA_EVENT_AGGREGATOR_MAX_DUPLICATES def run(self) -> None: - time_window_counters: Dict[AnnotationKey, int] = defaultdict(int) + time_window_counters: dict[AnnotationKey, int] = defaultdict(int) time_window_end = time.perf_counter() for annotation in self.inbound_events(): # pylint: disable=no-member; pylint doesn't understand generics @@ -136,17 +143,17 @@ def terminate(self) -> None: get_grafana_postman = cast(Callable[..., GrafanaEventPostman], partial(get_events_process, EVENTS_GRAFANA_POSTMAN_ID)) -def start_grafana_pipeline(_registry: Optional[EventsProcessesRegistry] = None) -> None: +def start_grafana_pipeline(_registry: EventsProcessesRegistry | None = None) -> None: start_grafana_annotator(_registry=_registry) start_grafana_aggregator(_registry=_registry) start_grafana_postman(_registry=_registry) -def set_grafana_url(url: str, _registry: Optional[EventsProcessesRegistry] = None) -> None: +def set_grafana_url(url: str, _registry: EventsProcessesRegistry | None = None) -> None: get_grafana_postman(_registry=_registry).set_grafana_url(url) -def start_posting_grafana_annotations(_registry: Optional[EventsProcessesRegistry] = None) -> None: +def start_posting_grafana_annotations(_registry: EventsProcessesRegistry | None = None) -> None: get_grafana_postman(_registry=_registry).start_posting_grafana_annotations() diff --git a/sdcm/sct_events/group_common_events.py b/sdcm/sct_events/group_common_events.py index 24ee814bba2..ea45e5aa2aa 100644 --- a/sdcm/sct_events/group_common_events.py +++ b/sdcm/sct_events/group_common_events.py @@ -11,14 +11,23 @@ # # Copyright (c) 2020 ScyllaDB -from contextlib import contextmanager, ExitStack, ContextDecorator +from collections.abc import Callable, Sequence +from contextlib import ( + AbstractContextManager, + ContextDecorator, + ExitStack, + contextmanager, +) from functools import wraps -from typing import ContextManager, Callable, Sequence from sdcm.sct_events import Severity -from sdcm.sct_events.filters import DbEventsFilter, EventsSeverityChangerFilter, EventsFilter -from sdcm.sct_events.loaders import YcsbStressEvent from sdcm.sct_events.database import DatabaseLogEvent +from sdcm.sct_events.filters import ( + DbEventsFilter, + EventsFilter, + EventsSeverityChangerFilter, +) +from sdcm.sct_events.loaders import YcsbStressEvent from sdcm.sct_events.monitors import PrometheusAlertManagerEvent @@ -306,13 +315,13 @@ def ignore_stream_mutation_fragments_errors(): yield -def decorate_with_context(context_list: list[Callable | ContextManager] | Callable | ContextManager): +def decorate_with_context(context_list: list[Callable | AbstractContextManager] | Callable | AbstractContextManager): """ helper to decorate a function to run with a list of callables that return context managers """ context_list = context_list if isinstance(context_list, Sequence) else [context_list] for context in context_list: - assert callable(context) or isinstance(context, ContextManager), \ + assert callable(context) or isinstance(context, AbstractContextManager), \ f"{context} - Should be contextmanager or callable that returns one" assert not isinstance(context, ContextDecorator), \ f"{context} - ContextDecorator shouldn't be used, since they are usable only one" @@ -324,7 +333,7 @@ def inner_func(*args, **kwargs): for context_manager in context_list: if callable(context_manager): cmanger = context_manager() - assert isinstance(cmanger, ContextManager) + assert isinstance(cmanger, AbstractContextManager) stack.enter_context(cmanger) else: stack.enter_context(context_manager) diff --git a/sdcm/sct_events/handlers/schema_disagreement.py b/sdcm/sct_events/handlers/schema_disagreement.py index 96dec95ccfa..3a410a209eb 100644 --- a/sdcm/sct_events/handlers/schema_disagreement.py +++ b/sdcm/sct_events/handlers/schema_disagreement.py @@ -16,7 +16,10 @@ from sdcm.sct_events import Severity from sdcm.sct_events.handlers import EventHandler -from sdcm.sct_events.loaders import CassandraStressLogEvent, SchemaDisagreementErrorEvent +from sdcm.sct_events.loaders import ( + CassandraStressLogEvent, + SchemaDisagreementErrorEvent, +) from sdcm.utils.sstable.s3_uploader import upload_sstables_to_s3 LOGGER = logging.getLogger(__name__) diff --git a/sdcm/sct_events/health.py b/sdcm/sct_events/health.py index 87089c0a8b6..36b269e01e7 100644 --- a/sdcm/sct_events/health.py +++ b/sdcm/sct_events/health.py @@ -11,9 +11,7 @@ # # Copyright (c) 2021 ScyllaDB -from typing import Type, Optional - -from sdcm.sct_events import Severity, SctEventProtocol +from sdcm.sct_events import SctEventProtocol, Severity from sdcm.sct_events.base import InformationalEvent from sdcm.sct_events.continuous_event import ContinuousEvent @@ -28,18 +26,18 @@ def __init__(self, node: str = None, message: str = None, error: str = None, sev class ClusterHealthValidatorEvent(ContinuousEvent): - MonitoringStatus: Type[SctEventProtocol] - NodeStatus: Type[SctEventProtocol] - NodePeersNulls: Type[SctEventProtocol] - NodeSchemaVersion: Type[SctEventProtocol] - NodesNemesis: Type[SctEventProtocol] - ScyllaCloudClusterServerDiagnostic: Type[SctEventProtocol] - Group0TokenRingInconsistency: Type[SctEventProtocol] + MonitoringStatus: type[SctEventProtocol] + NodeStatus: type[SctEventProtocol] + NodePeersNulls: type[SctEventProtocol] + NodeSchemaVersion: type[SctEventProtocol] + NodesNemesis: type[SctEventProtocol] + ScyllaCloudClusterServerDiagnostic: type[SctEventProtocol] + Group0TokenRingInconsistency: type[SctEventProtocol] def __init__(self, node=None, - message: Optional[str] = None, - error: Optional[str] = None, + message: str | None = None, + error: str | None = None, severity=Severity.NORMAL) -> None: self.node = str(node) if node else "" self.error = error if error else "" @@ -81,14 +79,14 @@ def msgfmt(self) -> str: class DataValidatorEvent(InformationalEvent, abstract=True): - DataValidator: Type[SctEventProtocol] - ImmutableRowsValidator: Type[SctEventProtocol] - UpdatedRowsValidator: Type[SctEventProtocol] - DeletedRowsValidator: Type[SctEventProtocol] + DataValidator: type[SctEventProtocol] + ImmutableRowsValidator: type[SctEventProtocol] + UpdatedRowsValidator: type[SctEventProtocol] + DeletedRowsValidator: type[SctEventProtocol] def __init__(self, - message: Optional[str] = None, - error: Optional[str] = None, + message: str | None = None, + error: str | None = None, severity: Severity = Severity.ERROR) -> None: super().__init__(severity=severity) diff --git a/sdcm/sct_events/loaders.py b/sdcm/sct_events/loaders.py index 5a7e9979724..7f6f0261a9a 100644 --- a/sdcm/sct_events/loaders.py +++ b/sdcm/sct_events/loaders.py @@ -11,18 +11,22 @@ # # Copyright (c) 2020 ScyllaDB -import re import json -import time import logging -from typing import Type, Optional, List, Tuple, Any +import re +import time +from typing import Any import dateutil.parser from invoke.runners import Result -from sdcm.sct_events import Severity, SctEventProtocol -from sdcm.sct_events.base import LogEvent, LogEventProtocol, T_log_event, SctEvent -from sdcm.sct_events.stress_events import BaseStressEvent, StressEvent, StressEventProtocol +from sdcm.sct_events import SctEventProtocol, Severity +from sdcm.sct_events.base import LogEvent, LogEventProtocol, SctEvent, T_log_event +from sdcm.sct_events.stress_events import ( + BaseStressEvent, + StressEvent, + StressEventProtocol, +) LOGGER = logging.getLogger(__name__) @@ -31,7 +35,7 @@ class GeminiStressEvent(BaseStressEvent): # pylint: disable=too-many-arguments, too-many-instance-attributes def __init__(self, node: Any, # noqa: PLR0913 cmd: str, - log_file_name: Optional[str] = None, + log_file_name: str | None = None, severity: Severity = Severity.NORMAL, publish_event: bool = True): self.node = str(node) @@ -40,7 +44,7 @@ def __init__(self, node: Any, # noqa: PLR0913 self.result = "" super().__init__(severity=severity, publish_event=publish_event) - def add_result(self, result: Optional[Result]): + def add_result(self, result: Result | None): if result != "": self.result += f"Exit code: {result.exited}\n" if result.stdout: @@ -79,11 +83,11 @@ class LatteStressEvent(StressEvent): class CassandraHarryEvent(StressEvent, abstract=True): - failure: Type[StressEventProtocol] - error: Type[SctEventProtocol] - timeout: Type[StressEventProtocol] - start: Type[StressEventProtocol] - finish: Type[StressEventProtocol] + failure: type[StressEventProtocol] + error: type[SctEventProtocol] + timeout: type[StressEventProtocol] + start: type[StressEventProtocol] + finish: type[StressEventProtocol] @property def msgfmt(self): @@ -101,30 +105,30 @@ class BaseYcsbStressEvent(StressEvent, abstract=True): class YcsbStressEvent(BaseYcsbStressEvent, abstract=True): - failure: Type[StressEventProtocol] - error: Type[StressEventProtocol] - start: Type[StressEventProtocol] - finish: Type[StressEventProtocol] + failure: type[StressEventProtocol] + error: type[StressEventProtocol] + start: type[StressEventProtocol] + finish: type[StressEventProtocol] YcsbStressEvent.add_stress_subevents(failure=Severity.CRITICAL, error=Severity.ERROR) class CDCReaderStressEvent(BaseYcsbStressEvent, abstract=True): - failure: Type[StressEventProtocol] - error: Type[StressEventProtocol] - start: Type[StressEventProtocol] - finish: Type[StressEventProtocol] + failure: type[StressEventProtocol] + error: type[StressEventProtocol] + start: type[StressEventProtocol] + finish: type[StressEventProtocol] CDCReaderStressEvent.add_stress_subevents(failure=Severity.CRITICAL, error=Severity.ERROR) class NdBenchStressEvent(StressEvent, abstract=True): - failure: Type[StressEventProtocol] - error: Type[StressEventProtocol] - start: Type[StressEventProtocol] - finish: Type[StressEventProtocol] + failure: type[StressEventProtocol] + error: type[StressEventProtocol] + start: type[StressEventProtocol] + finish: type[StressEventProtocol] NdBenchStressEvent.add_stress_subevents(start=Severity.NORMAL, @@ -134,8 +138,8 @@ class NdBenchStressEvent(StressEvent, abstract=True): class NdBenchErrorEvent(LogEvent, abstract=True): - Error: Type[LogEventProtocol] - Failure: Type[LogEventProtocol] + Error: type[LogEventProtocol] + Failure: type[LogEventProtocol] NdBenchErrorEvent.add_subevent_type("Error", severity=Severity.ERROR, regex=r"ERROR") @@ -151,9 +155,9 @@ class NdBenchErrorEvent(LogEvent, abstract=True): class KclStressEvent(StressEvent, abstract=True): - failure: Type[StressEventProtocol] - start: Type[StressEventProtocol] - finish: Type[StressEventProtocol] + failure: type[StressEventProtocol] + start: type[StressEventProtocol] + finish: type[StressEventProtocol] KclStressEvent.add_stress_subevents(failure=Severity.ERROR) @@ -164,12 +168,12 @@ class NoSQLBenchStressEvent(StressEvent): class CassandraStressLogEvent(LogEvent, abstract=True): - IOException: Type[LogEventProtocol] - ConsistencyError: Type[LogEventProtocol] - OperationOnKey: Type[LogEventProtocol] - TooManyHintsInFlight: Type[LogEventProtocol] - ShardAwareDriver: Type[LogEventProtocol] - SchemaDisagreement: Type[LogEventProtocol] + IOException: type[LogEventProtocol] + ConsistencyError: type[LogEventProtocol] + OperationOnKey: type[LogEventProtocol] + TooManyHintsInFlight: type[LogEventProtocol] + ShardAwareDriver: type[LogEventProtocol] + SchemaDisagreement: type[LogEventProtocol] class SchemaDisagreementErrorEvent(SctEvent): @@ -230,15 +234,15 @@ def msgfmt(self): ) CS_NORMAL_EVENTS = (CassandraStressLogEvent.ShardAwareDriver(), ) -CS_ERROR_EVENTS_PATTERNS: List[Tuple[re.Pattern, LogEventProtocol]] = \ +CS_ERROR_EVENTS_PATTERNS: list[tuple[re.Pattern, LogEventProtocol]] = \ [(re.compile(event.regex), event) for event in CS_ERROR_EVENTS] -CS_NORMAL_EVENTS_PATTERNS: List[Tuple[re.Pattern, LogEventProtocol]] = \ +CS_NORMAL_EVENTS_PATTERNS: list[tuple[re.Pattern, LogEventProtocol]] = \ [(re.compile(event.regex), event) for event in CS_NORMAL_EVENTS] class ScyllaBenchLogEvent(LogEvent, abstract=True): - ConsistencyError: Type[LogEventProtocol] + ConsistencyError: type[LogEventProtocol] ScyllaBenchLogEvent.add_subevent_type("ConsistencyError", severity=Severity.ERROR, regex=r"received only") @@ -247,12 +251,12 @@ class ScyllaBenchLogEvent(LogEvent, abstract=True): SCYLLA_BENCH_ERROR_EVENTS = ( ScyllaBenchLogEvent.ConsistencyError(), ) -SCYLLA_BENCH_ERROR_EVENTS_PATTERNS: List[Tuple[re.Pattern, LogEventProtocol]] = \ +SCYLLA_BENCH_ERROR_EVENTS_PATTERNS: list[tuple[re.Pattern, LogEventProtocol]] = \ [(re.compile(event.regex), event) for event in SCYLLA_BENCH_ERROR_EVENTS] CASSANDRA_HARRY_ERROR_EVENTS = ( ) -CASSANDRA_HARRY_ERROR_EVENTS_PATTERNS: List[Tuple[re.Pattern, LogEventProtocol]] = \ +CASSANDRA_HARRY_ERROR_EVENTS_PATTERNS: list[tuple[re.Pattern, LogEventProtocol]] = \ [(re.compile(event.regex), event) for event in CASSANDRA_HARRY_ERROR_EVENTS] @@ -265,7 +269,7 @@ class GeminiStressLogEvent(LogEvent[T_log_event], abstract=True): # pylint: dis "FATAL": "CRITICAL", } - GeminiEvent: Type[LogEventProtocol] + GeminiEvent: type[LogEventProtocol] def __init__(self, verbose=False): super().__init__(regex="", severity=Severity.CRITICAL) @@ -315,9 +319,9 @@ def msgfmt(self) -> str: class NoSQLBenchStressLogEvents(LogEvent, abstract=True): - ProgressIndicatorStoppedEvent: Type[LogEventProtocol] - ProgressIndicatorRunningEvent: Type[LogEventProtocol] - ProgressIndicatorFinishedEvent: Type[LogEventProtocol] + ProgressIndicatorStoppedEvent: type[LogEventProtocol] + ProgressIndicatorRunningEvent: type[LogEventProtocol] + ProgressIndicatorFinishedEvent: type[LogEventProtocol] NoSQLBenchStressLogEvents.add_subevent_type( diff --git a/sdcm/sct_events/nodetool.py b/sdcm/sct_events/nodetool.py index f20158c7b0b..90b8a2c2417 100644 --- a/sdcm/sct_events/nodetool.py +++ b/sdcm/sct_events/nodetool.py @@ -10,9 +10,9 @@ # See LICENSE for more details. # # Copyright (c) 2021 ScyllaDB -from dataclasses import dataclass, asdict +from dataclasses import asdict, dataclass from json import JSONEncoder -from typing import Any, Type +from typing import Any from sdcm.sct_events import Severity from sdcm.sct_events.continuous_event import ContinuousEvent @@ -65,5 +65,5 @@ def msgfmt(self) -> str: fmt += "\n{0.full_traceback}" return fmt - def to_json(self, encoder: Type[JSONEncoder] = NodetoolEventEncoder) -> str: + def to_json(self, encoder: type[JSONEncoder] = NodetoolEventEncoder) -> str: return super().to_json(encoder=encoder) diff --git a/sdcm/sct_events/operator.py b/sdcm/sct_events/operator.py index a2a2ccc1751..15b3a151fd8 100644 --- a/sdcm/sct_events/operator.py +++ b/sdcm/sct_events/operator.py @@ -11,25 +11,22 @@ # # Copyright (c) 2020 ScyllaDB import datetime -import re import logging +import re import time -from typing import List, Tuple, Type - from sdcm.sct_events import Severity -from sdcm.utils.remote_logger import KubernetesWrongSchedulingLogger from sdcm.sct_events.base import LogEvent, LogEventProtocol, T_log_event - +from sdcm.utils.remote_logger import KubernetesWrongSchedulingLogger LOGGER = logging.getLogger(__name__) class ScyllaOperatorLogEvent(LogEvent): - REAPPLY: Type[LogEventProtocol] - TLS_HANDSHAKE_ERROR: Type[LogEventProtocol] - OPERATOR_STARTED_INFO: Type[LogEventProtocol] - WRONG_SCHEDULED_PODS: Type[LogEventProtocol] + REAPPLY: type[LogEventProtocol] + TLS_HANDSHAKE_ERROR: type[LogEventProtocol] + OPERATOR_STARTED_INFO: type[LogEventProtocol] + WRONG_SCHEDULED_PODS: type[LogEventProtocol] def __init__(self, regex: str = None, severity=Severity.NORMAL): super().__init__(regex=regex, severity=severity) @@ -53,7 +50,7 @@ def add_info(self: T_log_event, node, line: str, line_number: int) -> T_log_even day = int(type_month_date[3:5]) self.source_timestamp = datetime.datetime( year=year, month=month, day=day, hour=int(hour), minute=int(minute), second=int(second), - microsecond=int(milliseconds), tzinfo=datetime.timezone.utc).timestamp() + microsecond=int(milliseconds), tzinfo=datetime.UTC).timestamp() except Exception: # pylint: disable=broad-except # noqa: BLE001 pass self.event_timestamp = time.time() @@ -86,7 +83,7 @@ def add_info(self: T_log_event, node, line: str, line_number: int) -> T_log_even ] -SCYLLA_OPERATOR_EVENT_PATTERNS: List[Tuple[re.Pattern, LogEventProtocol]] = \ +SCYLLA_OPERATOR_EVENT_PATTERNS: list[tuple[re.Pattern, LogEventProtocol]] = \ [(re.compile(event.regex, re.IGNORECASE), event) for event in SCYLLA_OPERATOR_EVENTS] diff --git a/sdcm/sct_events/prometheus.py b/sdcm/sct_events/prometheus.py index bd11966c852..1a3a5c36cbb 100644 --- a/sdcm/sct_events/prometheus.py +++ b/sdcm/sct_events/prometheus.py @@ -20,16 +20,15 @@ import logging import threading -from typing import Tuple, Any +from typing import Any from sdcm.prometheus import nemesis_metrics_obj from sdcm.sct_events.events_processes import BaseEventsProcess, verbose_suppress - LOGGER = logging.getLogger(__name__) -class PrometheusDumper(BaseEventsProcess[Tuple[str, Any], None], threading.Thread): +class PrometheusDumper(BaseEventsProcess[tuple[str, Any], None], threading.Thread): def run(self) -> None: events_gauge = \ nemesis_metrics_obj().create_gauge("sct_events_gauge", diff --git a/sdcm/sct_events/setup.py b/sdcm/sct_events/setup.py index c09cdcc33e9..786e0901e75 100644 --- a/sdcm/sct_events/setup.py +++ b/sdcm/sct_events/setup.py @@ -11,26 +11,34 @@ # # Copyright (c) 2020 ScyllaDB -import json -import time import atexit +import json import logging -from typing import Union, Optional +import time from pathlib import Path from sdcm.sct_events import Severity -from sdcm.sct_events.event_handler import start_events_handler -from sdcm.sct_events.grafana import start_grafana_pipeline -from sdcm.sct_events.filters import DbEventsFilter, EventsSeverityChangerFilter from sdcm.sct_events.database import DatabaseLogEvent -from sdcm.sct_events.file_logger import start_events_logger -from sdcm.sct_events.events_device import start_events_main_device -from sdcm.sct_events.events_analyzer import start_events_analyzer from sdcm.sct_events.event_counter import start_events_counter -from sdcm.sct_events.events_processes import \ - EVENTS_MAIN_DEVICE_ID, EVENTS_FILE_LOGGER_ID, EVENTS_ANALYZER_ID, \ - EVENTS_GRAFANA_ANNOTATOR_ID, EVENTS_GRAFANA_AGGREGATOR_ID, EVENTS_GRAFANA_POSTMAN_ID, \ - EventsProcessesRegistry, create_default_events_process_registry, get_events_process, EVENTS_HANDLER_ID, EVENTS_COUNTER_ID +from sdcm.sct_events.event_handler import start_events_handler +from sdcm.sct_events.events_analyzer import start_events_analyzer +from sdcm.sct_events.events_device import start_events_main_device +from sdcm.sct_events.events_processes import ( + EVENTS_ANALYZER_ID, + EVENTS_COUNTER_ID, + EVENTS_FILE_LOGGER_ID, + EVENTS_GRAFANA_AGGREGATOR_ID, + EVENTS_GRAFANA_ANNOTATOR_ID, + EVENTS_GRAFANA_POSTMAN_ID, + EVENTS_HANDLER_ID, + EVENTS_MAIN_DEVICE_ID, + EventsProcessesRegistry, + create_default_events_process_registry, + get_events_process, +) +from sdcm.sct_events.file_logger import start_events_logger +from sdcm.sct_events.filters import DbEventsFilter, EventsSeverityChangerFilter +from sdcm.sct_events.grafana import start_grafana_pipeline EVENTS_DEVICE_START_DELAY = 1 # seconds EVENTS_SUBSCRIBERS_START_DELAY = 3 # seconds @@ -39,8 +47,8 @@ LOGGER = logging.getLogger(__name__) -def start_events_device(log_dir: Optional[Union[str, Path]] = None, - _registry: Optional[EventsProcessesRegistry] = None) -> None: +def start_events_device(log_dir: str | Path | None = None, + _registry: EventsProcessesRegistry | None = None) -> None: if _registry is None: if log_dir is None: raise RuntimeError("Should provide log_dir or instance of EventsProcessesRegistry") @@ -86,7 +94,7 @@ def start_events_device(log_dir: Optional[Union[str, Path]] = None, atexit.register(stop_events_device, _registry=_registry) -def stop_events_device(_registry: Optional[EventsProcessesRegistry] = None) -> None: +def stop_events_device(_registry: EventsProcessesRegistry | None = None) -> None: LOGGER.debug("Stop all events consumers...") processes = ( EVENTS_FILE_LOGGER_ID, diff --git a/sdcm/sct_events/stress_events.py b/sdcm/sct_events/stress_events.py index c0d7e12d9ca..e5126d07bf2 100644 --- a/sdcm/sct_events/stress_events.py +++ b/sdcm/sct_events/stress_events.py @@ -10,9 +10,9 @@ # See LICENSE for more details. # # Copyright (c) 2021 ScyllaDB -from typing import Any, Optional, List, Protocol +from typing import Any, Protocol -from sdcm.sct_events import Severity, SctEventProtocol +from sdcm.sct_events import SctEventProtocol, Severity from sdcm.sct_events.continuous_event import ContinuousEvent @@ -20,12 +20,12 @@ class BaseStressEvent(ContinuousEvent, abstract=True): # pylint: disable=too-many-arguments @classmethod def add_stress_subevents(cls, # noqa: PLR0913 - failure: Optional[Severity] = None, - error: Optional[Severity] = None, - timeout: Optional[Severity] = None, - start: Optional[Severity] = Severity.NORMAL, - finish: Optional[Severity] = Severity.NORMAL, - warning: Optional[Severity] = None) -> None: + failure: Severity | None = None, + error: Severity | None = None, + timeout: Severity | None = None, + start: Severity | None = Severity.NORMAL, + finish: Severity | None = Severity.NORMAL, + warning: Severity | None = None) -> None: if failure is not None: cls.add_subevent_type("failure", severity=failure) if error is not None: @@ -42,9 +42,9 @@ def add_stress_subevents(cls, # noqa: PLR0913 class StressEventProtocol(SctEventProtocol, Protocol): node: str - stress_cmd: Optional[str] - log_file_name: Optional[str] - errors: Optional[List[str]] + stress_cmd: str | None + log_file_name: str | None + errors: list[str] | None @property def errors_formatted(self): @@ -55,9 +55,9 @@ class StressEvent(BaseStressEvent, abstract=True): # pylint: disable=too-many-arguments def __init__(self, # noqa: PLR0913 node: Any, - stress_cmd: Optional[str] = None, - log_file_name: Optional[str] = None, - errors: Optional[List[str]] = None, + stress_cmd: str | None = None, + log_file_name: str | None = None, + errors: list[str] | None = None, severity: Severity = Severity.NORMAL, publish_event: bool = True): self.node = str(node) diff --git a/sdcm/sct_events/system.py b/sdcm/sct_events/system.py index b9db7de6f1b..b6f0bf405ab 100644 --- a/sdcm/sct_events/system.py +++ b/sdcm/sct_events/system.py @@ -14,11 +14,18 @@ import re import sys import time -from typing import Any, Optional, Sequence, Type, List, Tuple +from collections.abc import Sequence from traceback import format_stack +from typing import Any from sdcm.sct_events import Severity -from sdcm.sct_events.base import SctEvent, SystemEvent, InformationalEvent, LogEvent, LogEventProtocol +from sdcm.sct_events.base import ( + InformationalEvent, + LogEvent, + LogEventProtocol, + SctEvent, + SystemEvent, +) from sdcm.sct_events.continuous_event import ContinuousEvent @@ -55,13 +62,13 @@ class TestFrameworkEvent(InformationalEvent): # pylint: disable=too-many-instan # pylint: disable=too-many-arguments def __init__(self, # noqa: PLR0913 source: Any, - source_method: Optional = None, - args: Optional[Sequence] = None, - kwargs: Optional[dict] = None, - message: Optional = None, - exception: Optional = None, - trace: Optional = None, - severity: Optional[Severity] = None): + source_method: Any | None = None, + args: Sequence | None = None, + kwargs: dict | None = None, + message: Any | None = None, + exception: Any | None = None, + trace: Any | None = None, + severity: Severity | None = None): if severity is None: severity = Severity.ERROR @@ -206,7 +213,7 @@ def __init__(self, # noqa: PLR0913 corefile_url: str, backtrace: str, download_instructions: str, - source_timestamp: Optional[float] = None): + source_timestamp: float | None = None): super().__init__(severity=Severity.ERROR) @@ -244,7 +251,7 @@ class TestResultEvent(InformationalEvent, Exception): _head = f"{' TEST RESULTS ':=^{_marker_width}}" _ending = "=" * _marker_width - def __init__(self, test_status: str, events: dict, event_timestamp: Optional[float] = None): + def __init__(self, test_status: str, events: dict, event_timestamp: float | None = None): self._ok = test_status == "SUCCESS" super().__init__(severity=Severity.NORMAL if self._ok else Severity.ERROR) @@ -281,9 +288,9 @@ def __reduce__(self): class InstanceStatusEvent(LogEvent, abstract=True): - STARTUP: Type[LogEventProtocol] - REBOOT: Type[LogEventProtocol] - POWER_OFF: Type[LogEventProtocol] + STARTUP: type[LogEventProtocol] + REBOOT: type[LogEventProtocol] + POWER_OFF: type[LogEventProtocol] InstanceStatusEvent.add_subevent_type("STARTUP", severity=Severity.WARNING, regex="kernel: Linux version") @@ -297,5 +304,5 @@ class InstanceStatusEvent(LogEvent, abstract=True): InstanceStatusEvent.POWER_OFF(), ) -INSTANCE_STATUS_EVENTS_PATTERNS: List[Tuple[re.Pattern, LogEventProtocol]] = \ +INSTANCE_STATUS_EVENTS_PATTERNS: list[tuple[re.Pattern, LogEventProtocol]] = \ [(re.compile(event.regex, re.IGNORECASE), event) for event in INSTANCE_STATUS_EVENTS] diff --git a/sdcm/sct_events/workload_prioritisation.py b/sdcm/sct_events/workload_prioritisation.py index 56d7ae9679b..9e16a91a96f 100644 --- a/sdcm/sct_events/workload_prioritisation.py +++ b/sdcm/sct_events/workload_prioritisation.py @@ -11,22 +11,21 @@ # # Copyright (c) 2021 ScyllaDB import logging -from typing import Type, Optional -from sdcm.sct_events import Severity, SctEventProtocol +from sdcm.sct_events import SctEventProtocol, Severity from sdcm.sct_events.base import InformationalEvent LOGGER = logging.getLogger(__name__) class WorkloadPrioritisationEvent(InformationalEvent, abstract=True): - CpuNotHighEnough: Type[SctEventProtocol] - RatioValidationEvent: Type[SctEventProtocol] - SlaTestResult: Type[SctEventProtocol] - EmptyPrometheusData: Type[SctEventProtocol] + CpuNotHighEnough: type[SctEventProtocol] + RatioValidationEvent: type[SctEventProtocol] + SlaTestResult: type[SctEventProtocol] + EmptyPrometheusData: type[SctEventProtocol] def __init__(self, - message: Optional[str] = None, + message: str | None = None, severity=Severity.ERROR) -> None: super().__init__(severity=severity) self.message = message if message else "" diff --git a/sdcm/sct_provision/__init__.py b/sdcm/sct_provision/__init__.py index 40b2b9fd898..93492386087 100644 --- a/sdcm/sct_provision/__init__.py +++ b/sdcm/sct_provision/__init__.py @@ -12,9 +12,10 @@ # Copyright (c) 2021 ScyllaDB from typing import Literal -from sdcm.sct_provision.azure.azure_region_definition_builder import AzureDefinitionBuilder +from sdcm.sct_provision.azure.azure_region_definition_builder import ( + AzureDefinitionBuilder, +) from sdcm.sct_provision.region_definition_builder import RegionDefinitionBuilder - region_definition_builder = RegionDefinitionBuilder() region_definition_builder.register_builder(backend="azure", builder_class=AzureDefinitionBuilder) diff --git a/sdcm/sct_provision/aws/cluster.py b/sdcm/sct_provision/aws/cluster.py index dd1b5f86666..1908d65f0f5 100644 --- a/sdcm/sct_provision/aws/cluster.py +++ b/sdcm/sct_provision/aws/cluster.py @@ -13,22 +13,32 @@ import abc from functools import cached_property -from typing import List, Dict from pydantic import BaseModel + from sdcm import cluster from sdcm.provision.aws.instance_parameters import AWSInstanceParams from sdcm.provision.aws.provisioner import AWSInstanceProvisioner +from sdcm.provision.aws.utils import create_cluster_placement_groups_aws from sdcm.provision.common.provision_plan import ProvisionPlan from sdcm.provision.common.provision_plan_builder import ProvisionPlanBuilder from sdcm.provision.common.provisioner import TagsType from sdcm.sct_config import SCTConfiguration -from sdcm.sct_provision.aws.instance_parameters_builder import ScyllaInstanceParamsBuilder, \ - LoaderInstanceParamsBuilder, MonitorInstanceParamsBuilder, OracleScyllaInstanceParamsBuilder -from sdcm.sct_provision.aws.user_data import ScyllaUserDataBuilder, AWSInstanceUserDataBuilder -from sdcm.sct_provision.common.utils import INSTANCE_PROVISION_SPOT, INSTANCE_PROVISION_SPOT_FLEET +from sdcm.sct_provision.aws.instance_parameters_builder import ( + LoaderInstanceParamsBuilder, + MonitorInstanceParamsBuilder, + OracleScyllaInstanceParamsBuilder, + ScyllaInstanceParamsBuilder, +) +from sdcm.sct_provision.aws.user_data import ( + AWSInstanceUserDataBuilder, + ScyllaUserDataBuilder, +) +from sdcm.sct_provision.common.utils import ( + INSTANCE_PROVISION_SPOT, + INSTANCE_PROVISION_SPOT_FLEET, +) from sdcm.test_config import TestConfig -from sdcm.provision.aws.utils import create_cluster_placement_groups_aws class ClusterNode(BaseModel): @@ -43,7 +53,7 @@ def name(self): return self.node_name_prefix + '-' + str(self.node_num) @property - def tags(self) -> Dict[str, str]: + def tags(self) -> dict[str, str]: return self.parent_cluster.tags | {'NodeIndex': str(self.node_num)} @@ -99,19 +109,19 @@ def _user_prefix(self): @property def cluster_name(self): - return '%s-%s' % (cluster.prepend_user_prefix(self._user_prefix, self._cluster_postfix), self._short_id) + return f'{cluster.prepend_user_prefix(self._user_prefix, self._cluster_postfix)}-{self._short_id}' @property def placement_group_name(self): if self.params.get("use_placement_group") and self._USE_PLACEMENT_GROUP: - return '%s-%s' % ( + return '{}-{}'.format( cluster.prepend_user_prefix(self._user_prefix, "placement_group"), self._short_id) else: return None @property def _node_prefix(self): - return '%s-%s' % (cluster.prepend_user_prefix(self._user_prefix, self._node_postfix), self._short_id) + return f'{cluster.prepend_user_prefix(self._user_prefix, self._node_postfix)}-{self._short_id}' @property def _short_id(self): @@ -121,16 +131,16 @@ def _short_id(self): def tags(self): return self.common_tags | {"NodeType": str(self._NODE_TYPE), "UserName": self.params.get(self._USER_PARAM)} - def _az_nodes(self, region_id: int) -> List[int]: + def _az_nodes(self, region_id: int) -> list[int]: az_nodes = [0] * len(self._azs) for node_num in range(self._node_nums[region_id]): az_nodes[node_num % len(self._azs)] += 1 return az_nodes - def _node_tags(self, region_id: int, az_id: int) -> List[TagsType]: + def _node_tags(self, region_id: int, az_id: int) -> list[TagsType]: return [node.tags for node in self.nodes if node.region_id == region_id and node.az_id == az_id] - def _node_names(self, region_id: int, az_id: int) -> List[str]: + def _node_names(self, region_id: int, az_id: int) -> list[str]: return [node.name for node in self.nodes if node.region_id == region_id and node.az_id == az_id] @property @@ -144,11 +154,11 @@ def _user_data(self) -> str: pass @cached_property - def _regions(self) -> List[str]: + def _regions(self) -> list[str]: return self.params.region_names @cached_property - def _regions_with_nodes(self) -> List[str]: + def _regions_with_nodes(self) -> list[str]: output = [] for region_id, region_name in enumerate(self.params.region_names): if len(self._node_nums) <= region_id: @@ -165,7 +175,7 @@ def _azs(self) -> str: return self.params.get('availability_zone').split(',') @cached_property - def _node_nums(self) -> List[int]: + def _node_nums(self) -> list[int]: node_nums = self.params.get(self._NODE_NUM_PARAM_NAME) if isinstance(node_nums, list): return [int(num) for num in node_nums] @@ -173,7 +183,7 @@ def _node_nums(self) -> List[int]: return [node_nums] if isinstance(node_nums, str): return [int(num) for num in node_nums.split()] - raise ValueError('Unexpected value of %s parameter' % (self._NODE_NUM_PARAM_NAME,)) + raise ValueError(f'Unexpected value of {self._NODE_NUM_PARAM_NAME} parameter') @property def _instance_type(self) -> str: @@ -184,7 +194,9 @@ def _test_duration(self) -> int: return self.params.get('test_duration') def _spot_low_price(self, region_id: int) -> float: - from sdcm.utils.pricing import AWSPricing # pylint: disable=import-outside-toplevel + from sdcm.utils.pricing import ( + AWSPricing, # pylint: disable=import-outside-toplevel + ) aws_pricing = AWSPricing() on_demand_price = float(aws_pricing.get_on_demand_instance_price( diff --git a/sdcm/sct_provision/aws/instance_parameters_builder.py b/sdcm/sct_provision/aws/instance_parameters_builder.py index ad202ce146d..048c0b5d48d 100644 --- a/sdcm/sct_provision/aws/instance_parameters_builder.py +++ b/sdcm/sct_provision/aws/instance_parameters_builder.py @@ -13,22 +13,28 @@ import abc from functools import cached_property -from typing import Union, List, Optional, Tuple from pydantic import Field from sdcm.cluster import UserRemoteCredentials -from sdcm.provision.aws.instance_parameters import AWSDiskMapping, AWSPlacementInfo, AWSDiskMappingEbsInfo +from sdcm.provision.aws.instance_parameters import ( + AWSDiskMapping, + AWSDiskMappingEbsInfo, + AWSPlacementInfo, +) from sdcm.provision.aws.instance_parameters_builder import AWSInstanceParamsBuilderBase from sdcm.provision.common.user_data import UserDataBuilderBase from sdcm.sct_config import SCTConfiguration -from sdcm.utils.aws_utils import ec2_ami_get_root_device_name, get_ec2_network_configuration +from sdcm.utils.aws_utils import ( + ec2_ami_get_root_device_name, + get_ec2_network_configuration, +) class AWSInstanceParamsBuilder(AWSInstanceParamsBuilderBase, metaclass=abc.ABCMeta): - params: Union[SCTConfiguration, dict] = Field(as_dict=False) + params: SCTConfiguration | dict = Field(as_dict=False) region_id: int = Field(as_dict=False) - user_data_raw: Union[str, UserDataBuilderBase] = Field(as_dict=False) + user_data_raw: str | UserDataBuilderBase = Field(as_dict=False) availability_zone: int = 0 placement_group: str = None @@ -38,7 +44,7 @@ class AWSInstanceParamsBuilder(AWSInstanceParamsBuilderBase, metaclass=abc.ABCMe _INSTANCE_PROFILE_PARAM_NAME: str = None @property - def BlockDeviceMappings(self) -> List[AWSDiskMapping]: # pylint: disable=invalid-name + def BlockDeviceMappings(self) -> list[AWSDiskMapping]: # pylint: disable=invalid-name if not self.ImageId: return [] device_mappings = [] @@ -55,7 +61,7 @@ def BlockDeviceMappings(self) -> List[AWSDiskMapping]: # pylint: disable=invali return device_mappings @property - def ImageId(self) -> Optional[str]: # pylint: disable=invalid-name + def ImageId(self) -> str | None: # pylint: disable=invalid-name if not self._image_ids: return None return self._image_ids[self.region_id] @@ -65,7 +71,7 @@ def KeyName(self) -> str: # pylint: disable=invalid-name return self._credentials[self.region_id].key_pair_name @property - def NetworkInterfaces(self) -> List[dict]: # pylint: disable=invalid-name + def NetworkInterfaces(self) -> list[dict]: # pylint: disable=invalid-name output = [{'DeviceIndex': 0, **self._network_interface_params}] if self.params.get('extra_network_interface'): output.append({'DeviceIndex': 1, **self._network_interface_params}) @@ -82,13 +88,13 @@ def InstanceType(self) -> str: # pylint: disable=invalid-name return self.params.get(self._INSTANCE_TYPE_PARAM_NAME) @property - def Placement(self) -> Optional[AWSPlacementInfo]: # pylint: disable=invalid-name + def Placement(self) -> AWSPlacementInfo | None: # pylint: disable=invalid-name return AWSPlacementInfo( AvailabilityZone=self._region_name + self._availability_zones[self.availability_zone], GroupName=self.placement_group) @property - def UserData(self) -> Optional[str]: # pylint: disable=invalid-name + def UserData(self) -> str | None: # pylint: disable=invalid-name if not self.user_data_raw: return None if isinstance(self.user_data_raw, UserDataBuilderBase): @@ -104,15 +110,15 @@ def _root_device_size(self): return self.params.get(self._ROOT_DISK_SIZE_PARAM_NAME) @cached_property - def _image_ids(self) -> List[str]: + def _image_ids(self) -> list[str]: return self.params.get(self._IMAGE_ID_PARAM_NAME).split() @cached_property - def _availability_zones(self) -> List[str]: + def _availability_zones(self) -> list[str]: return self.params.get('availability_zone').split(',') @cached_property - def _ec2_network_configuration(self) -> Tuple[List[str], List[List[str]]]: + def _ec2_network_configuration(self) -> tuple[list[str], list[list[str]]]: return get_ec2_network_configuration( regions=self.params.region_names, availability_zones=self._availability_zones, @@ -120,11 +126,11 @@ def _ec2_network_configuration(self) -> Tuple[List[str], List[List[str]]]: ) @cached_property - def _ec2_subnet_ids(self) -> List[str]: + def _ec2_subnet_ids(self) -> list[str]: return self._ec2_network_configuration[1] @cached_property - def _ec2_security_group_ids(self) -> List[str]: + def _ec2_security_group_ids(self) -> list[str]: return self._ec2_network_configuration[0] @property @@ -151,7 +157,7 @@ class ScyllaInstanceParamsBuilder(AWSInstanceParamsBuilder): _INSTANCE_PROFILE_PARAM_NAME = 'aws_instance_profile_name_db' @property - def BlockDeviceMappings(self) -> List[AWSDiskMapping]: + def BlockDeviceMappings(self) -> list[AWSDiskMapping]: device_mappings = super().BlockDeviceMappings volume_type = self.params.get('data_volume_disk_type') disk_num = self.params.get('data_volume_disk_num') diff --git a/sdcm/sct_provision/aws/layout.py b/sdcm/sct_provision/aws/layout.py index e474b62b354..2d55f8ec6de 100644 --- a/sdcm/sct_provision/aws/layout.py +++ b/sdcm/sct_provision/aws/layout.py @@ -14,7 +14,12 @@ from functools import cached_property from sdcm.sct_provision.aws.cluster import ( - OracleDBCluster, DBCluster, LoaderCluster, MonitoringCluster, PlacementGroup) + DBCluster, + LoaderCluster, + MonitoringCluster, + OracleDBCluster, + PlacementGroup, +) from sdcm.sct_provision.common.layout import SCTProvisionLayout from sdcm.test_config import TestConfig diff --git a/sdcm/sct_provision/aws/user_data.py b/sdcm/sct_provision/aws/user_data.py index 830413ea89a..7a6e2ef0fd2 100644 --- a/sdcm/sct_provision/aws/user_data.py +++ b/sdcm/sct_provision/aws/user_data.py @@ -1,19 +1,23 @@ import base64 import json -from typing import Union from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from pydantic import Field from sdcm.provision.aws.configuration_script import AWSConfigurationScriptBuilder -from sdcm.provision.common.user_data import UserDataBuilderBase, DataDeviceType, ScyllaUserDataBuilderBase, RaidLevelType +from sdcm.provision.common.user_data import ( + DataDeviceType, + RaidLevelType, + ScyllaUserDataBuilderBase, + UserDataBuilderBase, +) from sdcm.provision.scylla_yaml import ScyllaYaml from sdcm.sct_config import SCTConfiguration class ScyllaUserDataBuilder(ScyllaUserDataBuilderBase): - params: Union[SCTConfiguration, dict] = Field(as_dict=False) + params: SCTConfiguration | dict = Field(as_dict=False) cluster_name: str bootstrap: bool = Field(default=None, as_dict=False) user_data_format_version: str = Field(default='2', as_dict=False) @@ -108,7 +112,7 @@ def return_in_format_v1(self) -> str: class AWSInstanceUserDataBuilder(UserDataBuilderBase): - params: Union[SCTConfiguration, dict] = Field(as_dict=False) + params: SCTConfiguration | dict = Field(as_dict=False) syslog_host_port: tuple[str, int] = None def to_string(self) -> str: diff --git a/sdcm/sct_provision/azure/azure_region_definition_builder.py b/sdcm/sct_provision/azure/azure_region_definition_builder.py index 4a2297a04b2..058fddb833e 100644 --- a/sdcm/sct_provision/azure/azure_region_definition_builder.py +++ b/sdcm/sct_provision/azure/azure_region_definition_builder.py @@ -10,10 +10,12 @@ # See LICENSE for more details. # # Copyright (c) 2022 ScyllaDB -from typing import Dict from sdcm.sct_provision.common.types import NodeTypeType -from sdcm.sct_provision.region_definition_builder import ConfigParamsMap, DefinitionBuilder +from sdcm.sct_provision.region_definition_builder import ( + ConfigParamsMap, + DefinitionBuilder, +) db_map = ConfigParamsMap(image_id="azure_image_db", type="azure_instance_type_db", @@ -30,7 +32,7 @@ user_name="ami_monitor_user", root_disk_size="root_disk_size_monitor") -mapper: Dict[NodeTypeType, ConfigParamsMap] = {"scylla-db": db_map, +mapper: dict[NodeTypeType, ConfigParamsMap] = {"scylla-db": db_map, "loader": loader_map, "monitor": monitor_map} diff --git a/sdcm/sct_provision/instances_provider.py b/sdcm/sct_provision/instances_provider.py index fc0924f6174..19931d8681d 100644 --- a/sdcm/sct_provision/instances_provider.py +++ b/sdcm/sct_provision/instances_provider.py @@ -12,13 +12,19 @@ # Copyright (c) 2022 ScyllaDB import logging -from typing import List, Any +from typing import Any from tenacity import retry, stop_after_attempt from sdcm.provision import provisioner_factory from sdcm.provision.helpers.cloud_init import wait_cloud_init_completes -from sdcm.provision.provisioner import PricingModel, VmInstance, ProvisionError, Provisioner, InstanceDefinition +from sdcm.provision.provisioner import ( + InstanceDefinition, + PricingModel, + Provisioner, + ProvisionError, + VmInstance, +) from sdcm.remote import RemoteCmdRunnerBase from sdcm.sct_config import SCTConfiguration from sdcm.sct_provision import region_definition_builder @@ -28,14 +34,14 @@ @retry(stop=stop_after_attempt(3), reraise=True) -def provision_with_retry(provisioner: Provisioner, definitions: List[InstanceDefinition], pricing_model: PricingModel - ) -> List[VmInstance]: +def provision_with_retry(provisioner: Provisioner, definitions: list[InstanceDefinition], pricing_model: PricingModel + ) -> list[VmInstance]: return provisioner.get_or_create_instances(definitions=definitions, pricing_model=pricing_model) -def provision_instances_with_fallback(provisioner: Provisioner, definitions: List[InstanceDefinition], pricing_model: PricingModel, +def provision_instances_with_fallback(provisioner: Provisioner, definitions: list[InstanceDefinition], pricing_model: PricingModel, fallback_on_demand: bool - ) -> List[VmInstance]: + ) -> list[VmInstance]: try: provision_with_retry(provisioner, definitions=definitions, pricing_model=pricing_model) except ProvisionError: diff --git a/sdcm/sct_provision/region_definition_builder.py b/sdcm/sct_provision/region_definition_builder.py index 0517184b88e..4f8b3146327 100644 --- a/sdcm/sct_provision/region_definition_builder.py +++ b/sdcm/sct_provision/region_definition_builder.py @@ -13,13 +13,11 @@ import abc from dataclasses import dataclass from functools import cache -from typing import List, Dict, Type from sdcm.keystore import KeyStore, SSHKey from sdcm.provision.provisioner import InstanceDefinition from sdcm.sct_config import SCTConfiguration from sdcm.sct_provision.common.types import NodeTypeType - from sdcm.sct_provision.user_data_objects import SctUserDataObject from sdcm.sct_provision.user_data_objects.scylla import ScyllaUserDataObject from sdcm.sct_provision.user_data_objects.sshd import SshdUserDataObject @@ -36,7 +34,7 @@ class RegionDefinition: test_id: str region: str availability_zone: str - definitions: List[InstanceDefinition] + definitions: list[InstanceDefinition] @dataclass @@ -55,7 +53,7 @@ class DefinitionBuilder(abc.ABC): which maps sct params to proper attributes in InstanceDefinition. """ BACKEND: str - SCT_PARAM_MAPPER: Dict[NodeTypeType, ConfigParamsMap] + SCT_PARAM_MAPPER: dict[NodeTypeType, ConfigParamsMap] REGION_MAP: str def __init__(self, params: SCTConfiguration, test_config: TestConfig) -> None: @@ -64,7 +62,7 @@ def __init__(self, params: SCTConfiguration, test_config: TestConfig) -> None: self.test_id = self.params.get("test_id") @property - def regions(self) -> List[str]: + def regions(self) -> list[str]: return self.params.get(self.REGION_MAP) def build_instance_definition(self, region: str, node_type: NodeTypeType, index: int, instance_type: str = None) -> InstanceDefinition: @@ -108,7 +106,7 @@ def build_region_definition(self, region: str, availability_zone: str, n_db_node return RegionDefinition(backend=self.BACKEND, test_id=self.test_id, region=region, availability_zone=availability_zone, definitions=definitions) - def build_all_region_definitions(self) -> List[RegionDefinition]: + def build_all_region_definitions(self) -> list[RegionDefinition]: """Builds all instances definitions in all regions based on SCT test configuration.""" region_definitions = [] availability_zone = self.params.get("availability_zone") @@ -127,7 +125,7 @@ def build_all_region_definitions(self) -> List[RegionDefinition]: def _get_ssh_key() -> SSHKey: return KeyStore().get_gce_ssh_key_pair() - def _get_node_count_for_each_region(self, n_str: str) -> List[int]: + def _get_node_count_for_each_region(self, n_str: str) -> list[int]: """generates node count for each region from configuration parameter string (e.g. n_db_nodes). When parameter string has less regions defined than regions, fills with zero for each missing region. @@ -136,8 +134,8 @@ def _get_node_count_for_each_region(self, n_str: str) -> List[int]: region_count = len(regions) return ([int(v) for v in str(n_str).split()] + [0] * region_count)[:region_count] - def _get_user_data_objects(self, instance_name: str, node_type: NodeTypeType) -> List[SctUserDataObject]: - user_data_object_classes: List[Type[SctUserDataObject]] = [ + def _get_user_data_objects(self, instance_name: str, node_type: NodeTypeType) -> list[SctUserDataObject]: + user_data_object_classes: list[type[SctUserDataObject]] = [ SyslogNgUserDataObject, SshdUserDataObject, ScyllaUserDataObject, @@ -159,7 +157,7 @@ class RegionDefinitionBuilder: def __init__(self) -> None: self._builder_classes = {} - def register_builder(self, backend: str, builder_class: Type[DefinitionBuilder]) -> None: + def register_builder(self, backend: str, builder_class: type[DefinitionBuilder]) -> None: """Registers builder for given backend Must be used before calling RegionDefinitionBuilder for given backend.""" diff --git a/sdcm/sct_provision/user_data_objects/sshd.py b/sdcm/sct_provision/user_data_objects/sshd.py index 0bd9b8c2958..1563b871022 100644 --- a/sdcm/sct_provision/user_data_objects/sshd.py +++ b/sdcm/sct_provision/user_data_objects/sshd.py @@ -12,7 +12,11 @@ # Copyright (c) 2022 ScyllaDB from dataclasses import dataclass -from sdcm.provision.common.utils import configure_sshd_script, restart_sshd_service, configure_ssh_accept_rsa +from sdcm.provision.common.utils import ( + configure_ssh_accept_rsa, + configure_sshd_script, + restart_sshd_service, +) from sdcm.sct_provision.user_data_objects import SctUserDataObject diff --git a/sdcm/sct_provision/user_data_objects/syslog_ng.py b/sdcm/sct_provision/user_data_objects/syslog_ng.py index b006d3ba7ce..cd997a368f6 100644 --- a/sdcm/sct_provision/user_data_objects/syslog_ng.py +++ b/sdcm/sct_provision/user_data_objects/syslog_ng.py @@ -13,7 +13,10 @@ from dataclasses import dataclass from sdcm.provision.common.configuration_script import SYSLOGNG_LOG_THROTTLE_PER_SECOND -from sdcm.provision.common.utils import configure_syslogng_target_script, restart_syslogng_service +from sdcm.provision.common.utils import ( + configure_syslogng_target_script, + restart_syslogng_service, +) from sdcm.sct_provision.user_data_objects import SctUserDataObject diff --git a/sdcm/sct_runner.py b/sdcm/sct_runner.py index 9e19b309fe6..028d4deeb0d 100644 --- a/sdcm/sct_runner.py +++ b/sdcm/sct_runner.py @@ -14,58 +14,74 @@ #pylint: disable=too-many-lines from __future__ import annotations +import datetime import logging -import string import random +import string import tempfile -import datetime +from abc import ABC, abstractmethod from contextlib import suppress +from dataclasses import dataclass, field from enum import Enum from functools import cached_property from itertools import chain from math import ceil from typing import TYPE_CHECKING -from abc import ABC, abstractmethod -from dataclasses import dataclass, field import boto3 +import google.api_core.exceptions import pytz from azure.core.exceptions import ResourceNotFoundError as AzureResourceNotFoundError from azure.mgmt.compute.models import GalleryImageVersion from azure.mgmt.compute.v2021_07_01.models import VirtualMachine -from azure.mgmt.resource.resources.v2021_04_01.models import TagsPatchResource, TagsPatchOperation -import google.api_core.exceptions +from azure.mgmt.resource.resources.v2021_04_01.models import ( + TagsPatchOperation, + TagsPatchResource, +) from google.cloud import compute_v1 from mypy_boto3_ec2 import EC2Client from mypy_boto3_ec2.service_resource import Instance from sct_ssh import ssh_run_cmd from sdcm.keystore import KeyStore -from sdcm.provision.provisioner import InstanceDefinition, PricingModel, VmInstance, provisioner_factory +from sdcm.provision.provisioner import ( + InstanceDefinition, + PricingModel, + VmInstance, + provisioner_factory, +) from sdcm.remote import RemoteCmdRunnerBase, shell_script_cmd -from sdcm.utils.common import list_instances_aws, aws_tags_to_dict, list_instances_gce, gce_meta_to_dict -from sdcm.utils.aws_utils import ec2_instance_wait_public_ip, ec2_ami_get_root_device_name +from sdcm.test_config import TestConfig from sdcm.utils.aws_region import AwsRegion +from sdcm.utils.aws_utils import ( + ec2_ami_get_root_device_name, + ec2_instance_wait_public_ip, +) +from sdcm.utils.azure_region import AzureOsState, AzureRegion, region_name_to_location +from sdcm.utils.azure_utils import AzureService, list_instances_azure +from sdcm.utils.common import ( + aws_tags_to_dict, + gce_meta_to_dict, + list_instances_aws, + list_instances_gce, +) +from sdcm.utils.context_managers import environment from sdcm.utils.gce_utils import ( SUPPORTED_PROJECTS, + create_instance, + disk_from_image, + gce_public_addresses, + gce_set_labels, get_gce_compute_addresses_client, get_gce_compute_images_client, get_gce_compute_instances_client, - create_instance, - disk_from_image, - wait_for_extended_operation, random_zone, - gce_set_labels, - gce_public_addresses, + wait_for_extended_operation, ) -from sdcm.utils.azure_utils import AzureService, list_instances_azure -from sdcm.utils.azure_region import AzureOsState, AzureRegion, region_name_to_location -from sdcm.utils.context_managers import environment -from sdcm.test_config import TestConfig if TYPE_CHECKING: # pylint: disable=ungrouped-imports - from typing import Optional, Any, Type + from typing import Any from mypy_boto3_ec2.literals import InstanceTypeType @@ -92,16 +108,16 @@ def datetime_from_formatted(date_string: str) -> datetime.datetime: @dataclass class SctRunnerInfo: # pylint: disable=too-many-instance-attributes - sct_runner_class: Type[SctRunner] = field(repr=False) + sct_runner_class: type[SctRunner] = field(repr=False) cloud_service_instance: EC2Client | AzureService | None = field(repr=False) region_az: str instance: VirtualMachine | compute_v1.Instance | Any = field(repr=False) instance_name: str public_ips: list[str] test_id: str | None = None - launch_time: Optional[datetime.datetime] = None - keep: Optional[str] = None - keep_action: Optional[str] = None + launch_time: datetime.datetime | None = None + keep: str | None = None + keep_action: str | None = None logs_collected: bool = False @property @@ -175,15 +191,17 @@ def instance_root_disk_size(test_duration) -> int: def key_pair(self) -> SSHKey: ... - def get_remoter(self, host, connect_timeout: Optional[float] = None) -> RemoteCmdRunnerBase: + def get_remoter(self, host, connect_timeout: float | None = None) -> RemoteCmdRunnerBase: self._ssh_pkey_file = tempfile.NamedTemporaryFile(mode="w", delete=False) # pylint: disable=consider-using-with self._ssh_pkey_file.write(self.key_pair.private_key.decode()) self._ssh_pkey_file.flush() return RemoteCmdRunnerBase.create_remoter(hostname=host, user=self.LOGIN_USER, key_file=self._ssh_pkey_file.name, connect_timeout=connect_timeout) - def install_prereqs(self, public_ip: str, connect_timeout: Optional[int] = None) -> None: - from sdcm.cluster_docker import AIO_MAX_NR_RECOMMENDED_VALUE # pylint: disable=import-outside-toplevel + def install_prereqs(self, public_ip: str, connect_timeout: int | None = None) -> None: + from sdcm.cluster_docker import ( + AIO_MAX_NR_RECOMMENDED_VALUE, # pylint: disable=import-outside-toplevel + ) LOGGER.info("Connecting instance...") remoter = self.get_remoter(host=public_ip, connect_timeout=connect_timeout) @@ -377,7 +395,7 @@ def create_image(self) -> None: LOGGER.info("No need to copy SCT Runner image since it already exists in `%s'", self.region_name) @abstractmethod - def _get_base_image(self, image: Optional[Any] = None) -> Any: + def _get_base_image(self, image: Any | None = None) -> Any: ... def create_instance(self, # pylint: disable=too-many-arguments # noqa: PLR0913 @@ -840,7 +858,7 @@ def _get_image_id(self, image: Any) -> Any: def _copy_source_image_to_region(self) -> None: LOGGER.debug("gce images are global, not need to copy") - def _get_base_image(self, image: Optional[Any] = None) -> Any: + def _get_base_image(self, image: Any | None = None) -> Any: if image is None: image = self.image return image.self_link @@ -1031,7 +1049,7 @@ def _copy_source_image_to_region(self) -> None: region_name=self.region_name, ) - def _get_base_image(self, image: Optional[Any] = None) -> Any: + def _get_base_image(self, image: Any | None = None) -> Any: if image is None: image = self.image if isinstance(image, GalleryImageVersion): @@ -1209,7 +1227,7 @@ def clean_sct_runners(test_status: str, # noqa: PLR0912 timeout_flag = bool(ssh_run_cmd_result.stdout) if ssh_run_cmd_result else False - utc_now = datetime.datetime.now(tz=datetime.timezone.utc) + utc_now = datetime.datetime.now(tz=datetime.UTC) LOGGER.info("UTC now: %s", utc_now) if not dry_run and test_runner_ip: diff --git a/sdcm/scylla_bench_thread.py b/sdcm/scylla_bench_thread.py index 6b280d1952a..c7d363f7ce0 100644 --- a/sdcm/scylla_bench_thread.py +++ b/sdcm/scylla_bench_thread.py @@ -11,23 +11,22 @@ # # Copyright (c) 2020 ScyllaDB +import contextlib +import logging import os import re -import uuid import time -import logging -import contextlib +import uuid from enum import Enum from sdcm.loader import ScyllaBenchStressExporter from sdcm.prometheus import nemesis_metrics_obj -from sdcm.sct_events.loaders import ScyllaBenchEvent, SCYLLA_BENCH_ERROR_EVENTS_PATTERNS -from sdcm.utils.common import FileFollowerThread, convert_metric_to_ms +from sdcm.sct_events.loaders import SCYLLA_BENCH_ERROR_EVENTS_PATTERNS, ScyllaBenchEvent from sdcm.stress_thread import DockerBasedStressThread +from sdcm.utils.common import FileFollowerThread, convert_metric_to_ms from sdcm.utils.docker_remote import RemoteDocker from sdcm.wait import wait_for - LOGGER = logging.getLogger(__name__) diff --git a/sdcm/send_email.py b/sdcm/send_email.py index 62a865969ff..6cced587119 100644 --- a/sdcm/send_email.py +++ b/sdcm/send_email.py @@ -11,15 +11,15 @@ # # Copyright (c) 2020 ScyllaDB -import smtplib +import copy +import json +import logging import os.path +import smtplib import subprocess -import logging import tempfile -import json -import copy import traceback -from typing import Optional, Sequence, Tuple +from collections.abc import Sequence from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -28,7 +28,12 @@ import jinja2 from sdcm.keystore import KeyStore -from sdcm.utils.common import list_instances_gce, list_instances_aws, list_resources_docker, format_timestamp +from sdcm.utils.common import ( + format_timestamp, + list_instances_aws, + list_instances_gce, + list_resources_docker, +) from sdcm.utils.gce_utils import gce_public_addresses LOGGER = logging.getLogger(__name__) @@ -48,7 +53,7 @@ def __init__(self, current_size, limit): super().__init__() -class Email(): +class Email: # pylint: disable=too-many-instance-attributes """ Responsible for sending emails @@ -73,7 +78,7 @@ def _retrieve_credentials(self): self._password = creds["password"] def _connect(self): - self.conn = smtplib.SMTP(host=self._server_host, port=self._server_port) + self.conn = smtplib.SMTP(host=self._server_host, port=int(self._server_port)) self.conn.ehlo() self.conn.starttls() self.conn.login(user=self._user, password=self._password) @@ -168,7 +173,7 @@ def __init__(self, email_recipients=(), email_template_fp=None, logger=None, log self.logdir = logdir if logdir else tempfile.mkdtemp() @cached_property - def fields(self) -> Tuple[str, ...]: + def fields(self) -> tuple[str, ...]: return self.COMMON_EMAIL_FIELDS + self._fields def build_data_for_report(self, results): @@ -573,7 +578,7 @@ class PerfSimpleQueryReporter(BaseEmailReporter): def build_reporter(name: str, # noqa: PLR0912, PLR0911 email_recipients: Sequence[str] = (), - logdir: Optional[str] = None) -> Optional[BaseEmailReporter]: + logdir: str | None = None) -> BaseEmailReporter | None: # pylint: disable=too-many-return-statements,too-many-branches if "Gemini" in name: return GeminiEmailReporter(email_recipients=email_recipients, logdir=logdir) diff --git a/sdcm/sla/libs/sla_utils.py b/sdcm/sla/libs/sla_utils.py index 3b76b979082..175d4ab000c 100644 --- a/sdcm/sla/libs/sla_utils.py +++ b/sdcm/sla/libs/sla_utils.py @@ -179,7 +179,7 @@ def validate_scheduler_runtime(self, start_time, end_time, read_users, prometheu # Example of scheduler_runtime_per_sla: # {'10.0.2.177': {'sl:default': [410.5785714285715, 400.36428571428576], # 'sl:sl500_596ca81a': [177.11428571428573, 182.02857142857144]} - LOGGER.debug('SERVICE LEVEL GROUP - RUNTIMES: {}'.format(scheduler_runtime_per_sla)) + LOGGER.debug(f'SERVICE LEVEL GROUP - RUNTIMES: {scheduler_runtime_per_sla}') if not scheduler_runtime_per_sla: # Set this message as WARNING because I found that prometheus return empty answer despite the data # exists (I run this request manually and got data). Prometheus request doesn't fail, it succeeded but @@ -203,7 +203,7 @@ def validate_scheduler_runtime(self, start_time, end_time, read_users, prometheu if role_sl_attribute['sl_group_runtime'] == 0.0: # noqa: PLR2004 sl_group_runtime_zero = True - LOGGER.debug('RUN TIME PER ROLE: {}'.format(roles_full_info)) + LOGGER.debug(f'RUN TIME PER ROLE: {roles_full_info}') # We know and validate expected_ratio in the feature test. In the longevity we can not perform such kind # of validation because WP load runs in parallel with disruptive and non-disruptive nemeses, so we can not diff --git a/sdcm/sla/sla_tests.py b/sdcm/sla/sla_tests.py index 0b58c8c7e34..0297007b54a 100644 --- a/sdcm/sla/sla_tests.py +++ b/sdcm/sla/sla_tests.py @@ -7,10 +7,14 @@ from sdcm.sct_events import Severity from sdcm.sct_events.system import TestStepEvent from sdcm.sla.libs.sla_utils import SlaUtils -from sdcm.utils.adaptive_timeouts import adaptive_timeout, Operations +from sdcm.utils.adaptive_timeouts import Operations, adaptive_timeout from sdcm.utils.decorators import retrying -from sdcm.utils.loader_utils import DEFAULT_USER, DEFAULT_USER_PASSWORD, SERVICE_LEVEL_NAME_TEMPLATE -from test_lib.sla import create_sla_auth, ServiceLevel, Role +from sdcm.utils.loader_utils import ( + DEFAULT_USER, + DEFAULT_USER_PASSWORD, + SERVICE_LEVEL_NAME_TEMPLATE, +) +from test_lib.sla import Role, ServiceLevel, create_sla_auth LOGGER = logging.getLogger(__name__) diff --git a/sdcm/snitch_configuration.py b/sdcm/snitch_configuration.py index dcfaa583cfa..1dd63e2ab12 100644 --- a/sdcm/snitch_configuration.py +++ b/sdcm/snitch_configuration.py @@ -12,13 +12,12 @@ # Copyright (c) 2023 ScyllaDB import re -from typing import List class SnitchConfig: # pylint: disable=too-few-public-methods """Keeps all cassandra-rackdc.properties settings and function to apply them""" - def __init__(self, node: "sdcm.cluster.BaseNode", datacenters: List[str]): + def __init__(self, node: "sdcm.cluster.BaseNode", datacenters: list[str]): self._node = node self._is_multi_dc = len(datacenters) > 1 self._rack = f"RACK{node.rack}" @@ -36,7 +35,7 @@ def _get_dc_suffix(self) -> str: if self._is_multi_dc: ret = re.findall('-([a-z]+).*-', self._datacenter) if ret: - dc_suffix = 'scylla_node_{}'.format(ret[0]) + dc_suffix = f'scylla_node_{ret[0]}' else: dc_suffix = self._dc_prefix.replace('-', '_') return dc_suffix diff --git a/sdcm/stress/base.py b/sdcm/stress/base.py index 5147d292b8b..f63531dd3fd 100644 --- a/sdcm/stress/base.py +++ b/sdcm/stress/base.py @@ -11,18 +11,19 @@ # # Copyright (c) 2019 ScyllaDB +import concurrent.futures import logging import random -import concurrent.futures -from pathlib import Path from functools import cached_property +from pathlib import Path from sdcm.cluster import BaseLoaderSet -from sdcm.utils.common import generate_random_string -from sdcm.utils.docker_remote import RemoteDocker +from sdcm.remote.libssh2_client.exceptions import Failure from sdcm.sct_events import Severity from sdcm.sct_events.stress_events import StressEvent -from sdcm.remote.libssh2_client.exceptions import Failure +from sdcm.utils.common import generate_random_string +from sdcm.utils.docker_remote import RemoteDocker + LOGGER = logging.getLogger(__name__) @@ -60,7 +61,7 @@ def configure_executer(self): if self.round_robin: self.stress_num = 1 loaders = [self.loader_set.get_loader()] - LOGGER.debug("Round-Robin through loaders, Selected loader is {} ".format(loaders)) + LOGGER.debug(f"Round-Robin through loaders, Selected loader is {loaders} ") else: loaders = self.loader_set.nodes self.loaders = loaders diff --git a/sdcm/stress/latte_thread.py b/sdcm/stress/latte_thread.py index 607da7be790..f135338f24d 100644 --- a/sdcm/stress/latte_thread.py +++ b/sdcm/stress/latte_thread.py @@ -11,23 +11,23 @@ # # Copyright (c) 2023 ScyllaDB +import logging import os import re import time import uuid -import logging from pathlib import Path from sdcm.prometheus import nemesis_metrics_obj from sdcm.sct_events.loaders import LatteStressEvent +from sdcm.stress.base import DockerBasedStressThread from sdcm.utils.common import ( FileFollowerThread, generate_random_string, - get_sct_root_path, get_data_dir_path, + get_sct_root_path, ) from sdcm.utils.docker_remote import RemoteDocker -from sdcm.stress.base import DockerBasedStressThread LOGGER = logging.getLogger(__name__) @@ -178,8 +178,7 @@ def _run_stress(self, loader, loader_idx, cpu_idx): if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) - log_file_name = os.path.join(loader.logdir, 'latte-l%s-c%s-%s.log' % - (loader_idx, cpu_idx, uuid.uuid4())) + log_file_name = os.path.join(loader.logdir, f'latte-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log') LOGGER.debug('latter-stress local log: %s', log_file_name) LOGGER.debug("running: %s", stress_cmd) diff --git a/sdcm/stress_thread.py b/sdcm/stress_thread.py index edf04c19f2a..e95e738abc2 100644 --- a/sdcm/stress_thread.py +++ b/sdcm/stress_thread.py @@ -11,29 +11,37 @@ # # Copyright (c) 2019 ScyllaDB +import contextlib +import logging import os import re import time import uuid -import logging -import contextlib -from typing import Any +from functools import cached_property from itertools import chain from pathlib import Path -from functools import cached_property +from typing import Any -from sdcm.loader import CassandraStressExporter, CassandraStressHDRExporter from sdcm.cluster import BaseLoaderSet, BaseNode +from sdcm.loader import CassandraStressExporter, CassandraStressHDRExporter from sdcm.prometheus import nemesis_metrics_obj from sdcm.sct_events import Severity -from sdcm.utils.common import FileFollowerThread, get_data_dir_path, time_period_str_to_seconds, SoftTimeoutContext -from sdcm.utils.user_profile import get_profile_content, replace_scylla_qa_internal_path -from sdcm.sct_events.loaders import CassandraStressEvent, CS_ERROR_EVENTS_PATTERNS, CS_NORMAL_EVENTS_PATTERNS +from sdcm.sct_events.loaders import ( + CS_ERROR_EVENTS_PATTERNS, + CS_NORMAL_EVENTS_PATTERNS, + CassandraStressEvent, +) from sdcm.stress.base import DockerBasedStressThread +from sdcm.utils.common import ( + FileFollowerThread, + SoftTimeoutContext, + get_data_dir_path, + time_period_str_to_seconds, +) from sdcm.utils.docker_remote import RemoteDocker -from sdcm.utils.version_utils import get_docker_image_by_version from sdcm.utils.remote_logger import SSHLoggerBase - +from sdcm.utils.user_profile import get_profile_content, replace_scylla_qa_internal_path +from sdcm.utils.version_utils import get_docker_image_by_version LOGGER = logging.getLogger(__name__) @@ -122,9 +130,9 @@ def create_stress_cmd(self, cmd_runner, keyspace_idx, loader): # pylint: disabl self.keyspace_name = keyspace_name if self.keyspace_name: - stress_cmd = stress_cmd.replace(" -schema ", " -schema keyspace={} ".format(self.keyspace_name)) + stress_cmd = stress_cmd.replace(" -schema ", f" -schema keyspace={self.keyspace_name} ") elif 'keyspace=' not in stress_cmd: # if keyspace is defined in the command respect that - stress_cmd = stress_cmd.replace(" -schema ", " -schema keyspace=keyspace{} ".format(keyspace_idx)) + stress_cmd = stress_cmd.replace(" -schema ", f" -schema keyspace=keyspace{keyspace_idx} ") if self.compaction_strategy and "compaction(" not in stress_cmd: stress_cmd = stress_cmd.replace(" -schema ", f" -schema 'compaction(strategy={self.compaction_strategy})' ") @@ -389,7 +397,7 @@ def verify_results(self) -> (list[dict | None], list[str | None]): cs_summary.append(node_cs_res) for line in lines: if 'java.io.IOException' in line: - errors += ['%s: %s' % (node, line.strip())] + errors += [f'{node}: {line.strip()}'] return cs_summary, errors diff --git a/sdcm/test_config.py b/sdcm/test_config.py index 9536733ef59..c4d87ac48a8 100644 --- a/sdcm/test_config.py +++ b/sdcm/test_config.py @@ -3,25 +3,22 @@ import os from datetime import datetime from pathlib import Path -from typing import Optional, Dict from unittest.mock import MagicMock import requests from argus.client.sct.client import ArgusSCTClient - from sdcm.keystore import KeyStore from sdcm.provision.common.configuration_script import ConfigurationScriptBuilder from sdcm.sct_events import Severity from sdcm.sct_events.system import TestFrameworkEvent from sdcm.utils.argus import ArgusError, get_argus_client -from sdcm.utils.net import get_my_ip from sdcm.utils.decorators import retrying from sdcm.utils.docker_utils import ContainerManager from sdcm.utils.get_username import get_username from sdcm.utils.ldap import LdapServerNotReady from sdcm.utils.metaclasses import Singleton - +from sdcm.utils.net import get_my_ip LOGGER = logging.getLogger(__name__) @@ -156,7 +153,7 @@ def keep_cluster(cls, node_type, val='destroy'): cls.KEEP_ALIVE_MONITOR_NODES = bool(val == 'keep') @classmethod - def should_keep_alive(cls, node_type: Optional[str]) -> bool: + def should_keep_alive(cls, node_type: str | None) -> bool: if cls.TEST_DURATION >= 11 * 60: return True if node_type is None: @@ -174,7 +171,7 @@ def mixed_cluster(cls, val=False): cls.MIXED_CLUSTER = val @classmethod - def common_tags(cls) -> Dict[str, str]: + def common_tags(cls) -> dict[str, str]: job_name = os.environ.get('JOB_NAME') tags = dict(RunByUser=get_username(), TestName=str(cls.test_name()), @@ -243,7 +240,7 @@ def configure_rsyslog(cls, node, enable_ngrok=False): "bind_tls": False } res = requests.post('http://localhost:4040/api/tunnels', json=tunnel) - assert res.ok, "failed to add a ngrok tunnel [{}, {}]".format(res, res.text) + assert res.ok, f"failed to add a ngrok tunnel [{res}, {res.text}]" ngrok_address = res.json()['public_url'].replace('tcp://', '') address, port = ngrok_address.split(':') diff --git a/sdcm/tester.py b/sdcm/tester.py index e307ceb5649..40ff3a79a2f 100644 --- a/sdcm/tester.py +++ b/sdcm/tester.py @@ -11,121 +11,180 @@ # # Copyright (c) 2016 ScyllaDB # pylint: disable=too-many-lines -from collections import defaultdict -from copy import deepcopy -from dataclasses import asdict - +import json import logging import os import re +import signal +import threading import time import traceback import unittest import unittest.mock -from typing import NamedTuple, Optional, Union, List, Dict, Any +from collections import defaultdict +from copy import deepcopy +from dataclasses import asdict +from functools import cache, wraps +from typing import Any, NamedTuple from uuid import uuid4 -from functools import wraps, cache -import threading -import signal -import json import botocore import yaml -from invoke.exceptions import UnexpectedExit, Failure - -from cassandra.concurrent import execute_concurrent_with_args # pylint: disable=no-name-in-module +from argus.backend.util.enums import TestStatus +from argus.client.base import ArgusClientError +from argus.client.sct.client import ArgusSCTClient +from argus.client.sct.types import EventsInfo, LogLink, Package from cassandra import ConsistencyLevel from cassandra.cluster import Session # pylint: disable=no-name-in-module +from cassandra.concurrent import ( + execute_concurrent_with_args, # pylint: disable=no-name-in-module +) from cassandra.query import SimpleStatement # pylint: disable=no-name-in-module +from invoke.exceptions import Failure, UnexpectedExit -from argus.client.sct.client import ArgusSCTClient -from argus.client.base import ArgusClientError -from argus.client.sct.types import Package, EventsInfo, LogLink -from argus.backend.util.enums import TestStatus -from sdcm import nemesis, cluster_docker, cluster_k8s, cluster_baremetal, db_stats, wait -from sdcm.cluster import BaseCluster, NoMonitorSet, SCYLLA_DIR, TestConfig, UserRemoteCredentials, BaseLoaderSet, BaseMonitorSet, \ - BaseScyllaCluster, BaseNode, MINUTE_IN_SEC -from sdcm.cluster_azure import ScyllaAzureCluster, LoaderSetAzure, MonitorSetAzure -from sdcm.cluster_gce import ScyllaGCECluster -from sdcm.cluster_gce import LoaderSetGCE -from sdcm.cluster_gce import MonitorSetGCE -from sdcm.cluster_aws import CassandraAWSCluster -from sdcm.cluster_aws import ScyllaAWSCluster -from sdcm.cluster_aws import LoaderSetAWS -from sdcm.cluster_aws import MonitorSetAWS -from sdcm.cluster_k8s import mini_k8s, gke, eks +from sdcm import cluster_baremetal, cluster_docker, cluster_k8s, db_stats, nemesis, wait +from sdcm.cassandra_harry_thread import CassandraHarryThread +from sdcm.cdclog_reader_thread import CDCLogReaderThread +from sdcm.cluster import ( + MINUTE_IN_SEC, + SCYLLA_DIR, + BaseCluster, + BaseLoaderSet, + BaseMonitorSet, + BaseNode, + BaseScyllaCluster, + NoMonitorSet, + TestConfig, + UserRemoteCredentials, +) +from sdcm.cluster_aws import ( + CassandraAWSCluster, + LoaderSetAWS, + MonitorSetAWS, + ScyllaAWSCluster, +) +from sdcm.cluster_azure import LoaderSetAzure, MonitorSetAzure, ScyllaAzureCluster +from sdcm.cluster_gce import LoaderSetGCE, MonitorSetGCE, ScyllaGCECluster +from sdcm.cluster_k8s import eks, gke, mini_k8s from sdcm.cluster_k8s.eks import MonitorSetEKS +from sdcm.db_stats import PrometheusDBStats +from sdcm.gemini_thread import GeminiStressThread +from sdcm.kcl_thread import CompareTablesSizesThread, KclStressThread +from sdcm.keystore import KeyStore +from sdcm.localhost import LocalHost +from sdcm.logcollector import ( + BaseSCTLogCollector, + KubernetesAPIServerLogCollector, + KubernetesLogCollector, + KubernetesMustGatherLogCollector, + LoaderLogCollector, + MonitorLogCollector, + PythonSCTLogCollector, + ScyllaLogCollector, + SirenManagerLogCollector, +) +from sdcm.ndbench_thread import NdBenchStressThread +from sdcm.nosql_thread import NoSQLBenchStressThread from sdcm.provision.azure.provisioner import AzureProvisioner from sdcm.provision.provisioner import provisioner_factory +from sdcm.remote import RemoteCmdRunnerBase +from sdcm.results_analyze import ( + LatencyDuringOperationsPerformanceAnalyzer, + PerformanceResultsAnalyzer, + SpecifiedStatsPerformanceAnalyzer, +) from sdcm.scan_operation_thread import ScanOperationThread -from sdcm.nosql_thread import NoSQLBenchStressThread +from sdcm.sct_config import init_and_verify_sct_config +from sdcm.sct_events import Severity +from sdcm.sct_events.events_analyzer import stop_events_analyzer +from sdcm.sct_events.file_logger import ( + get_events_grouped_by_category, + get_logger_event_summary, +) +from sdcm.sct_events.grafana import start_posting_grafana_annotations +from sdcm.sct_events.setup import start_events_device, stop_events_device +from sdcm.sct_events.system import ( + InfoEvent, + TestFrameworkEvent, + TestResultEvent, + TestTimeoutEvent, +) from sdcm.scylla_bench_thread import ScyllaBenchThread -from sdcm.cassandra_harry_thread import CassandraHarryThread +from sdcm.send_email import ( + build_reporter, + get_running_instances_for_email_report, + read_email_data_from_file, + save_email_data_to_file, +) +from sdcm.stress.latte_thread import LatteStressThread +from sdcm.stress_thread import CassandraStressThread, get_timeout_from_stress_cmd from sdcm.tombstone_gc_verification_thread import TombstoneGcVerificationThread +from sdcm.utils import alternator from sdcm.utils.alternator.consts import NO_LWT_TABLE_NAME +from sdcm.utils.auth_context import temp_authenticator from sdcm.utils.aws_kms import AwsKms from sdcm.utils.aws_region import AwsRegion -from sdcm.utils.aws_utils import init_monitoring_info_from_params, get_ec2_services, \ - get_common_params, init_db_info_from_params, ec2_ami_get_root_device_name +from sdcm.utils.aws_utils import ( + ec2_ami_get_root_device_name, + get_common_params, + get_ec2_services, + init_db_info_from_params, + init_monitoring_info_from_params, +) from sdcm.utils.ci_tools import get_job_name, get_job_url -from sdcm.utils.common import format_timestamp, wait_ami_available, update_certificates, \ - download_dir_from_cloud, get_post_behavior_actions, get_testrun_status, download_encrypt_keys, rows_to_list, \ - make_threads_be_daemonic_by_default, ParallelObject, clear_out_all_exit_hooks, change_default_password -from sdcm.utils.database_query_utils import PartitionsValidationAttributes, fetch_all_rows -from sdcm.utils.get_username import get_username +from sdcm.utils.common import ( + ParallelObject, + change_default_password, + clear_out_all_exit_hooks, + download_dir_from_cloud, + download_encrypt_keys, + format_timestamp, + get_post_behavior_actions, + get_testrun_status, + make_threads_be_daemonic_by_default, + rows_to_list, + update_certificates, + wait_ami_available, +) +from sdcm.utils.csrangehistogram import ( + CSHistogramTagTypes, + CSWorkloadTypes, + make_cs_range_histogram_summary, + make_cs_range_histogram_summary_by_interval, +) +from sdcm.utils.database_query_utils import ( + PartitionsValidationAttributes, + fetch_all_rows, +) from sdcm.utils.decorators import log_run_info, retrying +from sdcm.utils.gce_utils import get_gce_compute_instances_client +from sdcm.utils.get_username import get_username from sdcm.utils.git import get_git_commit_id, get_git_status_info -from sdcm.utils.ldap import LDAP_USERS, LDAP_PASSWORD, LDAP_ROLE, LDAP_BASE_OBJECT, \ - LdapConfigurationError, LdapServerType +from sdcm.utils.latency import analyze_hdr_percentiles, calculate_latency +from sdcm.utils.ldap import ( + LDAP_BASE_OBJECT, + LDAP_PASSWORD, + LDAP_ROLE, + LDAP_USERS, + LdapConfigurationError, + LdapServerType, +) from sdcm.utils.log import configure_logging, handle_exception -from sdcm.db_stats import PrometheusDBStats -from sdcm.results_analyze import PerformanceResultsAnalyzer, SpecifiedStatsPerformanceAnalyzer, \ - LatencyDuringOperationsPerformanceAnalyzer -from sdcm.sct_config import init_and_verify_sct_config -from sdcm.sct_events import Severity -from sdcm.sct_events.setup import start_events_device, stop_events_device -from sdcm.sct_events.system import InfoEvent, TestFrameworkEvent, TestResultEvent, TestTimeoutEvent -from sdcm.sct_events.file_logger import get_events_grouped_by_category, get_logger_event_summary -from sdcm.sct_events.events_analyzer import stop_events_analyzer -from sdcm.sct_events.grafana import start_posting_grafana_annotations -from sdcm.stress_thread import CassandraStressThread, get_timeout_from_stress_cmd -from sdcm.gemini_thread import GeminiStressThread from sdcm.utils.log_time_consistency import DbLogTimeConsistencyAnalyzer from sdcm.utils.net import get_my_ip, get_sct_runner_ip from sdcm.utils.operations_thread import ThreadParams -from sdcm.utils.replication_strategy_utils import LocalReplicationStrategy, NetworkTopologyReplicationStrategy -from sdcm.utils.threads_and_processes_alive import gather_live_processes_and_dump_to_file, \ - gather_live_threads_and_dump_to_file -from sdcm.utils.version_utils import get_relocatable_pkg_url -from sdcm.ycsb_thread import YcsbStressThread -from sdcm.ndbench_thread import NdBenchStressThread -from sdcm.kcl_thread import KclStressThread, CompareTablesSizesThread -from sdcm.stress.latte_thread import LatteStressThread -from sdcm.localhost import LocalHost -from sdcm.cdclog_reader_thread import CDCLogReaderThread -from sdcm.logcollector import ( - KubernetesAPIServerLogCollector, - KubernetesLogCollector, - KubernetesMustGatherLogCollector, - LoaderLogCollector, - MonitorLogCollector, - BaseSCTLogCollector, - PythonSCTLogCollector, - ScyllaLogCollector, - SirenManagerLogCollector, -) -from sdcm.send_email import build_reporter, read_email_data_from_file, get_running_instances_for_email_report, \ - save_email_data_to_file -from sdcm.utils import alternator from sdcm.utils.profiler import ProfilerFactory -from sdcm.remote import RemoteCmdRunnerBase -from sdcm.utils.gce_utils import get_gce_compute_instances_client -from sdcm.utils.auth_context import temp_authenticator -from sdcm.keystore import KeyStore -from sdcm.utils.latency import calculate_latency, analyze_hdr_percentiles -from sdcm.utils.csrangehistogram import CSHistogramTagTypes, CSWorkloadTypes, make_cs_range_histogram_summary, \ - make_cs_range_histogram_summary_by_interval from sdcm.utils.raft.common import validate_raft_on_nodes +from sdcm.utils.replication_strategy_utils import ( + LocalReplicationStrategy, + NetworkTopologyReplicationStrategy, +) +from sdcm.utils.threads_and_processes_alive import ( + gather_live_processes_and_dump_to_file, + gather_live_threads_and_dump_to_file, +) +from sdcm.utils.version_utils import get_relocatable_pkg_url +from sdcm.ycsb_thread import YcsbStressThread from test_lib.compaction import CompactionStrategy CLUSTER_CLOUD_IMPORT_ERROR = "" @@ -138,7 +197,9 @@ configure_logging(exception_handler=handle_exception, variables={'log_dir': TestConfig().logdir()}) try: - from botocore.vendored.requests.packages.urllib3.contrib.pyopenssl import extract_from_urllib3 + from botocore.vendored.requests.packages.urllib3.contrib.pyopenssl import ( + extract_from_urllib3, + ) # Don't use pyOpenSSL in urllib3 - it causes an ``OpenSSL.SSL.Error`` # exception when we try an API call on an idled persistent connection. @@ -265,14 +326,14 @@ def critical_failure_handler(signum, frame): # pylint: disable=unused-argument class SchemaVersion(NamedTuple): schema_id: str - node_ips: List[str] + node_ips: list[str] class ClusterInformation(NamedTuple): name: str snitch: str partitioner: str - schema_versions: List[SchemaVersion] + schema_versions: list[SchemaVersion] class ClusterTester(db_stats.TestStatsMixin, unittest.TestCase): # pylint: disable=too-many-instance-attributes,too-many-public-methods @@ -280,8 +341,8 @@ class ClusterTester(db_stats.TestStatsMixin, unittest.TestCase): # pylint: disa localhost = None events_processes_registry = None monitors: BaseMonitorSet = None - loaders: Union[BaseLoaderSet, LoaderSetAWS, LoaderSetGCE] = None - db_cluster: Union[BaseCluster, BaseScyllaCluster] = None + loaders: BaseLoaderSet | LoaderSetAWS | LoaderSetGCE = None + db_cluster: BaseCluster | BaseScyllaCluster = None @property def k8s_cluster(self): @@ -1068,12 +1129,12 @@ def get_cluster_gce(self, loader_info, db_info, monitor_info): # noqa: PLR0912, elif isinstance(n_db_nodes, str): # latest type to support multiple datacenters db_info['n_nodes'] = [int(n) for n in n_db_nodes.split()] else: - self.fail('Unsupported parameter type: {}'.format(type(n_db_nodes))) + self.fail(f'Unsupported parameter type: {type(n_db_nodes)}') cpu = self.params.get('gce_instance_type_cpu_db') # unit is GB mem = self.params.get('gce_instance_type_mem_db') if cpu and mem: - db_info['type'] = 'custom-{}-{}-ext'.format(cpu, int(mem) * 1024) + db_info['type'] = f'custom-{cpu}-{int(mem) * 1024}-ext' if db_info['type'] is None: db_info['type'] = self.params.get('gce_instance_type_db') if db_info['disk_type'] is None: @@ -1170,7 +1231,7 @@ def get_cluster_azure(self, loader_info, db_info, monitor_info): # noqa: PLR091 # pylint: disable=too-many-branches,too-many-statements,too-many-locals regions = self.params.get('azure_region_name') test_id = str(TestConfig().test_id()) - provisioners: List[AzureProvisioner] = [] + provisioners: list[AzureProvisioner] = [] for region in regions: provisioners.append(provisioner_factory.create_provisioner(backend="azure", test_id=test_id, region=region, availability_zone=self.params.get('availability_zone'))) @@ -1181,7 +1242,7 @@ def get_cluster_azure(self, loader_info, db_info, monitor_info): # noqa: PLR091 elif isinstance(n_db_nodes, str): # latest type to support multiple datacenters db_info['n_nodes'] = [int(n) for n in n_db_nodes.split()] else: - self.fail('Unsupported parameter type: {}'.format(type(n_db_nodes))) + self.fail(f'Unsupported parameter type: {type(n_db_nodes)}') db_info['type'] = self.params.get('azure_instance_type_db') if loader_info['n_nodes'] is None: n_loader_nodes = self.params.get('n_loaders') @@ -1190,7 +1251,7 @@ def get_cluster_azure(self, loader_info, db_info, monitor_info): # noqa: PLR091 elif isinstance(n_loader_nodes, str): # latest type to support multiple datacenters loader_info['n_nodes'] = [int(n) for n in n_loader_nodes.split()] else: - self.fail('Unsupported parameter type: {}'.format(type(n_loader_nodes))) + self.fail(f'Unsupported parameter type: {type(n_loader_nodes)}') azure_image = self.params.get("azure_image_db").strip() user_prefix = self.params.get('user_prefix') self.credentials.append(UserRemoteCredentials(key_file="~/.ssh/scylla-test")) @@ -1245,7 +1306,7 @@ def get_cluster_aws(self, loader_info, db_info, monitor_info): # noqa: PLR0912, elif isinstance(n_loader_nodes, str): # latest type to support multiple datacenters loader_info['n_nodes'] = [int(n) for n in n_loader_nodes.split()] else: - self.fail('Unsupported parameter type: {}'.format(type(n_loader_nodes))) + self.fail(f'Unsupported parameter type: {type(n_loader_nodes)}') if loader_info['type'] is None: loader_info['type'] = self.params.get('instance_type_loader') if loader_info['disk_size'] is None: @@ -1796,7 +1857,7 @@ def _cs_add_node_flag(self, stress_cmd): ip = ','.join(self.db_cluster.get_node_public_ips()) else: ip = self.db_cluster.get_node_private_ips()[0] - stress_cmd = '%s -node %s' % (stress_cmd, ip) + stress_cmd = f'{stress_cmd} -node {ip}' return stress_cmd def run_stress(self, stress_cmd, duration=None): @@ -2187,7 +2248,7 @@ def is_keyspace_in_cluster(session, keyspace_name): def wait_validate_keyspace_existence(self, session, keyspace_name, timeout=180, step=5): # pylint: disable=invalid-name - text = 'waiting for the keyspace "{}" to be created in the cluster'.format(keyspace_name) + text = f'waiting for the keyspace "{keyspace_name}" to be created in the cluster' does_keyspace_exist = wait.wait_for(func=self.is_keyspace_in_cluster, step=step, text=text, timeout=timeout, session=session, keyspace_name=keyspace_name, throw_exc=False) return does_keyspace_exist @@ -2210,10 +2271,10 @@ def create_keyspace(self, keyspace_name, replication_factor): else: replication_strategy = NetworkTopologyReplicationStrategy(**replication_factor) - execution_result = session.execute('CREATE KEYSPACE IF NOT EXISTS %s WITH replication=%s' - % (keyspace_name, str(replication_strategy))) + execution_result = session.execute( + f'CREATE KEYSPACE IF NOT EXISTS {keyspace_name} WITH replication={str(replication_strategy)}') if execution_result: - self.log.debug("keyspace creation result: {}".format(execution_result.response_future)) + self.log.debug(f"keyspace creation result: {execution_result.response_future}") with self.db_cluster.cql_connection_patient(validation_node) as session: does_keyspace_exist = self.wait_validate_keyspace_existence(session, keyspace_name) return does_keyspace_exist @@ -2228,20 +2289,18 @@ def create_table(self, name, key_type="varchar", # pylint: disable=too-many-arg additional_columns = "" if columns is not None: for key, value in columns.items(): - additional_columns = "%s, %s %s" % (additional_columns, key, value) + additional_columns = f"{additional_columns}, {key} {value}" if additional_columns == "": - query = ('CREATE COLUMNFAMILY IF NOT EXISTS %s (key %s, c varchar, v varchar, ' - 'PRIMARY KEY(key, c)) WITH comment=\'test cf\'' % - (name, key_type)) + query = ('CREATE COLUMNFAMILY IF NOT EXISTS {} (key {}, c varchar, v varchar, ' + 'PRIMARY KEY(key, c)) WITH comment=\'test cf\''.format(name, key_type)) else: - query = ('CREATE COLUMNFAMILY IF NOT EXISTS %s (key %s PRIMARY KEY%s) ' - 'WITH comment=\'test cf\'' % - (name, key_type, additional_columns)) + query = ('CREATE COLUMNFAMILY IF NOT EXISTS {} (key {} PRIMARY KEY{}) ' + 'WITH comment=\'test cf\''.format(name, key_type, additional_columns)) if compression is not None: - query = ('%s AND compression = { \'sstable_compression\': ' - '\'%sCompressor\' }' % (query, compression)) + query = (f'{query} AND compression = {{ \'sstable_compression\': ' + f'\'{compression}Compressor\' }}') else: # if a compression option is omitted, C* # will default to lz4 compression @@ -2257,17 +2316,16 @@ def create_table(self, name, key_type="varchar", # pylint: disable=too-many-arg query += prefix + compaction_clause + postfix if read_repair is not None: - query = '%s AND read_repair_chance=%f' % (query, read_repair) + query = f'{query} AND read_repair_chance={read_repair:f}' if gc_grace is not None: query = '%s AND gc_grace_seconds=%d' % (query, gc_grace) if speculative_retry is not None: - query = ('%s AND speculative_retry=\'%s\'' % - (query, speculative_retry)) + query = (f'{query} AND speculative_retry=\'{speculative_retry}\'') if scylla_encryption_options: - query = '%s AND scylla_encryption_options=%s' % (query, scylla_encryption_options) + query = f'{query} AND scylla_encryption_options={scylla_encryption_options}' if compact_storage: query += ' AND COMPACT STORAGE' - self.log.debug('CQL query to execute: {}'.format(query)) + self.log.debug(f'CQL query to execute: {query}') with self.db_cluster.cql_connection_patient(node=self.db_cluster.nodes[0], keyspace=keyspace_name) as session: session.execute(query) time.sleep(0.2) @@ -2275,9 +2333,9 @@ def create_table(self, name, key_type="varchar", # pylint: disable=too-many-arg def truncate_cf(self, ks_name: str, table_name: str, session: Session, truncate_timeout_sec: int | None = None): try: timeout = f" USING TIMEOUT {truncate_timeout_sec}s" if truncate_timeout_sec else "" - session.execute('TRUNCATE TABLE {0}.{1}{2}'.format(ks_name, table_name, timeout)) + session.execute(f'TRUNCATE TABLE {ks_name}.{table_name}{timeout}') except Exception as ex: # pylint: disable=broad-except # noqa: BLE001 - self.log.debug('Failed to truncate base table {0}.{1}. Error: {2}'.format(ks_name, table_name, str(ex))) + self.log.debug(f'Failed to truncate base table {ks_name}.{table_name}. Error: {str(ex)}') def create_materialized_view(self, ks_name, base_table_name, mv_name, mv_partition_key, mv_clustering_key, session, # noqa: PLR0913 # pylint: disable=too-many-arguments @@ -2294,7 +2352,7 @@ def create_materialized_view(self, ks_name, base_table_name, mv_name, mv_partiti mv_clustering_key = mv_clustering_key if isinstance(mv_clustering_key, list) else list(mv_clustering_key) for kc in mv_partition_key + mv_clustering_key: # pylint: disable=invalid-name - where_clause.append('{} is not null'.format(kc)) + where_clause.append(f'{kc} is not null') pk_clause = ', '.join(pk for pk in mv_partition_key) cl_clause = ', '.join(cl for cl in mv_clustering_key) @@ -2310,31 +2368,30 @@ def create_materialized_view(self, ks_name, base_table_name, mv_name, mv_partiti pk=pk_clause, cl=cl_clause) if compression is not None: - query = ('%s AND compression = { \'sstable_compression\': ' - '\'%sCompressor\' }' % (query, compression)) + query = (f'{query} AND compression = {{ \'sstable_compression\': ' + f'\'{compression}Compressor\' }}') if read_repair is not None: - query = '%s AND read_repair_chance=%f' % (query, read_repair) + query = f'{query} AND read_repair_chance={read_repair:f}' if gc_grace is not None: query = '%s AND gc_grace_seconds=%d' % (query, gc_grace) if speculative_retry is not None: - query = ('%s AND speculative_retry=\'%s\'' % - (query, speculative_retry)) + query = (f'{query} AND speculative_retry=\'{speculative_retry}\'') if compact_storage: query += ' AND COMPACT STORAGE' - self.log.debug('MV create statement: {}'.format(query)) + self.log.debug(f'MV create statement: {query}') session.execute(query) def _wait_for_view(self, scylla_cluster, session, key_space, view): - self.log.debug("Waiting for view {}.{} to finish building...".format(key_space, view)) + self.log.debug(f"Waiting for view {key_space}.{view} to finish building...") def _view_build_finished(live_nodes_amount): result = self.rows_to_list(session.execute("SELECT status FROM system_distributed.view_build_status WHERE " - "keyspace_name='{0}' " - "AND view_name='{1}'".format(key_space, view))) - self.log.debug('View build status result: {}'.format(result)) + f"keyspace_name='{key_space}' " + f"AND view_name='{view}'")) + self.log.debug(f'View build status result: {result}') return len([status for status in result if status[0] == 'SUCCESS']) >= live_nodes_amount attempts = 20 @@ -2351,22 +2408,21 @@ def _view_build_finished(live_nodes_amount): time.sleep(3) attempts -= 1 - raise Exception("View {}.{} not built".format(key_space, view)) + raise Exception(f"View {key_space}.{view} not built") def _wait_for_view_build_start(self, session, key_space, view, seconds_to_wait=20): def _check_build_started(): result = self.rows_to_list(session.execute("SELECT last_token FROM system.views_builds_in_progress " - "WHERE keyspace_name='{0}' AND view_name='{1}'".format(key_space, - view))) - self.log.debug('View build in progress: {}'.format(result)) + f"WHERE keyspace_name='{key_space}' AND view_name='{view}'")) + self.log.debug(f'View build in progress: {result}') return result != [] self.log.debug("Ensure view building started.") start = time.time() while not _check_build_started(): if time.time() - start > seconds_to_wait: - raise Exception("View building didn't start in {} seconds".format(seconds_to_wait)) + raise Exception(f"View building didn't start in {seconds_to_wait} seconds") @staticmethod def rows_to_list(rows): @@ -2380,8 +2436,8 @@ def copy_table(self, node, src_keyspace, src_table, dest_keyspace, # pylint: di Copy data from . to . if copy_data is True """ result = True - create_statement = "SELECT * FROM system_schema.table where table_name = '%s' " \ - "and keyspace_name = '%s'" % (src_table, src_keyspace) + create_statement = "SELECT * FROM system_schema.table where table_name = '{}' " \ + "and keyspace_name = '{}'".format(src_table, src_keyspace) if not self.create_table_as(node, src_keyspace, src_table, dest_keyspace, dest_table, create_statement, columns_list): return False @@ -2405,9 +2461,9 @@ def copy_view(self, node, src_keyspace, src_view, dest_keyspace, # pylint: disa Copy data from . to . if copy_data is True """ result = True - create_statement = "SELECT * FROM system_schema.views where view_name = '%s' " \ - "and keyspace_name = '%s'" % (src_view, src_keyspace) - self.log.debug('Start create table with statement: {}'.format(create_statement)) + create_statement = "SELECT * FROM system_schema.views where view_name = '{}' " \ + "and keyspace_name = '{}'".format(src_view, src_keyspace) + self.log.debug(f'Start create table with statement: {create_statement}') if not self.create_table_as(node, src_keyspace, src_view, dest_keyspace, dest_table, create_statement, columns_list): return False @@ -2434,8 +2490,7 @@ def create_table_as(self, node, src_keyspace, src_table, # noqa: PLR0913 result = rows_to_list(session.execute(create_statement)) if result: - result = session.execute("SELECT * FROM {keyspace}.{table} LIMIT 1".format(keyspace=src_keyspace, - table=src_table)) + result = session.execute(f"SELECT * FROM {src_keyspace}.{src_table} LIMIT 1") primary_keys = [] # Create table with table/view structure @@ -2469,9 +2524,9 @@ def create_table_as(self, node, src_keyspace, src_table, # noqa: PLR0913 create_cql = 'create table {keyspace}.{name} ({columns}, PRIMARY KEY ({pk}))' \ .format(keyspace=dest_keyspace, name=dest_table, - columns=', '.join(['%s %s' % (c[0], c[1]) for c in columns]), + columns=', '.join([f'{c[0]} {c[1]}' for c in columns]), pk=', '.join(primary_keys)) - self.log.debug('Create new table with cql: {}'.format(create_cql)) + self.log.debug(f'Create new table with cql: {create_cql}') session.execute(create_cql) return True return False @@ -2500,7 +2555,7 @@ def copy_data_between_tables(self, node, src_keyspace, src_table, dest_keyspace, return False # TODO: Temporary function. Will be removed - self.log.debug('Rows in the {} MV before saving: {}'.format(src_table, len(source_table_rows))) + self.log.debug(f'Rows in the {src_table} MV before saving: {len(source_table_rows)}') insert_statement = session.prepare( 'insert into {keyspace}.{name} ({columns}) ' @@ -2548,12 +2603,12 @@ def copy_data_between_tables(self, node, src_keyspace, src_table, dest_keyspace, return True def get_tables_id_of_keyspace(self, session, keyspace_name): - query = "SELECT id FROM system_schema.tables WHERE keyspace_name='{}' ".format(keyspace_name) + query = f"SELECT id FROM system_schema.tables WHERE keyspace_name='{keyspace_name}' " table_id = self.rows_to_list(session.execute(query)) return table_id[0] def get_tables_name_of_keyspace(self, session, keyspace_name): - query = "SELECT table_name FROM system_schema.tables WHERE keyspace_name='{}' ".format(keyspace_name) + query = f"SELECT table_name FROM system_schema.tables WHERE keyspace_name='{keyspace_name}' " table_id = self.rows_to_list(session.execute(query)) return table_id[0] @@ -2563,11 +2618,11 @@ def get_truncated_time_from_system_local(self, session): # pylint: disable=inva return truncated_time def get_truncated_time_from_system_truncated(self, session, table_id): # pylint: disable=invalid-name - query = "SELECT truncated_at FROM system.truncated WHERE table_uuid={}".format(table_id) + query = f"SELECT truncated_at FROM system.truncated WHERE table_uuid={table_id}" truncated_time = self.rows_to_list(session.execute(query)) return truncated_time[0] - def get_describecluster_info(self) -> Optional[ClusterInformation]: # pylint: disable=too-many-locals + def get_describecluster_info(self) -> ClusterInformation | None: # pylint: disable=too-many-locals """ Runs the 'nodetool describecluster' command on a node. @@ -2767,7 +2822,7 @@ def tearDown(self): self.argus_finalize_test_run() self.argus_heartbeat_stop_signal.set() - self.log.info('Test ID: {}'.format(self.test_config.test_id())) + self.log.info(f'Test ID: {self.test_config.test_id()}') self._check_alive_routines_and_report_them() self._check_if_db_log_time_consistency_looks_good() self.remove_python_exit_hooks() @@ -2906,8 +2961,7 @@ def alter_table_encryption(self, table, scylla_encryption_options=None, upgrades 'scylla_encryption_options is not set, skipping to enable encryption at-rest for all test tables') else: with self.db_cluster.cql_connection_patient(self.db_cluster.nodes[0]) as session: - query = "ALTER TABLE {table} WITH scylla_encryption_options = {scylla_encryption_options};".format( - table=table, scylla_encryption_options=scylla_encryption_options) + query = f"ALTER TABLE {table} WITH scylla_encryption_options = {scylla_encryption_options};" self.log.debug('enable encryption at-rest for table {table}, query:\n\t{query}'.format(**locals())) session.execute(query) if upgradesstables: @@ -2925,14 +2979,14 @@ def alter_test_tables_encryption(self, scylla_encryption_options=None, upgradess table, scylla_encryption_options=scylla_encryption_options, upgradesstables=upgradesstables) def get_num_of_hint_files(self, node): - result = node.remoter.run("sudo find {0.scylla_hints_dir} -name *.log -type f| wc -l".format(self), + result = node.remoter.run(f"sudo find {self.scylla_hints_dir} -name *.log -type f| wc -l", verbose=True) total_hint_files = int(result.stdout.strip()) self.log.debug("Number of hint files on '%s': %s.", node.name, total_hint_files) return total_hint_files def get_num_shards(self, node): - result = node.remoter.run("sudo ls -1 {0.scylla_hints_dir}| wc -l".format(self), verbose=True) + result = node.remoter.run(f"sudo ls -1 {self.scylla_hints_dir}| wc -l", verbose=True) return int(result.stdout.strip()) @retrying(n=3, sleep_time=15, allowed_exceptions=(AssertionError,)) @@ -2944,7 +2998,7 @@ def hints_sending_in_progress(self): self.log.debug("scylla_hints_manager_sent: %s", results) assert results, "No results from Prometheus" # if all are zeros the result will be False, otherwise we are still sending - return any((float(v[1]) for v in results[0]["values"])) + return any(float(v[1]) for v in results[0]["values"]) @retrying(n=30, sleep_time=60, allowed_exceptions=(AssertionError, UnexpectedExit, Failure)) def wait_for_hints_to_be_sent(self, node, num_dest_nodes): @@ -2963,7 +3017,7 @@ def verify_no_drops_and_errors(self, starting_from): results = self.prometheus_db.query(query=query, start=starting_from, end=time.time()) err_msg = "There were hint manager %s detected during the test!" % ( "drops" if "dropped" in query else "errors") - assert any((float(v[1]) for v in results[0]["values"])) is False, err_msg + assert any(float(v[1]) for v in results[0]["values"]) is False, err_msg def get_data_set_size(self, cs_cmd): # pylint: disable=inconsistent-return-statements """:returns value of n in stress comand, that is approximation and currently doesn't take in consideration @@ -3114,7 +3168,7 @@ def is_compaction_running(self): self.log.debug("scylla_compaction_manager_compactions: {results}".format(**locals())) assert results or results == [], "No results from Prometheus" # if any result values is not zero - there are running compactions. - return any((float(v[1]) for v in results[0]["values"])) + return any(float(v[1]) for v in results[0]["values"]) def wait_compactions_are_running(self, n=20, sleep_time=60): # pylint: disable=invalid-name # Wait until there are running compactions @@ -3151,7 +3205,7 @@ def is_metric_has_data(): self.log.debug("metric_has_data: %s", results) assert results, "No results from Prometheus" if results: - assert any((float(v[1]) for v in results[0]["values"])) > 0, f"{metric_query} didn't has data in it" + assert any(float(v[1]) for v in results[0]["values"]) > 0, f"{metric_query} didn't has data in it" is_metric_has_data() @@ -3253,10 +3307,10 @@ def get_test_failures(self): ).publish_or_dump(default_logger=self.log) def stop_all_nodes_except_for(self, node): - self.log.debug("Stopping all nodes except for: {}".format(node.name)) + self.log.debug(f"Stopping all nodes except for: {node.name}") for c_node in [n for n in self.db_cluster.nodes if n != node]: - self.log.debug("Stopping node: {}".format(c_node.name)) + self.log.debug(f"Stopping node: {c_node.name}") c_node.stop_scylla_server() def start_all_nodes(self): @@ -3264,7 +3318,7 @@ def start_all_nodes(self): self.log.debug("Starting all nodes") # restarting all nodes twice in order to pervent no-seed node issues for c_node in self.db_cluster.nodes * 2: - self.log.debug("Starting node: {} ({})".format(c_node.name, c_node.public_ip_address)) + self.log.debug(f"Starting node: {c_node.name} ({c_node.public_ip_address})") c_node.start_scylla_server(verify_up=False) time.sleep(10) self.log.debug("Wait DB is up after all nodes were started") @@ -3272,16 +3326,16 @@ def start_all_nodes(self): c_node.wait_db_up() def start_all_nodes_except_for(self, node): - self.log.debug("Starting all nodes except for: {}".format(node.name)) + self.log.debug(f"Starting all nodes except for: {node.name}") node_list = [n for n in self.db_cluster.nodes if n != node] # Start down seed nodes first, if exists for c_node in [n for n in node_list if n.is_seed]: - self.log.debug("Starting seed node: {}".format(c_node.name)) + self.log.debug(f"Starting seed node: {c_node.name}") c_node.start_scylla_server() for c_node in [n for n in node_list if not n.is_seed]: - self.log.debug("Starting non-seed node: {}".format(c_node.name)) + self.log.debug(f"Starting non-seed node: {c_node.name}") c_node.start_scylla_server() node.wait_db_up() @@ -3317,12 +3371,12 @@ def get_used_capacity(self, node) -> float: # pylint: disable=too-many-locals assert fs_size_res[0], "Could not resolve capacity query result." kb_size = 2 ** 10 mb_size = kb_size * 1024 - self.log.debug("fs_size_res: {}".format(fs_size_res)) - self.log.debug("used_capacity_query: {}".format(used_capacity_query)) + self.log.debug(f"fs_size_res: {fs_size_res}") + self.log.debug(f"used_capacity_query: {used_capacity_query}") used_cap_res = self.prometheus_db.query( query=used_capacity_query, start=int(time.time()) - 5, end=int(time.time())) - self.log.debug("used_cap_res: {}".format(used_cap_res)) + self.log.debug(f"used_cap_res: {used_cap_res}") assert used_cap_res, "No results from Prometheus" used_size_mb = float(used_cap_res[0]["values"][0][1]) / float(mb_size) @@ -3336,7 +3390,7 @@ def print_nodes_used_capacity(self): for node in self.db_cluster.nodes: used_capacity = self.get_used_capacity(node=node) self.log.debug( - "Node {} ({}) used capacity is: {}".format(node.name, node.private_ip_address, used_capacity)) + f"Node {node.name} ({node.private_ip_address}) used capacity is: {used_capacity}") def get_nemesises_stats(self): nemesis_stats = {} @@ -3418,7 +3472,7 @@ def send_email(self): self.log.warning('Test is not configured to send html reports') except Exception as details: # pylint: disable=broad-except # noqa: BLE001 - self.log.error("Error during sending email: {}".format(details)) + self.log.error(f"Error during sending email: {details}") else: self.log.warning("Email is not configured: %s or no email data: %s", send_email, email_data) @@ -3447,7 +3501,7 @@ def all_nodes_scylla_shards(self): return all_nodes_shards - def _get_common_email_data(self) -> Dict[str, Any]: + def _get_common_email_data(self) -> dict[str, Any]: """Helper for subclasses which extracts common data for email.""" if self.db_cluster: @@ -3526,7 +3580,7 @@ def get_test_results(self, source, severity=None): output.append(result['message']) return output - def _get_live_node(self) -> Optional[BaseNode]: # pylint: disable=inconsistent-return-statements + def _get_live_node(self) -> BaseNode | None: # pylint: disable=inconsistent-return-statements if not self.db_cluster or not self.db_cluster.nodes: self.log.error("Cluster object was not initialized") return None @@ -3540,7 +3594,7 @@ def _get_live_node(self) -> Optional[BaseNode]: # pylint: disable=inconsistent- return None @staticmethod - def _check_ssh_node_connectivity(node: BaseNode) -> Optional[BaseNode]: + def _check_ssh_node_connectivity(node: BaseNode) -> BaseNode | None: node.wait_ssh_up(verbose=False, timeout=50) return node diff --git a/sdcm/tombstone_gc_verification_thread.py b/sdcm/tombstone_gc_verification_thread.py index 2138e5bfa78..718ad512b0a 100644 --- a/sdcm/tombstone_gc_verification_thread.py +++ b/sdcm/tombstone_gc_verification_thread.py @@ -5,7 +5,7 @@ import time from sdcm import wait -from sdcm.cluster import BaseScyllaCluster, BaseCluster +from sdcm.cluster import BaseCluster, BaseScyllaCluster from sdcm.remote import LocalCmdRunner from sdcm.sct_events import Severity from sdcm.sct_events.database import TombstoneGcVerificationEvent diff --git a/sdcm/utils/adaptive_timeouts/__init__.py b/sdcm/utils/adaptive_timeouts/__init__.py index 1c0b26ffd7b..b775e9b0322 100644 --- a/sdcm/utils/adaptive_timeouts/__init__.py +++ b/sdcm/utils/adaptive_timeouts/__init__.py @@ -5,8 +5,12 @@ from typing import Any from sdcm.sct_events.system import SoftTimeoutEvent -from sdcm.utils.adaptive_timeouts.load_info_store import NodeLoadInfoService, AdaptiveTimeoutStore, ESAdaptiveTimeoutStore, \ - NodeLoadInfoServices +from sdcm.utils.adaptive_timeouts.load_info_store import ( + AdaptiveTimeoutStore, + ESAdaptiveTimeoutStore, + NodeLoadInfoService, + NodeLoadInfoServices, +) LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/adaptive_timeouts/load_info_store.py b/sdcm/utils/adaptive_timeouts/load_info_store.py index 1b103c98d35..da18dcb37f6 100644 --- a/sdcm/utils/adaptive_timeouts/load_info_store.py +++ b/sdcm/utils/adaptive_timeouts/load_info_store.py @@ -11,16 +11,16 @@ # # Copyright (c) 2023 ScyllaDB import logging +import re import time import uuid from collections import defaultdict from datetime import datetime from functools import cached_property -import re from typing import Any import yaml -from cachetools import cached, TTLCache +from cachetools import TTLCache, cached from sdcm.es import ES from sdcm.remote import RemoteCmdRunner diff --git a/sdcm/utils/alternator/__init__.py b/sdcm/utils/alternator/__init__.py index fd1e042ecb2..b58611956a6 100644 --- a/sdcm/utils/alternator/__init__.py +++ b/sdcm/utils/alternator/__init__.py @@ -1,4 +1 @@ -from sdcm.utils.alternator import api -from sdcm.utils.alternator import consts -from sdcm.utils.alternator import enums -from sdcm.utils.alternator import schemas +from sdcm.utils.alternator import api, consts, enums, schemas diff --git a/sdcm/utils/alternator/api.py b/sdcm/utils/alternator/api.py index a7d5c56db7c..b526b734f66 100644 --- a/sdcm/utils/alternator/api.py +++ b/sdcm/utils/alternator/api.py @@ -8,8 +8,8 @@ from mypy_boto3_dynamodb import DynamoDBClient, DynamoDBServiceResource from mypy_boto3_dynamodb.service_resource import Table -from sdcm.utils.alternator import schemas, enums, consts -from sdcm.utils.alternator.consts import TABLE_NAME, NO_LWT_TABLE_NAME +from sdcm.utils.alternator import consts, enums, schemas +from sdcm.utils.alternator.consts import NO_LWT_TABLE_NAME, TABLE_NAME from sdcm.utils.common import normalize_ipv6_url LOGGER = logging.getLogger(__name__) @@ -63,19 +63,19 @@ def create_table(self, node, # pylint: disable=too-many-arguments # noqa: PLR0 schema = schema.value schema = schemas.ALTERNATOR_SCHEMAS[schema] dynamodb_api = self.get_dynamodb_api(node=node) - LOGGER.debug("Creating a new table '{}' using node '{}'".format(table_name, node.name)) + LOGGER.debug(f"Creating a new table '{table_name}' using node '{node.name}'") table = dynamodb_api.resource.create_table( TableName=table_name, BillingMode="PAY_PER_REQUEST", **schema, **kwargs) if wait_until_table_exists: waiter = dynamodb_api.client.get_waiter('table_exists') waiter.wait(TableName=table_name, WaiterConfig=dict(Delay=1, MaxAttempts=100)) - LOGGER.info("The table '{}' successfully created..".format(table_name)) + LOGGER.info(f"The table '{table_name}' successfully created..") response = dynamodb_api.client.describe_table(TableName=table_name) if isolation: self.set_write_isolation(node=node, isolation=isolation, table_name=table_name) - LOGGER.debug("Table's schema and configuration are: {}".format(response)) + LOGGER.debug(f"Table's schema and configuration are: {response}") return table def update_table_ttl(self, node, table_name, enabled: bool = True): @@ -96,15 +96,15 @@ def _scan_table(part_scan_idx=None): parallel_params, result, still_running_while = {}, [], True if is_parallel_scan: parallel_params = {"TotalSegments": threads_num, "Segment": part_scan_idx} - LOGGER.debug("Starting parallel scan part '{}' on table '{}'".format(part_scan_idx + 1, table_name)) + LOGGER.debug(f"Starting parallel scan part '{part_scan_idx + 1}' on table '{table_name}'") else: - LOGGER.debug("Starting full scan on table '{}'".format(table_name)) + LOGGER.debug(f"Starting full scan on table '{table_name}'") while still_running_while: response = table.scan(**parallel_params, **kwargs) result.extend(response["Items"]) still_running_while = 'LastEvaluatedKey' in response - LOGGER.debug("Founding the following items:\n{}".format(pformat(result))) + LOGGER.debug(f"Founding the following items:\n{pformat(result)}") return result if is_parallel_scan: @@ -160,6 +160,6 @@ def delete_table(self, node, table_name: consts.TABLE_NAME, wait_until_table_rem if wait_until_table_removed: waiter = dynamodb_api.client.get_waiter('table_not_exists') waiter.wait(TableName=table_name) - LOGGER.info("The '{}' table successfully removed".format(table_name)) + LOGGER.info(f"The '{table_name}' table successfully removed") else: - LOGGER.info("Send request to removed '{}' table".format(table_name)) + LOGGER.info(f"Send request to removed '{table_name}' table") diff --git a/sdcm/utils/alternator/enums.py b/sdcm/utils/alternator/enums.py index 82edd964f40..1ccd48947dd 100644 --- a/sdcm/utils/alternator/enums.py +++ b/sdcm/utils/alternator/enums.py @@ -1,4 +1,4 @@ -from enum import auto, Enum +from enum import Enum, auto class WriteIsolation(Enum): diff --git a/sdcm/utils/alternator/schemas.py b/sdcm/utils/alternator/schemas.py index e52a703c3c9..d2128025ca4 100644 --- a/sdcm/utils/alternator/schemas.py +++ b/sdcm/utils/alternator/schemas.py @@ -1,5 +1,4 @@ -from sdcm.utils.alternator import consts -from sdcm.utils.alternator import enums +from sdcm.utils.alternator import consts, enums HASH_SCHEMA = dict( KeySchema=[ diff --git a/sdcm/utils/argus.py b/sdcm/utils/argus.py index 6e435ab80c8..d76ba9e946e 100644 --- a/sdcm/utils/argus.py +++ b/sdcm/utils/argus.py @@ -1,8 +1,9 @@ import logging from uuid import UUID -from argus.client.sct.client import ArgusSCTClient from argus.client.base import ArgusClientError +from argus.client.sct.client import ArgusSCTClient + from sdcm.keystore import KeyStore LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/auth_context.py b/sdcm/utils/auth_context.py index 5546a9da8f7..4b886165a74 100644 --- a/sdcm/utils/auth_context.py +++ b/sdcm/utils/auth_context.py @@ -1,6 +1,6 @@ import logging - from contextlib import contextmanager + from sdcm.cluster import BaseNode VALID_AUTHENTICATORS = [ diff --git a/sdcm/utils/auto_ssh.py b/sdcm/utils/auto_ssh.py index 1d8067457f3..7821b4fdd73 100644 --- a/sdcm/utils/auto_ssh.py +++ b/sdcm/utils/auto_ssh.py @@ -14,7 +14,6 @@ import os from functools import cached_property - AUTO_SSH_IMAGE = "jnovack/autossh:1.2.2" AUTO_SSH_LOGFILE = "autossh.log" diff --git a/sdcm/utils/aws_builder.py b/sdcm/utils/aws_builder.py index c5de246410b..5ec86a5c738 100644 --- a/sdcm/utils/aws_builder.py +++ b/sdcm/utils/aws_builder.py @@ -14,16 +14,16 @@ import logging from functools import cached_property -import jenkins -import click import boto3 import botocore +import click +import jenkins import requests from mypy_boto3_ec2 import EC2ServiceResource -from sdcm.utils.aws_region import AwsRegion -from sdcm.sct_runner import AwsSctRunner from sdcm.keystore import KeyStore +from sdcm.sct_runner import AwsSctRunner +from sdcm.utils.aws_region import AwsRegion LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/aws_kms.py b/sdcm/utils/aws_kms.py index f8b8e696111..bd26b74af4a 100644 --- a/sdcm/utils/aws_kms.py +++ b/sdcm/utils/aws_kms.py @@ -11,11 +11,11 @@ # # Copyright (c) 2023 ScyllaDB -from itertools import cycle import logging +from itertools import cycle -import botocore import boto3 +import botocore LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/aws_peering.py b/sdcm/utils/aws_peering.py index 66cd0b06ae5..e5569904b51 100644 --- a/sdcm/utils/aws_peering.py +++ b/sdcm/utils/aws_peering.py @@ -11,8 +11,8 @@ # # Copyright (c) 2022 ScyllaDB -import logging import itertools +import logging from botocore.exceptions import ClientError diff --git a/sdcm/utils/aws_region.py b/sdcm/utils/aws_region.py index d76fc43d330..65c5bb7853a 100644 --- a/sdcm/utils/aws_region.py +++ b/sdcm/utils/aws_region.py @@ -12,8 +12,8 @@ # Copyright (c) 2021 ScyllaDB import logging +from functools import cache, cached_property from ipaddress import ip_network -from functools import cached_property, cache import boto3 import botocore @@ -43,7 +43,9 @@ def __init__(self, region_name): self.resource: EC2ServiceResource = boto3.resource("ec2", region_name=region_name) # cause import straight from common create cyclic dependency - from sdcm.utils.common import all_aws_regions # pylint: disable=import-outside-toplevel + from sdcm.utils.common import ( + all_aws_regions, # pylint: disable=import-outside-toplevel + ) region_index = all_aws_regions(cached=True).index(self.region_name) cidr = ip_network(self.SCT_VPC_CIDR_TMPL.format(region_index)) diff --git a/sdcm/utils/aws_utils.py b/sdcm/utils/aws_utils.py index 94e2b11768f..fd4db062574 100644 --- a/sdcm/utils/aws_utils.py +++ b/sdcm/utils/aws_utils.py @@ -11,21 +11,20 @@ # # Copyright (c) 2021 ScyllaDB import functools -import time import logging +import time from functools import cached_property -from typing import List, Dict import boto3 from botocore.exceptions import ClientError -from mypy_boto3_ec2 import EC2ServiceResource, EC2Client +from mypy_boto3_ec2 import EC2Client, EC2ServiceResource from mypy_boto3_ec2.literals import ArchitectureTypeType -from sdcm.utils.decorators import retrying +from sdcm.keystore import KeyStore +from sdcm.test_config import TestConfig from sdcm.utils.aws_region import AwsRegion +from sdcm.utils.decorators import retrying from sdcm.wait import wait_for -from sdcm.test_config import TestConfig -from sdcm.keystore import KeyStore LOGGER = logging.getLogger(__name__) AwsArchType = ArchitectureTypeType | str @@ -56,23 +55,23 @@ def cluster_owned_objects_filter(self): return [{"Name": f"tag:{self.owned_object_tag_name}", 'Values': ['owned']}] @property - def attached_security_group_ids(self) -> List[str]: + def attached_security_group_ids(self) -> list[str]: return [group_desc['GroupId'] for group_desc in self.ec2_client.describe_security_groups(Filters=self.cluster_owned_objects_filter)['SecurityGroups']] @property - def attached_nodegroup_names(self) -> List[str]: + def attached_nodegroup_names(self) -> list[str]: return self._get_attached_nodegroup_names() @property - def failed_to_delete_nodegroup_names(self) -> List[str]: + def failed_to_delete_nodegroup_names(self) -> list[str]: return self._get_attached_nodegroup_names(status='DELETE_FAILED') @property - def deleting_nodegroup_names(self) -> List[str]: + def deleting_nodegroup_names(self) -> list[str]: return self._get_attached_nodegroup_names(status='DELETING') - def _get_attached_nodegroup_names(self, status: str = None) -> List[str]: + def _get_attached_nodegroup_names(self, status: str = None) -> list[str]: if status is None: return self.eks_client.list_nodegroups(clusterName=self.short_cluster_name)['nodegroups'] output = [] @@ -210,7 +209,7 @@ def destroy_oidc_provider(self): self.short_cluster_name, exc) -def init_monitoring_info_from_params(monitor_info: dict, params: dict, regions: List): +def init_monitoring_info_from_params(monitor_info: dict, params: dict, regions: list): if monitor_info['n_nodes'] is None: monitor_info['n_nodes'] = params.get('n_monitor_nodes') if monitor_info['type'] is None: @@ -232,7 +231,7 @@ def init_monitoring_info_from_params(monitor_info: dict, params: dict, regions: return monitor_info -def init_db_info_from_params(db_info: dict, params: dict, regions: List, root_device: str = None): +def init_db_info_from_params(db_info: dict, params: dict, regions: list, root_device: str = None): if db_info['n_nodes'] is None: n_db_nodes = params.get('n_db_nodes') if isinstance(n_db_nodes, int): # legacy type @@ -281,7 +280,7 @@ def init_db_info_from_params(db_info: dict, params: dict, regions: List, root_de return db_info -def get_common_params(params: dict, regions: List, credentials: List, services: List, availability_zone: str = None) -> dict: +def get_common_params(params: dict, regions: list, credentials: list, services: list, availability_zone: str = None) -> dict: availability_zones = [availability_zone] if availability_zone else params.get('availability_zone').split(',') ec2_security_group_ids, ec2_subnet_ids = get_ec2_network_configuration( regions=regions, @@ -335,7 +334,7 @@ def get_ec2_services(regions): return services -def tags_as_ec2_tags(tags: Dict[str, str]) -> List[Dict[str, str]]: +def tags_as_ec2_tags(tags: dict[str, str]) -> list[dict[str, str]]: return [{"Key": key, "Value": value} for key, value in tags.items()] diff --git a/sdcm/utils/azure_region.py b/sdcm/utils/azure_region.py index 55fb03d81a5..f7992bf22f9 100644 --- a/sdcm/utils/azure_region.py +++ b/sdcm/utils/azure_region.py @@ -13,13 +13,13 @@ from __future__ import annotations -import os +import binascii import enum import logging -import binascii -from typing import TYPE_CHECKING -from functools import cached_property +import os from contextlib import suppress +from functools import cached_property +from typing import TYPE_CHECKING from azure.core.exceptions import ResourceNotFoundError from azure.mgmt.compute.models import TargetRegion @@ -28,7 +28,6 @@ if TYPE_CHECKING: # pylint: disable=ungrouped-imports - from typing import Optional from azure.mgmt.compute.models import Gallery, GalleryImageVersion, VirtualMachine from azure.mgmt.network.models import ( @@ -129,13 +128,13 @@ def sct_virtual_network_name(self) -> str: def sct_subnet_name(self) -> str: # pylint: disable=no-self-use; pylint doesn't now about cached_property return "default" - def common_parameters(self, location: Optional[str] = None, tags: Optional[dict] = None) -> dict: + def common_parameters(self, location: str | None = None, tags: dict | None = None) -> dict: return { "location": location or self.location, "tags": tags or {}, } - def create_sct_resource_group(self, tags: Optional[dict] = None) -> None: + def create_sct_resource_group(self, tags: dict | None = None) -> None: LOGGER.info("Going to create SCT resource group...") if self.azure_service.resource.resource_groups.check_existence( resource_group_name=self.sct_resource_group_name, @@ -155,7 +154,7 @@ def sct_gallery(self) -> Gallery: gallery_name=self.sct_gallery_name, ) - def create_sct_gallery(self, tags: Optional[dict] = None) -> None: + def create_sct_gallery(self, tags: dict | None = None) -> None: LOGGER.info("Going to create SCT shared image gallery...") with suppress(ResourceNotFoundError): LOGGER.info("Gallery `%s' already exists", self.sct_gallery.name) @@ -175,7 +174,7 @@ def sct_network_security_group(self) -> NetworkSecurityGroup: network_security_group_name=self.sct_network_security_group_name, ) - def create_sct_network_security_group(self, tags: Optional[dict] = None) -> None: + def create_sct_network_security_group(self, tags: dict | None = None) -> None: LOGGER.info("Going to create SCT network security group...") with suppress(ResourceNotFoundError): LOGGER.info("Network security group `%s' already exists in resource group `%s'", @@ -206,7 +205,7 @@ def sct_virtual_network(self) -> VirtualNetwork: virtual_network_name=self.sct_virtual_network_name, ) - def create_sct_virtual_network(self, tags: Optional[dict] = None) -> None: + def create_sct_virtual_network(self, tags: dict | None = None) -> None: LOGGER.info("Going to create SCT virtual network...") with suppress(ResourceNotFoundError): LOGGER.info("Virtual network `%s' already exists in resource group `%s'", @@ -250,7 +249,7 @@ def create_sct_subnet(self) -> None: def create_public_ip_address(self, public_ip_address_name: str, - tags: Optional[dict] = None) -> PublicIPAddress: + tags: dict | None = None) -> PublicIPAddress: return self.azure_service.network.public_ip_addresses.begin_create_or_update( resource_group_name=self.sct_resource_group_name, public_ip_address_name=public_ip_address_name, @@ -265,7 +264,7 @@ def create_public_ip_address(self, def create_network_interface(self, network_interface_name: str, - tags: Optional[dict] = None, + tags: dict | None = None, create_public_ip_address: bool = True) -> NetworkInterface: parameters = self.common_parameters(tags=tags) | { "ip_configurations": [{ @@ -294,11 +293,11 @@ def create_virtual_machine(self, # pylint: disable=too-many-arguments # noqa: vm_size: str, image: dict[str, str], os_state: AzureOsState = AzureOsState.GENERALIZED, - computer_name: Optional[str] = None, - admin_username: Optional[str] = None, - admin_public_key: Optional[str] = None, - disk_size: Optional[int] = None, - tags: Optional[dict] = None, + computer_name: str | None = None, + admin_username: str | None = None, + admin_public_key: str | None = None, + disk_size: int | None = None, + tags: dict | None = None, create_public_ip_address: bool = True, spot: bool = False) -> VirtualMachine: if os_state is AzureOsState.GENERALIZED: @@ -366,7 +365,7 @@ def deallocate_virtual_machine(self, vm_name: str) -> None: def create_gallery_image(self, gallery_image_name: str, os_state: AzureOsState, - tags: Optional[dict] = None) -> None: + tags: dict | None = None) -> None: with suppress(ResourceNotFoundError): gallery_image = self.azure_service.compute.gallery_images.get( resource_group_name=self.sct_gallery_resource_group_name, @@ -406,7 +405,7 @@ def create_gallery_image_version(self, gallery_image_name: str, gallery_image_version_name: str, source_id: str, - tags: Optional[dict] = None) -> None: + tags: dict | None = None) -> None: self.azure_service.compute.gallery_image_versions.begin_create_or_update( resource_group_name=self.sct_gallery_resource_group_name, gallery_name=self.sct_gallery_name, @@ -438,7 +437,7 @@ def append_target_region_to_image_version(self, gallery_image_version=gallery_image_version, ).wait() - def configure(self, tags: Optional[dict] = None) -> None: + def configure(self, tags: dict | None = None) -> None: LOGGER.info("Configuring `%s' region...", self.region_name) self.create_sct_resource_group(tags=tags) self.create_sct_network_security_group(tags=tags) diff --git a/sdcm/utils/azure_utils.py b/sdcm/utils/azure_utils.py index 997f8bc0378..f3acac42242 100644 --- a/sdcm/utils/azure_utils.py +++ b/sdcm/utils/azure_utils.py @@ -14,20 +14,20 @@ from __future__ import annotations import logging -from typing import NamedTuple, TYPE_CHECKING from functools import cached_property from itertools import chain +from typing import TYPE_CHECKING, NamedTuple +from azure.core.credentials import AzureNamedKeyCredential from azure.identity import ClientSecretCredential from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.compute.models import VirtualMachine from azure.mgmt.network import NetworkManagementClient from azure.mgmt.resource import ResourceManagementClient -from azure.storage.blob import BlobServiceClient -from azure.core.credentials import AzureNamedKeyCredential -from azure.mgmt.subscription import SubscriptionClient from azure.mgmt.resourcegraph import ResourceGraphClient -from azure.mgmt.resourcegraph.models import QueryRequestOptions, QueryRequest +from azure.mgmt.resourcegraph.models import QueryRequest, QueryRequestOptions +from azure.mgmt.subscription import SubscriptionClient +from azure.storage.blob import BlobServiceClient from sdcm.keystore import KeyStore from sdcm.utils.decorators import retrying @@ -35,7 +35,7 @@ if TYPE_CHECKING: # pylint: disable=ungrouped-imports - from typing import Optional, Callable, Iterator + from collections.abc import Callable, Iterator from azure.core.credentials import TokenCredential from azure.mgmt.resource.resources.models import Resource @@ -63,7 +63,7 @@ class VirtualMachineIPs(NamedTuple): private_ip: str - public_ip: Optional[str] + public_ip: str | None class AzureService(metaclass=Singleton): @@ -207,7 +207,7 @@ def paged_query() -> Iterator[list]: return chain.from_iterable(paged_query()) -def list_instances_azure(tags_dict: Optional[dict[str, str]] = None, +def list_instances_azure(tags_dict: dict[str, str] | None = None, running: bool = False, verbose: bool = False) -> list[VirtualMachine]: query_bits = [ diff --git a/sdcm/utils/benchmarks.py b/sdcm/utils/benchmarks.py index d4023714355..6b3e74c7b3a 100644 --- a/sdcm/utils/benchmarks.py +++ b/sdcm/utils/benchmarks.py @@ -13,16 +13,16 @@ import json import logging import re -from dataclasses import dataclass, field, asdict +from dataclasses import asdict, dataclass, field from typing import NamedTuple from sdcm.es import ES from sdcm.remote import RemoteCmdRunnerBase, shell_script_cmd from sdcm.test_config import TestConfig from sdcm.utils.common import ParallelObject +from sdcm.utils.decorators import retrying from sdcm.utils.git import clone_repo from sdcm.utils.metaclasses import Singleton -from sdcm.utils.decorators import retrying LOGGER = logging.getLogger(__name__) ES_INDEX = "node_benchmarks" diff --git a/sdcm/utils/cdc/options.py b/sdcm/utils/cdc/options.py index a57907f4c35..20112eb4501 100644 --- a/sdcm/utils/cdc/options.py +++ b/sdcm/utils/cdc/options.py @@ -1,8 +1,6 @@ +import logging import re import time -import logging - -from typing import Dict, Union, List from random import choice, randint LOGGER = logging.getLogger(__name__) @@ -29,7 +27,7 @@ CDC_TTL_REGEXP] -def parse_cdc_blob_settings(blob: bytes) -> Dict[str, Union[bool, str]]: +def parse_cdc_blob_settings(blob: bytes) -> dict[str, bool | str]: """parse blob object with cdc setttings cdc settings stored in MetaTableData as blob. @@ -72,7 +70,7 @@ def parse_cdc_blob_settings(blob: bytes) -> Dict[str, Union[bool, str]]: return cdc_settings -def get_table_cdc_properties(session, ks_name: str, table_name: str) -> Dict[str, Union[bool, str]]: +def get_table_cdc_properties(session, ks_name: str, table_name: str) -> dict[str, bool | str]: """Return cdc settings for table Get cdc settings from table meta data and return dict @@ -102,7 +100,7 @@ def get_table_cdc_properties(session, ks_name: str, table_name: str) -> Dict[str return cdc_settings -def toggle_cdc_property(name: str, value: Union[str, bool]) -> Union[bool, str]: +def toggle_cdc_property(name: str, value: str | bool) -> bool | str: if name in ["ttl"]: return str(randint(300, 3600)) else: @@ -110,5 +108,5 @@ def toggle_cdc_property(name: str, value: Union[str, bool]) -> Union[bool, str]: return choice(list(variants)) -def get_cdc_settings_names() -> List[str]: +def get_cdc_settings_names() -> list[str]: return list(CDC_SETTINGS_NAMES_VALUES.keys()) diff --git a/sdcm/utils/cloud_monitor/__init__.py b/sdcm/utils/cloud_monitor/__init__.py index 5461b41116e..c9aa2be33a8 100644 --- a/sdcm/utils/cloud_monitor/__init__.py +++ b/sdcm/utils/cloud_monitor/__init__.py @@ -1 +1 @@ -from sdcm.utils.cloud_monitor.cloud_monitor import cloud_report, cloud_qa_report +from sdcm.utils.cloud_monitor.cloud_monitor import cloud_qa_report, cloud_report diff --git a/sdcm/utils/cloud_monitor/cloud_monitor.py b/sdcm/utils/cloud_monitor/cloud_monitor.py index 077b19e41b0..e33139e878d 100644 --- a/sdcm/utils/cloud_monitor/cloud_monitor.py +++ b/sdcm/utils/cloud_monitor/cloud_monitor.py @@ -1,11 +1,17 @@ -import sys import random +import sys from datetime import datetime from logging import getLogger + from sdcm.send_email import Email +from sdcm.utils.cloud_monitor.report import ( + BaseReport, + DetailedReport, + GeneralReport, + NonQaInstancesTimeDistributionReport, + QAonlyInstancesTimeDistributionReport, +) from sdcm.utils.cloud_monitor.resources.instances import CloudInstances -from sdcm.utils.cloud_monitor.report import BaseReport, GeneralReport, DetailedReport, \ - QAonlyInstancesTimeDistributionReport, NonQaInstancesTimeDistributionReport from sdcm.utils.cloud_monitor.resources.static_ips import StaticIPs LOGGER = getLogger(__name__) diff --git a/sdcm/utils/cloud_monitor/report.py b/sdcm/utils/cloud_monitor/report.py index 004a6b3c07a..8d553cbd1b0 100644 --- a/sdcm/utils/cloud_monitor/report.py +++ b/sdcm/utils/cloud_monitor/report.py @@ -14,10 +14,9 @@ import os import tempfile from abc import abstractmethod - -from datetime import datetime, timedelta from collections import defaultdict from copy import deepcopy +from datetime import datetime, timedelta import jinja2 import pytz diff --git a/sdcm/utils/cloud_monitor/resources/instances.py b/sdcm/utils/cloud_monitor/resources/instances.py index dfddf693f05..b6cc6f3fa94 100644 --- a/sdcm/utils/cloud_monitor/resources/instances.py +++ b/sdcm/utils/cloud_monitor/resources/instances.py @@ -1,17 +1,22 @@ +from datetime import UTC, datetime, timezone from logging import getLogger -from datetime import datetime, timezone -from boto3 import client as boto3_client from azure.mgmt.compute.models import VirtualMachine +from boto3 import client as boto3_client from google.cloud.compute_v1.types import Instance as GceInstance from sdcm.utils.azure_utils import AzureService -from sdcm.utils.cloud_monitor.common import InstanceLifecycle, NA +from sdcm.utils.cloud_monitor.common import NA, InstanceLifecycle from sdcm.utils.cloud_monitor.resources import CloudInstance, CloudResources -from sdcm.utils.common import aws_tags_to_dict, gce_meta_to_dict, list_instances_aws, list_instances_gce -from sdcm.utils.pricing import AWSPricing, GCEPricing, AzurePricing -from sdcm.utils.gce_utils import SUPPORTED_PROJECTS +from sdcm.utils.common import ( + aws_tags_to_dict, + gce_meta_to_dict, + list_instances_aws, + list_instances_gce, +) from sdcm.utils.context_managers import environment +from sdcm.utils.gce_utils import SUPPORTED_PROJECTS +from sdcm.utils.pricing import AWSPricing, AzurePricing, GCEPricing LOGGER = getLogger(__name__) @@ -107,7 +112,7 @@ def __init__(self, instance: VirtualMachine, resource_group: str): tags = instance.tags or {} creation_time = tags.get("creation_time", None) if creation_time: - creation_time = datetime.fromisoformat(creation_time).replace(tzinfo=timezone.utc) + creation_time = datetime.fromisoformat(creation_time).replace(tzinfo=UTC) super().__init__( cloud="azure", name=instance.name, diff --git a/sdcm/utils/cloud_monitor/resources/static_ips.py b/sdcm/utils/cloud_monitor/resources/static_ips.py index ca8995257fb..745d4e1642c 100644 --- a/sdcm/utils/cloud_monitor/resources/static_ips.py +++ b/sdcm/utils/cloud_monitor/resources/static_ips.py @@ -1,11 +1,15 @@ from logging import getLogger + from google.cloud import compute_v1 from sdcm.utils.azure_utils import AzureService from sdcm.utils.cloud_monitor.common import NA from sdcm.utils.cloud_monitor.resources import CloudResources -from sdcm.utils.common import list_elastic_ips_aws, aws_tags_to_dict, list_static_ips_gce - +from sdcm.utils.common import ( + aws_tags_to_dict, + list_elastic_ips_aws, + list_static_ips_gce, +) LOGGER = getLogger(__name__) diff --git a/sdcm/utils/common.py b/sdcm/utils/common.py index f3fb06538ff..3098981c69b 100644 --- a/sdcm/utils/common.py +++ b/sdcm/utils/common.py @@ -13,82 +13,89 @@ # pylint: disable=too-many-lines -from __future__ import absolute_import, annotations +from __future__ import annotations import atexit +import concurrent.futures +import copy +import ctypes +import datetime +import errno +import getpass +import hashlib +import io import itertools -import os +import json import logging +import os import random -import socket -import time -import datetime -import errno -import threading +import re import select import shutil -import copy +import socket import string -import warnings -import getpass -import json -import re -import uuid -import zipfile -import io import tempfile +import threading +import time import traceback -import ctypes -from typing import Iterable, List, Callable, Optional, Dict, Union, Literal, Any, Type -from urllib.parse import urlparse -from unittest.mock import MagicMock, Mock -from textwrap import dedent -from contextlib import closing, contextmanager -from functools import wraps, cached_property, lru_cache, singledispatch +import uuid +import warnings +import zipfile from collections import defaultdict, namedtuple -import concurrent.futures -from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError +from collections.abc import Callable, Iterable +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as FuturesTimeoutError from concurrent.futures.thread import _python_exit -import hashlib +from contextlib import closing, contextmanager +from functools import cached_property, lru_cache, singledispatch, wraps from pathlib import Path +from textwrap import dedent +from typing import Any, Literal +from unittest.mock import MagicMock, Mock +from urllib.parse import urlparse -import requests import boto3 -from mypy_boto3_s3 import S3Client, S3ServiceResource -from mypy_boto3_ec2 import EC2Client, EC2ServiceResource -from mypy_boto3_ec2.service_resource import Image as EC2Image +import requests from botocore.exceptions import ClientError -import docker # pylint: disable=wrong-import-order; false warning because of docker import (local file vs. package) +from google.cloud.compute_v1 import Image as GceImage +from google.cloud.compute_v1 import ListImagesRequest +from google.cloud.compute_v1.types import Instance as GceInstance +from google.cloud.compute_v1.types import Metadata as GceMetadata from google.cloud.storage import Blob as GceBlob -from google.cloud.compute_v1.types import Metadata as GceMetadata, Instance as GceInstance -from google.cloud.compute_v1 import ListImagesRequest, Image as GceImage +from mypy_boto3_ec2 import EC2Client, EC2ServiceResource +from mypy_boto3_ec2.service_resource import Image as EC2Image +from mypy_boto3_s3 import S3Client, S3ServiceResource from packaging.version import Version from prettytable import PrettyTable +import docker # pylint: disable=wrong-import-order; false warning because of docker import (local file vs. package) +from sdcm import wait +from sdcm.keystore import KeyStore from sdcm.provision.azure.provisioner import AzureProvisioner +from sdcm.remote import LocalCmdRunner, RemoteCmdRunnerBase from sdcm.sct_events import Severity from sdcm.sct_events.system import CpuNotHighEnoughEvent, SoftTimeoutEvent from sdcm.utils.argus import ArgusError, get_argus_client, terminate_resource_in_argus from sdcm.utils.aws_kms import AwsKms -from sdcm.utils.aws_utils import EksClusterCleanupMixin, AwsArchType, get_scylla_images_ec2_resource - -from sdcm.utils.ssh_agent import SSHAgent +from sdcm.utils.aws_utils import ( + AwsArchType, + EksClusterCleanupMixin, + get_scylla_images_ec2_resource, +) +from sdcm.utils.context_managers import environment from sdcm.utils.decorators import retrying -from sdcm import wait -from sdcm.utils.ldap import DEFAULT_PWD_SUFFIX, SASLAUTHD_AUTHENTICATOR, LdapServerType -from sdcm.keystore import KeyStore from sdcm.utils.docker_utils import ContainerManager -from sdcm.utils.gce_utils import GcloudContainerMixin, gce_public_addresses -from sdcm.remote import LocalCmdRunner -from sdcm.remote import RemoteCmdRunnerBase from sdcm.utils.gce_utils import ( - get_gce_compute_instances_client, - get_gce_compute_images_client, + GcloudContainerMixin, + gce_public_addresses, get_gce_compute_addresses_client, + get_gce_compute_images_client, + get_gce_compute_instances_client, get_gce_compute_regions_client, get_gce_storage_client, ) -from sdcm.utils.context_managers import environment +from sdcm.utils.ldap import DEFAULT_PWD_SUFFIX, SASLAUTHD_AUTHENTICATOR, LdapServerType +from sdcm.utils.ssh_agent import SSHAgent LOGGER = logging.getLogger('utils') DEFAULT_AWS_REGION = "eu-west-1" @@ -97,7 +104,7 @@ SCYLLA_GCE_IMAGES_PROJECT = "scylla-images" -class KeyBasedLock(): # pylint: disable=too-few-public-methods +class KeyBasedLock: # pylint: disable=too-few-public-methods """Class designed for creating locks based on hashable keys.""" def __init__(self): @@ -117,7 +124,7 @@ def deprecation(message): def _remote_get_hash(remoter, file_path): try: - result = remoter.run('md5sum {}'.format(file_path), verbose=True) + result = remoter.run(f'md5sum {file_path}', verbose=True) return result.stdout.strip().split()[0] except Exception as details: # pylint: disable=broad-except # noqa: BLE001 LOGGER.error(str(details)) @@ -125,7 +132,7 @@ def _remote_get_hash(remoter, file_path): def _remote_get_file(remoter, src, dst, user_agent=None): - cmd = 'curl -L {} -o {}'.format(src, dst) + cmd = f'curl -L {src} -o {dst}' if user_agent: cmd += ' --user-agent %s' % user_agent return remoter.run(cmd, ignore_status=True) @@ -227,10 +234,10 @@ def verify_scylla_repo_file(content, is_rhel_like=True): valid_prefix = True break LOGGER.debug(line) - assert valid_prefix, 'Repository content has invalid line: {}'.format(line) + assert valid_prefix, f'Repository content has invalid line: {line}' -class S3Storage(): +class S3Storage: bucket_name = 'cloudius-jenkins-test' enable_multipart_threshold_size = 1024 * 1024 * 1024 # 1GB multipart_chunksize = 50 * 1024 * 1024 # 50 MB @@ -260,19 +267,17 @@ def search_by_path(self, path=''): def generate_url(self, file_path, dest_dir=''): bucket_name = self.bucket_name file_name = os.path.basename(os.path.normpath(file_path)) - return "https://{bucket_name}.s3.amazonaws.com/{dest_dir}/{file_name}".format(dest_dir=dest_dir, - file_name=file_name, - bucket_name=bucket_name) + return f"https://{bucket_name}.s3.amazonaws.com/{dest_dir}/{file_name}" def upload_file(self, file_path, dest_dir=''): s3_url = self.generate_url(file_path, dest_dir) - s3_obj = "{}/{}".format(dest_dir, os.path.basename(file_path)) + s3_obj = f"{dest_dir}/{os.path.basename(file_path)}" try: - LOGGER.info("Uploading '{file_path}' to {s3_url}".format(file_path=file_path, s3_url=s3_url)) + LOGGER.info(f"Uploading '{file_path}' to {s3_url}") self._bucket.upload_file(Filename=file_path, Key=s3_obj, Config=self.transfer_config) - LOGGER.info("Uploaded to {0}".format(s3_url)) + LOGGER.info(f"Uploaded to {s3_url}") LOGGER.info("Set public read access") self.set_public_access(key=s3_obj) return s3_url @@ -295,10 +300,10 @@ def set_public_access(self, key): acl_obj.put(ACL='', AccessControlPolicy={'Grants': grants, 'Owner': acl_obj.owner}) def download_file(self, link, dst_dir): - key_name = link.replace("https://{0.bucket_name}.s3.amazonaws.com/".format(self), "") + key_name = link.replace(f"https://{self.bucket_name}.s3.amazonaws.com/", "") file_name = os.path.basename(key_name) try: - LOGGER.info("Downloading {0} from {1}".format(key_name, self.bucket_name)) + LOGGER.info(f"Downloading {key_name} from {self.bucket_name}") self._bucket.download_file(Key=key_name, Filename=os.path.join(dst_dir, file_name), Config=self.transfer_config) @@ -306,7 +311,7 @@ def download_file(self, link, dst_dir): return os.path.join(os.path.abspath(dst_dir), file_name) except Exception as details: # pylint: disable=broad-except # noqa: BLE001 - LOGGER.warning("File {} is not downloaded by reason: {}".format(key_name, details)) + LOGGER.warning(f"File {key_name} is not downloaded by reason: {details}") return "" @@ -341,7 +346,7 @@ def convert_to_date(date_str): if log_type in log_file: results.append({"file_path": log_file, "type": log_type, - "link": "https://{}.s3.amazonaws.com/{}".format(S3Storage.bucket_name, log_file), + "link": f"https://{S3Storage.bucket_name}.s3.amazonaws.com/{log_file}", "date": convert_to_date(log_file.split('/')[1]) }) break @@ -441,12 +446,9 @@ def inner(*args, **kwargs): fun_args = args fun_kwargs = kwargs fun_name = fun.__name__ - LOGGER.debug("[{thread_name}] {fun_name}({fun_args}, {fun_kwargs})".format(thread_name=thread_name, - fun_name=fun_name, - fun_args=fun_args, - fun_kwargs=fun_kwargs)) + LOGGER.debug(f"[{thread_name}] {fun_name}({fun_args}, {fun_kwargs})") return_val = fun(*args, **kwargs) - LOGGER.debug("[{thread_name}] Done.".format(thread_name=thread_name)) + LOGGER.debug(f"[{thread_name}] Done.") return return_val return inner @@ -454,13 +456,13 @@ def inner(*args, **kwargs): results = [] if not self.disable_logging: - LOGGER.debug("Executing in parallel: '{}' on {}".format(func.__name__, self.objects)) + LOGGER.debug(f"Executing in parallel: '{func.__name__}' on {self.objects}") func = func_wrap(func) futures = [] for obj in self.objects: - if unpack_objects and isinstance(obj, (list, tuple)): + if unpack_objects and isinstance(obj, list | tuple): futures.append((self._thread_pool.submit(func, *obj), obj)) elif unpack_objects and isinstance(obj, dict): futures.append((self._thread_pool.submit(func, **obj), obj)) @@ -488,7 +490,7 @@ def inner(*args, **kwargs): raise ParallelObjectException(results=results) return results - def call_objects(self, ignore_exceptions: bool = False) -> list["ParallelObjectResult"]: + def call_objects(self, ignore_exceptions: bool = False) -> list[ParallelObjectResult]: """ Use the ParallelObject run() method to call a list of callables in parallel. Rather than running a single function @@ -573,7 +575,7 @@ def __init__(self, obj, result=None, exc=None): class ParallelObjectException(Exception): - def __init__(self, results: List[ParallelObjectResult]): + def __init__(self, results: list[ParallelObjectResult]): super().__init__() self.results = results @@ -636,7 +638,7 @@ def clean_cloud_resources(tags_dict, config=None, dry_run=False): # pylint: dis return True -def docker_current_container_id() -> Optional[str]: +def docker_current_container_id() -> str | None: with open("/proc/1/cgroup", encoding="utf-8") as cgroup: for line in cgroup: match = DOCKER_CGROUP_RE.search(line) @@ -645,11 +647,11 @@ def docker_current_container_id() -> Optional[str]: return None -def list_clients_docker(builder_name: Optional[str] = None, verbose: bool = False) -> Dict[str, docker.DockerClient]: +def list_clients_docker(builder_name: str | None = None, verbose: bool = False) -> dict[str, docker.DockerClient]: log = LOGGER if verbose else Mock() docker_clients = {} - def get_builder_docker_client(builder: Dict[str, str]) -> None: + def get_builder_docker_client(builder: dict[str, str]) -> None: if not can_connect_to(builder["public_ip"], 22, timeout=5): log.error("%(name)s: can't establish connection to %(public_ip)s:22, port is closed", builder) return @@ -685,11 +687,11 @@ def get_builder_docker_client(builder: Dict[str, str]) -> None: return docker_clients -def list_resources_docker(tags_dict: Optional[dict] = None, - builder_name: Optional[str] = None, +def list_resources_docker(tags_dict: dict | None = None, + builder_name: str | None = None, running: bool = False, group_as_builder: bool = False, - verbose: bool = False) -> Dict[str, Union[list, dict]]: + verbose: bool = False) -> dict[str, list | dict]: log = LOGGER if verbose else Mock() filters = {} @@ -742,7 +744,7 @@ def get_images(builder_name: str, docker_client: docker.DockerClient) -> None: return dict(containers=containers, images=images) -def clean_resources_docker(tags_dict: dict, builder_name: Optional[str] = None, dry_run: bool = False) -> None: +def clean_resources_docker(tags_dict: dict, builder_name: str | None = None, dry_run: bool = False) -> None: assert tags_dict, "tags_dict not provided (can't clean all instances)" def delete_container(container): @@ -812,7 +814,7 @@ def get_instances(region): client: EC2Client = boto3.client('ec2', region_name=region) custom_filter = [] if tags_dict: - custom_filter = [{'Name': 'tag:{}'.format(key), + custom_filter = [{'Name': f'tag:{key}', 'Values': value if isinstance(value, list) else [value]} for key, value in tags_dict.items()] response = client.describe_instances(Filters=custom_filter) @@ -842,7 +844,7 @@ def get_instances(region): total_items = sum([len(value) for _, value in instances.items()]) if verbose: - LOGGER.info("Found total of {} instances.".format(total_items)) + LOGGER.info(f"Found total of {total_items} instances.") return instances @@ -877,7 +879,7 @@ def clean_instances_aws(tags_dict: dict, regions=None, dry_run=False): if node_type and node_type == "sct-runner": LOGGER.info("Skipping Sct Runner instance '%s'", instance_id) continue - LOGGER.info("Going to delete '{instance_id}' [name={name}] ".format(instance_id=instance_id, name=name)) + LOGGER.info(f"Going to delete '{instance_id}' [name={name}] ") if not dry_run: response = client.terminate_instances(InstanceIds=[instance_id]) terminate_resource_in_argus(client=argus_client, resource_name=name) @@ -905,7 +907,7 @@ def get_elastic_ips(region): client: EC2Client = boto3.client('ec2', region_name=region) custom_filter = [] if tags_dict: - custom_filter = [{'Name': 'tag:{}'.format(key), + custom_filter = [{'Name': f'tag:{key}', 'Values': value if isinstance(value, list) else [value]} for key, value in tags_dict.items()] response = client.describe_addresses(Filters=custom_filter) @@ -979,7 +981,7 @@ def get_security_groups_ips(region): LOGGER.info('Going to list aws region "%s"', region) time.sleep(random.random()) client: EC2Client = boto3.client('ec2', region_name=region) - custom_filter = [{'Name': 'tag:{}'.format(key), + custom_filter = [{'Name': f'tag:{key}', 'Values': value if isinstance(value, list) else [value]} for key, value in tags_dict.items() if key != 'NodeType'] response = client.describe_security_groups(Filters=custom_filter) @@ -1245,7 +1247,7 @@ def filter_gce_by_tags(tags_dict, instances: list[GceInstance]) -> list[GceInsta return filtered_instances -def list_instances_gce(tags_dict: Optional[dict] = None, +def list_instances_gce(tags_dict: dict | None = None, running: bool = False, verbose: bool = True) -> list[GceInstance]: """List all instances with specific tags GCE.""" @@ -1288,7 +1290,7 @@ def list_static_ips_gce(verbose=False): class GkeCluster: - def __init__(self, cluster_info: dict, cleaner: "GkeCleaner"): + def __init__(self, cluster_info: dict, cleaner: GkeCleaner): self.cluster_info = cluster_info self.cleaner = cleaner @@ -1352,7 +1354,7 @@ def __del__(self): ContainerManager.destroy_all_containers(self) -def list_clusters_gke(tags_dict: Optional[dict] = None, verbose: bool = False) -> list: +def list_clusters_gke(tags_dict: dict | None = None, verbose: bool = False) -> list: clusters = GkeCleaner().list_gke_clusters() if tags_dict: clusters = filter_k8s_clusters_by_tags(tags_dict, clusters) @@ -1378,8 +1380,8 @@ def create_time(self): return self.body['createdAt'] -def list_clusters_eks(tags_dict: Optional[dict] = None, regions: list = None, - verbose: bool = False) -> List[EksCluster]: +def list_clusters_eks(tags_dict: dict | None = None, regions: list = None, + verbose: bool = False) -> list[EksCluster]: class EksCleaner: name = f"eks-cleaner-{uuid.uuid4()!s:.8}" _containers = {} @@ -1416,8 +1418,7 @@ def list_clusters(self) -> list: # pylint: disable=no-self-use def filter_k8s_clusters_by_tags(tags_dict: dict, - clusters: list[Union["EksCluster", "GkeCluster"]]) -> list[ - Union["EksCluster", "GkeCluster"]]: + clusters: list[EksCluster | GkeCluster]) -> list[EksCluster | GkeCluster]: if "NodeType" in tags_dict and "k8s" not in tags_dict.get("NodeType"): return [] @@ -1636,7 +1637,7 @@ def get_latest_scylla_ami_release(region: str = 'eu-west-1', return str(max(versions)) -def get_latest_scylla_release(product: Literal['scylla', 'scylla-enterprise']) -> str: +def get_latest_scylla_release(product: ScyllaProduct) -> str: """ get latest advertised scylla version from the same service scylla_setup is getting it """ @@ -1674,7 +1675,7 @@ def safe_kill(pid, signal): return False -class FileFollowerIterator(): # pylint: disable=too-few-public-methods +class FileFollowerIterator: # pylint: disable=too-few-public-methods def __init__(self, filename, thread_obj): self.filename = filename self.thread_obj = thread_obj @@ -1700,7 +1701,7 @@ def __iter__(self): yield line -class FileFollowerThread(): +class FileFollowerThread: def __init__(self): self.executor = concurrent.futures.ThreadPoolExecutor(1) # pylint: disable=consider-using-with self._stop_event = threading.Event() @@ -1976,7 +1977,7 @@ def get_branched_gce_images( return images[:1] -@lru_cache() +@lru_cache def ami_built_by_scylla(ami_id: str, region_name: str) -> bool: all_tags = get_ami_tags(ami_id, region_name) if owner_id := all_tags.get('owner_id'): @@ -1985,7 +1986,7 @@ def ami_built_by_scylla(ami_id: str, region_name: str) -> bool: return False -@lru_cache() +@lru_cache def get_ami_tags(ami_id, region_name): """ Get a list of tags of a specific AMI @@ -2048,7 +2049,7 @@ def remove_files(path): LOGGER.info("Remove temporary data manually: \"%s\"", path) -def create_remote_storage_dir(node, path='') -> Optional[str, None]: +def create_remote_storage_dir(node, path='') -> str | None: node_remote_dir = '/tmp' if not path: path = node.name @@ -2190,7 +2191,7 @@ def download_dir_from_cloud(url): parsed = urlparse(url) LOGGER.info("Downloading [%s] to [%s]", url, tmp_dir) if os.path.isdir(tmp_dir) and os.listdir(tmp_dir): - LOGGER.warning("[{}] already exists, skipping download".format(tmp_dir)) + LOGGER.warning(f"[{tmp_dir}] already exists, skipping download") elif url.startswith('s3://'): s3_download_dir(parsed.hostname, parsed.path, tmp_dir) elif url.startswith('gs://'): @@ -2198,7 +2199,7 @@ def download_dir_from_cloud(url): elif os.path.isdir(url): tmp_dir = url else: - raise ValueError("Unsupported url schema or non-existing directory [{}]".format(url)) + raise ValueError(f"Unsupported url schema or non-existing directory [{url}]") if not tmp_dir.endswith('/'): tmp_dir += '/' LOGGER.info("Finished downloading [%s]", url) @@ -2331,8 +2332,7 @@ def search_test_id_on_builder(builder): builder['public_ip'], user=builder["user"], key_file=builder["key_file"]) LOGGER.info('Search on %s', builder['name']) - result = remoter.run("find {where} -name test_id | xargs grep -rl {test_id}".format(where=base_path_on_builder, - test_id=test_id), + result = remoter.run(f"find {base_path_on_builder} -name test_id | xargs grep -rl {test_id}", ignore_status=True, verbose=False) if not result.exited and result.stdout: @@ -2409,11 +2409,11 @@ def clean_resources_according_post_behavior(params, config, logdir, dry_run=Fals def search_test_id_in_latest(logdir): test_id = None - result = LocalCmdRunner().run('cat {0}/latest/test_id'.format(logdir), ignore_status=True) + result = LocalCmdRunner().run(f'cat {logdir}/latest/test_id', ignore_status=True) if not result.exited and result.stdout: test_id = result.stdout.strip() - LOGGER.info("Found latest test_id: {}".format(test_id)) - LOGGER.info("Collect logs for test-run with test-id: {}".format(test_id)) + LOGGER.info(f"Found latest test_id: {test_id}") + LOGGER.info(f"Collect logs for test-run with test-id: {test_id}") else: LOGGER.error('test_id not found. Exit code: %s; Error details %s', result.exited, result.stderr) return test_id @@ -2654,7 +2654,7 @@ def approach_enospc(): free_space_size = int(result.stdout.split()[3]) occupy_space_size = int(free_space_size * 90 / 100) occupy_space_cmd = f'fallocate -l {occupy_space_size}K /var/lib/scylla/occupy_90percent.{time.time()}' - LOGGER.debug('Cost 90% free space on /var/lib/scylla/ by {}'.format(occupy_space_cmd)) + LOGGER.debug(f'Cost 90% free space on /var/lib/scylla/ by {occupy_space_cmd}') try: target_node.remoter.sudo(occupy_space_cmd, verbose=True) except Exception as details: # pylint: disable=broad-except # noqa: BLE001 @@ -2670,7 +2670,7 @@ def approach_enospc(): def clean_enospc_on_node(target_node, sleep_time): - LOGGER.debug('Sleep {} seconds before releasing space to scylla'.format(sleep_time)) + LOGGER.debug(f'Sleep {sleep_time} seconds before releasing space to scylla') time.sleep(sleep_time) LOGGER.debug('Delete occupy_90percent file to release space to scylla-server') @@ -2768,7 +2768,7 @@ def _shorten_alpha_sequences(value: str, max_alpha_chunk_size: int) -> str: return output -def _shorten_sequences_in_string(value: Union[str, List[str]], max_alpha_chunk_size: int) -> str: +def _shorten_sequences_in_string(value: str | list[str], max_alpha_chunk_size: int) -> str: chunks = [] if isinstance(value, str): tmp = value.split('-') @@ -2855,7 +2855,7 @@ def walk_thru_data(data, path: str, separator: str = '/') -> Any: continue if name[0] == '[' and name[-1] == ']': name = name[1:-1] # noqa: PLW2901 - if name.isalnum() and isinstance(current_value, (list, tuple, set)): + if name.isalnum() and isinstance(current_value, list | tuple | set): try: current_value = current_value[int(name)] except Exception: # pylint: disable=broad-except # noqa: BLE001 @@ -3046,7 +3046,7 @@ def SoftTimeoutContext(timeout: int, operation: str): # pylint: disable=invalid duration=duration).publish_or_dump() -def raise_exception_in_thread(thread: threading.Thread, exception_type: Type[BaseException]): +def raise_exception_in_thread(thread: threading.Thread, exception_type: type[BaseException]): res = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_ulong(thread.ident), ctypes.py_object(exception_type)) LOGGER.debug("PyThreadState_SetAsyncExc: return [%s]", res) @@ -3074,7 +3074,7 @@ def get_placement_groups(region): client: EC2Client = boto3.client('ec2', region_name=region) custom_filter = [] if tags_dict: - custom_filter = [{'Name': 'tag:{}'.format(key), + custom_filter = [{'Name': f'tag:{key}', 'Values': value if isinstance(value, list) else [value]} for key, value in tags_dict.items()] response = client.describe_placement_groups(Filters=custom_filter) @@ -3100,7 +3100,7 @@ def get_placement_groups(region): total_items = sum([len(value) for _, value in placement_groups.items()]) if verbose: - LOGGER.info("Found total of {} instances.".format(total_items)) + LOGGER.info(f"Found total of {total_items} instances.") return placement_groups @@ -3126,7 +3126,7 @@ def clean_placement_groups_aws(tags_dict: dict, regions=None, dry_run=False): client: EC2Client = boto3.client('ec2', region_name=region) for instance in instance_list: name = instance.get("GroupName") - LOGGER.info("Going to delete placement group '{name} ".format(name=name)) + LOGGER.info(f"Going to delete placement group '{name} ") if not dry_run: try: response = client.delete_placement_group(GroupName=name) diff --git a/sdcm/utils/compaction_ops.py b/sdcm/utils/compaction_ops.py index 59d616cabfa..40a55a1d597 100644 --- a/sdcm/utils/compaction_ops.py +++ b/sdcm/utils/compaction_ops.py @@ -1,10 +1,11 @@ import time -from typing import Callable, Optional, NamedTuple, Union +from collections.abc import Callable +from typing import NamedTuple from fabric.runners import Result from sct import LOGGER -from sdcm.cluster import BaseNode, BaseCluster, BaseScyllaCluster +from sdcm.cluster import BaseCluster, BaseNode, BaseScyllaCluster from sdcm.rest.storage_service_client import StorageServiceClient @@ -36,7 +37,7 @@ class CompactionOps: NODETOOL_CMD = NodetoolCommands() SCRUB_MODES = ScrubModes() - def __init__(self, cluster: Union[BaseCluster, BaseScyllaCluster], node: Optional[BaseNode] = None): + def __init__(self, cluster: BaseCluster | BaseScyllaCluster, node: BaseNode | None = None): self.cluster = cluster self.node = node if node else self.cluster.nodes[0] self.storage_service_client = StorageServiceClient(node=self.node) @@ -47,7 +48,7 @@ def trigger_major_compaction(self, keyspace: str = "keyspace1", cf: str = "stand def trigger_scrub_compaction(self, keyspace: str = "keyspace1", cf: str = "standard1", - scrub_mode: Optional[str] = None) -> Result: + scrub_mode: str | None = None) -> Result: params = {"keyspace": keyspace, "cf": cf, "scrub_mode": scrub_mode} return self.storage_service_client.scrub_ks_cf(**params) @@ -85,7 +86,7 @@ def stop_reshape_compaction(self) -> Result: def stop_validation_compaction(self) -> Result: return self.stop_scrub_compaction() - def disable_autocompaction_on_ks_cf(self, node: BaseNode, keyspace: str = "", cf: Optional[str] = ""): + def disable_autocompaction_on_ks_cf(self, node: BaseNode, keyspace: str = "", cf: str = ""): node = node if node else self.node node.run_nodetool(f'disableautocompaction {keyspace} {cf}') @@ -95,7 +96,7 @@ def _stop_compaction(self, nodetool_cmd: str) -> Result: @staticmethod def stop_on_user_compaction_logged(node: BaseNode, watch_for: str, timeout: int, - stop_func: Callable, mark: Optional[int] = None) -> Result: + stop_func: Callable, mark: int | None = None) -> Result: LOGGER.info("Starting to watch for user compaction logged...") start_time = time.time() with open(node.system_log, encoding="utf-8") as log_file: diff --git a/sdcm/utils/cshdrhistogram.py b/sdcm/utils/cshdrhistogram.py index 4e82b79417f..946545d127a 100644 --- a/sdcm/utils/cshdrhistogram.py +++ b/sdcm/utils/cshdrhistogram.py @@ -1,14 +1,12 @@ +import datetime import glob import os.path -import datetime import sys - from typing import Any from hdrh.histogram import HdrHistogram from hdrh.log import HistogramLogReader - CS_HDR_FILE_WC = "*/cs_hdr_*.hdr" TIME_INTERVAL = 300 diff --git a/sdcm/utils/csrangehistogram.py b/sdcm/utils/csrangehistogram.py index 0c12095f31f..e11dc5941c6 100644 --- a/sdcm/utils/csrangehistogram.py +++ b/sdcm/utils/csrangehistogram.py @@ -1,12 +1,12 @@ import glob -import os.path -import sys import logging import multiprocessing -from typing import Any -from enum import Enum -from dataclasses import asdict, dataclass, make_dataclass +import os.path +import sys from concurrent.futures.process import ProcessPoolExecutor +from dataclasses import asdict, dataclass, make_dataclass +from enum import Enum +from typing import Any from hdrh.histogram import HdrHistogram from hdrh.log import HistogramLogReader diff --git a/sdcm/utils/data_validator.py b/sdcm/utils/data_validator.py index d0188bb5414..5d33b9eb0ec 100644 --- a/sdcm/utils/data_validator.py +++ b/sdcm/utils/data_validator.py @@ -132,19 +132,17 @@ # running stress. # import json +import logging import os import re -import logging import uuid -from typing import NamedTuple, Optional +from typing import NamedTuple from sdcm.sct_events import Severity +from sdcm.sct_events.health import DataValidatorEvent from sdcm.test_config import TestConfig from sdcm.utils.database_query_utils import fetch_all_rows - from sdcm.utils.user_profile import get_profile_content -from sdcm.sct_events.health import DataValidatorEvent - LOGGER = logging.getLogger(__name__) @@ -153,8 +151,8 @@ class DataForValidation(NamedTuple): views: tuple # list of view names with data for validation actual_data: list expected_data: list - before_update_rows: Optional[list] - after_update_rows: Optional[list] + before_update_rows: list | None + after_update_rows: list | None # pylint: disable=too-many-instance-attributes, too-many-public-methods @@ -497,7 +495,7 @@ def list_of_view_names_for_update_test(self): self._validate_updated_per_view, )) def fetch_data_for_validation_after_update(self, during_nemesis: bool, views_set: tuple, session) -> \ - Optional[DataForValidation]: + DataForValidation | None: # views_set[0] - view name with rows before update # views_set[1] - view name with rows after update # views_set[2] - view name with all expected partition keys diff --git a/sdcm/utils/database_query_utils.py b/sdcm/utils/database_query_utils.py index 5f6c766c66f..a55dd06fa39 100644 --- a/sdcm/utils/database_query_utils.py +++ b/sdcm/utils/database_query_utils.py @@ -13,12 +13,11 @@ # pylint: disable=too-many-lines -from __future__ import absolute_import, annotations +from __future__ import annotations import logging import os import sys -from typing import List from cassandra import ConsistencyLevel @@ -126,8 +125,8 @@ def collect_partitions_info(self, ignore_limit_rows_number: bool = False) -> dic return None partitions[i] = pk_rows_num_result - stats_file.write('{i}:{rows}, '.format(i=i, rows=partitions[i])) - LOGGER.info('File with partitions row data: {}'.format(partitions_stats_file)) + stats_file.write(f'{i}:{partitions[i]}, ') + LOGGER.info(f'File with partitions row data: {partitions_stats_file}') if save_into_file_name == self.PARTITIONS_ROWS_BEFORE: self.partitions_rows_collected = True return partitions @@ -181,7 +180,7 @@ def get_table_clustering_order(ks_cf: str, ck_name: str, session) -> str: return clustering_order -def get_partition_keys(ks_cf: str, session, pk_name: str = 'pk', limit: int = None) -> List[str]: +def get_partition_keys(ks_cf: str, session, pk_name: str = 'pk', limit: int = None) -> list[str]: """ Return list of partitions from a requested table :param session: diff --git a/sdcm/utils/decorators.py b/sdcm/utils/decorators.py index 878343f488d..8357077d31c 100644 --- a/sdcm/utils/decorators.py +++ b/sdcm/utils/decorators.py @@ -11,20 +11,18 @@ # # Copyright (c) 2020 ScyllaDB -import sys -import time -import logging import datetime import json +import logging import os +import sys +import time +from collections.abc import Callable +from functools import cached_property, partial, wraps -from functools import wraps, partial, cached_property -from typing import Optional, Callable from sdcm.sct_events.database import DatabaseLogEvent - from sdcm.sct_events.event_counter import EventCounterContextManager - LOGGER = logging.getLogger(__name__) @@ -113,7 +111,7 @@ def inner(*args, **kwargs): class_name = "" if args and func.__name__ in dir(args[0]): class_name = " <%s>" % args[0].__class__.__name__ - action = "%s%s" % (msg, class_name) + action = f"{msg}{class_name}" start_time = datetime.datetime.now() LOGGER.debug("BEGIN: %s", action) res = func(*args, **kwargs) @@ -155,7 +153,7 @@ def wrapped(*args, **kwargs): return wrapped -def latency_calculator_decorator(original_function: Optional[Callable] = None, *, legend: Optional[str] = None): # noqa: PLR0915 +def latency_calculator_decorator(original_function: Callable | None = None, *, legend: str | None = None): # noqa: PLR0915 """ Gets the start time, end time and then calculates the latency based on function 'calculate_latency'. diff --git a/sdcm/utils/distro.py b/sdcm/utils/distro.py index 2c9d8ee34b0..ebdc1f68270 100644 --- a/sdcm/utils/distro.py +++ b/sdcm/utils/distro.py @@ -13,6 +13,7 @@ import enum import logging + from sdcm.utils.decorators import static_init # pylint: disable=unused-import LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/docker_remote.py b/sdcm/utils/docker_remote.py index 395255c9751..1117e8fcd71 100644 --- a/sdcm/utils/docker_remote.py +++ b/sdcm/utils/docker_remote.py @@ -1,6 +1,6 @@ import logging import shlex -from functools import cached_property, cache +from functools import cache, cached_property from pathlib import Path from sdcm.cluster import BaseNode diff --git a/sdcm/utils/docker_utils.py b/sdcm/utils/docker_utils.py index c025053102a..baf3b50316c 100644 --- a/sdcm/utils/docker_utils.py +++ b/sdcm/utils/docker_utils.py @@ -11,25 +11,30 @@ # # Copyright (c) 2020 ScyllaDB +import logging import os import re -import logging import warnings +from functools import cache from pprint import pformat from types import SimpleNamespace # pylint: disable=no-name-in-module -from typing import List, Optional, Union, Any, Tuple, Protocol -from functools import cache +from typing import Any, Protocol import docker -from docker.errors import DockerException, NotFound, ImageNotFound, NullResource, BuildError -from docker.models.images import Image +from docker.errors import ( + BuildError, + DockerException, + ImageNotFound, + NotFound, + NullResource, +) from docker.models.containers import Container - +from docker.models.images import Image +from sdcm.keystore import KeyStore, pub_key_from_private_key_file from sdcm.remote import LOCALRUNNER from sdcm.remote.base import CommandRunner from sdcm.remote.remote_file import remote_file -from sdcm.keystore import pub_key_from_private_key_file, KeyStore -from sdcm.utils.decorators import retrying, Retry +from sdcm.utils.decorators import Retry, retrying DOCKER_API_CALL_TIMEOUT = 180 # seconds @@ -53,11 +58,11 @@ class DockerClient(docker.DockerClient): def __call__(self, cmd, timeout=10): deprecation("consider to use Docker Python module instead of using Docker CLI commands") - res = LOCALRUNNER.run('docker {}'.format(cmd), ignore_status=True, timeout=timeout) + res = LOCALRUNNER.run(f'docker {cmd}', ignore_status=True, timeout=timeout) if res.exit_status: if 'No such container:' in res.stderr: raise NotFound(res.stderr) - raise DockerException('command: {}, error: {}, output: {}'.format(cmd, res.stderr, res.stdout)) + raise DockerException(f'command: {cmd}, error: {res.stderr}, output: {res.stdout}') return res @@ -67,7 +72,7 @@ def __call__(self, cmd, timeout=10): class _Name(SimpleNamespace): # pylint: disable=too-few-public-methods - def __init__(self, name: Optional[str]) -> None: + def __init__(self, name: str) -> None: family, *member = (None, ) if name is None else name.split(":", 1) super().__init__(full=name, family=family, member=member[0] if member else None, member_as_args=tuple(member)) @@ -148,7 +153,7 @@ class ContainerManager: # pylint: disable=too-many-public-methods) default_docker_client = _docker @classmethod - def get_docker_client(cls, instance: INodeWithContainerManager, name: Optional[str] = None) -> DockerClient: + def get_docker_client(cls, instance: INodeWithContainerManager, name: str | None = None) -> DockerClient: container = None if name is None else cls.get_container(instance, name, raise_not_found_exc=False) if container is None: return cls._get_docker_client_for_new_container(instance, _Name(name)) @@ -162,8 +167,8 @@ def _get_docker_client_for_new_container(cls, instance: INodeWithContainerManage def _get_attr_for_name(instance: INodeWithContainerManager, name: _Name, attr: str, - default: Optional[Any] = None, - name_only_lookup: bool = False) -> Optional[Any]: + default: Any | None = None, + name_only_lookup: bool = False) -> Any | None: attr_candidate_list = [] # Try to get attribute value for container family. @@ -184,7 +189,7 @@ def _get_attr_for_name(instance: INodeWithContainerManager, return default @classmethod - def get_container(cls, instance: INodeWithContainerManager, name, raise_not_found_exc: bool = True) -> Optional[Container]: + def get_container(cls, instance: INodeWithContainerManager, name, raise_not_found_exc: bool = True) -> Container | None: assert name is not None, "None is not allowed as a container name" container = instance._containers.get(name) @@ -319,7 +324,7 @@ def get_ip_address(cls, instance: INodeWithContainerManager, name: str) -> str: return ip_address @classmethod - def get_container_port(cls, instance: INodeWithContainerManager, name: str, port: Union[int, str]) -> Optional[int]: + def get_container_port(cls, instance: INodeWithContainerManager, name: str, port: int | str) -> int | None: container = cls.get_container(instance, name) try: return int(container.ports[f"{port}/tcp"][0]["HostPort"]) @@ -343,7 +348,7 @@ def get_host_volume_path(cls, instance: INodeWithContainerManager, container_nam def get_environ(cls, instance: INodeWithContainerManager, name: str) -> dict: container = cls.get_container(instance, name) - def normalize(key: Any, value: Any = None) -> Tuple[Any, Any]: + def normalize(key: Any, value: Any = None) -> tuple[Any, Any]: return (key, value, ) return dict(normalize(*(item.split("=", 1))) for item in container.attrs["Config"]["Env"]) @@ -415,11 +420,11 @@ def build_container_image(cls, instance: INodeWithContainerManager, name: str, * return image @classmethod - def _get_container_image_tag(cls, instance: INodeWithContainerManager, name: _Name) -> Optional[str]: + def _get_container_image_tag(cls, instance: INodeWithContainerManager, name: _Name) -> str | None: return cls._get_attr_for_name(instance, name, "container_image_tag", name_only_lookup=True) @classmethod - def _get_container_image_dockerfile_args(cls, instance: INodeWithContainerManager, name: _Name) -> Optional[dict]: + def _get_container_image_dockerfile_args(cls, instance: INodeWithContainerManager, name: _Name) -> dict | None: return cls._get_attr_for_name(instance, name, "container_image_dockerfile_args", name_only_lookup=True) @staticmethod @@ -427,7 +432,7 @@ def _build_args(**kwargs): return kwargs @classmethod - def get_containers_by_prefix(cls, prefix: str, docker_client: DockerClient = None) -> List[Container]: + def get_containers_by_prefix(cls, prefix: str, docker_client: DockerClient = None) -> list[Container]: docker_client = docker_client or cls.default_docker_client return docker_client.containers.list(all=True, filters={"name": f'{prefix}*'}) diff --git a/sdcm/utils/es_index.py b/sdcm/utils/es_index.py index b46695cb7d0..72327045223 100644 --- a/sdcm/utils/es_index.py +++ b/sdcm/utils/es_index.py @@ -1,12 +1,12 @@ -import logging import json -import sys +import logging import os - -from typing import Any +import sys from pathlib import Path +from typing import Any from sdcm.es import ES + sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) diff --git a/sdcm/utils/es_queries.py b/sdcm/utils/es_queries.py index 780c2dd60c3..e8e4c763c7c 100644 --- a/sdcm/utils/es_queries.py +++ b/sdcm/utils/es_queries.py @@ -1,8 +1,6 @@ import logging - from datetime import datetime, timedelta - LOGGER = logging.getLogger(__name__) @@ -36,7 +34,7 @@ def filter_test_details(self): test_details = 'test_details.job_name:\"{}\" '.format( self.test_doc['_source']['test_details']['job_name'].split('/')[0]) test_details += self.test_cmd_details() - test_details += ' AND test_details.time_completed: {}'.format(self.date_re) + test_details += f' AND test_details.time_completed: {self.date_re}' test_details += ' AND test_details.test_name: {}'.format(self.test_name.replace(":", r"\:")) return test_details @@ -95,8 +93,8 @@ def filter_test_details(self): test_details = self.build_filter_job_name() if not self.use_wide_query: test_details += self.test_cmd_details() - test_details += ' AND test_details.time_completed: {}'.format(self.date_re) - test_details += ' AND {}'.format(self.build_filter_test_name()) + test_details += f' AND test_details.time_completed: {self.date_re}' + test_details += f' AND {self.build_filter_test_name()}' return test_details def build_filter_job_name(self): @@ -149,7 +147,7 @@ def get_query_filter_by_job_prefix(job_item): full_job_name = self.test_doc['_source']['test_details']['job_name'] if '/' in full_job_name: full_job_name = f'"{full_job_name}"' - job_filter_query = r'test_details.job_name.keyword: {} '.format(full_job_name) + job_filter_query = rf'test_details.job_name.keyword: {full_job_name} ' return job_filter_query @@ -199,8 +197,8 @@ def test_cmd_details(self): continue param_val = self.test_doc['_source']['test_details'][cassandra_stress][param] if param in ['profile', 'ops']: - param_val = "\"{}\"".format(param_val) - test_details += ' AND test_details.{}.{}: {}'.format(cassandra_stress, param, param_val) + param_val = f"\"{param_val}\"" + test_details += f' AND test_details.{cassandra_stress}.{param}: {param_val}' return test_details @@ -238,7 +236,7 @@ def test_cmd_details(self): param_val = self.test_doc['_source']['test_details'][ycsb].get(param) if not param_val: continue - test_details += ' AND test_details.{}.{}: {}'.format(ycsb, param, param_val) + test_details += f' AND test_details.{ycsb}.{param}: {param_val}' return test_details @@ -255,7 +253,7 @@ def filter_test_details(self): test_details = 'test_details.job_name: \"{}\" '.format( self.test_doc['_source']['test_details']['job_name'].split('/')[0]) test_details += self.test_cmd_details() - test_details += ' AND test_details.time_completed: {}'.format(self.date_re) + test_details += f' AND test_details.time_completed: {self.date_re}' test_details += r' AND test_details.sub_type: cdc* ' return test_details diff --git a/sdcm/utils/file.py b/sdcm/utils/file.py index 53dca2e29fc..a5b964f37d0 100644 --- a/sdcm/utils/file.py +++ b/sdcm/utils/file.py @@ -11,8 +11,9 @@ # # Copyright (c) 2020 ScyllaDB -from typing import Optional, TextIO, List, Union, AnyStr, Iterable +from collections.abc import Iterable from re import Pattern +from typing import AnyStr, TextIO # pylint: disable=too-few-public-methods @@ -35,8 +36,8 @@ class File: """ # pylint: disable=too-many-arguments - def __init__(self, path: str, mode: str = 'r', buffering: Optional[int] = None, # noqa: PLR0913 - encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, + def __init__(self, path: str, mode: str = 'r', buffering: int | None = None, # noqa: PLR0913 + encoding: str | None = None, errors: str | None = None, newline: str | None = None, closefd: bool = True): self.path = path self.mode = mode @@ -89,7 +90,7 @@ def write(self, any_str: AnyStr) -> 'File': self._io.write(any_str) return self - def writelines(self, list_of_any_str: List[AnyStr]) -> 'File': + def writelines(self, list_of_any_str: list[AnyStr]) -> 'File': self._io.writelines(list_of_any_str) return self @@ -103,10 +104,10 @@ def read(self, num_bytes: int = -1) -> AnyStr: def readline(self, limit: int = -1) -> AnyStr: return self._io.readline(limit) - def readlines(self, hint: int = -1) -> List[AnyStr]: + def readlines(self, hint: int = -1) -> list[AnyStr]: return self._io.readlines(hint) - def read_lines_filtered(self, *patterns: Union[Pattern]) -> Iterable[str]: + def read_lines_filtered(self, *patterns: Pattern) -> Iterable[str]: """ Read lines from the file, filter them and yield :param patterns: List of patterns @@ -123,8 +124,7 @@ def generator(): def iterate_lines(self) -> Iterable[str]: def generator(): - for line in self._io: - yield line + yield from self._io return ReiterableGenerator(generator=generator) def __getattr__(self, item): diff --git a/sdcm/utils/gce_builder.py b/sdcm/utils/gce_builder.py index e27b9bd34b9..932eecf1ef5 100644 --- a/sdcm/utils/gce_builder.py +++ b/sdcm/utils/gce_builder.py @@ -10,23 +10,23 @@ # See LICENSE for more details. # # Copyright (c) 2023 ScyllaDB -import sys import logging -from typing import Any +import sys from functools import cached_property +from typing import Any import click +import google.api_core.exceptions import requests -from google.oauth2 import service_account -from google.cloud import compute_v1 from google.api_core.extended_operation import ExtendedOperation -import google.api_core.exceptions +from google.cloud import compute_v1 +from google.oauth2 import service_account +from sdcm.keystore import KeyStore +from sdcm.sct_runner import GceSctRunner +from sdcm.utils.context_managers import environment from sdcm.utils.gce_region import GceRegion from sdcm.utils.gce_utils import SUPPORTED_PROJECTS, SUPPORTED_REGIONS -from sdcm.utils.context_managers import environment -from sdcm.sct_runner import GceSctRunner -from sdcm.keystore import KeyStore LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/gce_region.py b/sdcm/utils/gce_region.py index b4fbfcbcc38..85b0cbc76ad 100644 --- a/sdcm/utils/gce_region.py +++ b/sdcm/utils/gce_region.py @@ -14,18 +14,16 @@ import logging from functools import cached_property -import googleapiclient.errors -from googleapiclient.discovery import build -from google.oauth2 import service_account import google import google.api_core.exceptions -from google.cloud import storage -from google.cloud import compute_v1 +import googleapiclient.errors +from google.cloud import compute_v1, storage from google.cloud.compute_v1 import Firewall +from google.oauth2 import service_account +from googleapiclient.discovery import build from sdcm.keystore import KeyStore - LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/gce_utils.py b/sdcm/utils/gce_utils.py index cb4744999e9..2408d2ed9d3 100644 --- a/sdcm/utils/gce_utils.py +++ b/sdcm/utils/gce_utils.py @@ -11,24 +11,22 @@ # # Copyright (c) 2020 ScyllaDB -import os -import re import json -import random import logging +import os +import random +import re import time -from typing import Any, List, Literal +from typing import Any -from google.oauth2 import service_account -from google.cloud import compute_v1 -from google.cloud.compute_v1 import Image -from google.cloud import storage from google.api_core.extended_operation import ExtendedOperation +from google.cloud import compute_v1, storage +from google.cloud.compute_v1 import Image +from google.oauth2 import service_account from googleapiclient.discovery import build from sdcm.keystore import KeyStore -from sdcm.utils.docker_utils import ContainerManager, DockerException, Container - +from sdcm.utils.docker_utils import Container, ContainerManager, DockerException # NOTE: we cannot use neither 'slim' nor 'alpine' versions because we need the 'beta' component be installed. GOOGLE_CLOUD_SDK_IMAGE = "google/cloud-sdk:437.0.1" @@ -374,14 +372,14 @@ def create_instance( # pylint: disable=too-many-arguments,too-many-locals,too-m project_id: str, zone: str, instance_name: str, - disks: List[compute_v1.AttachedDisk], + disks: list[compute_v1.AttachedDisk], machine_type: str = "n1-standard-1", network_name: str = None, subnetwork_link: str = None, internal_ip: str = None, external_access: bool = False, external_ipv4: str = None, - accelerators: List[compute_v1.AcceleratorConfig] = None, + accelerators: list[compute_v1.AcceleratorConfig] = None, spot: bool = False, instance_termination_action: str = "STOP", custom_hostname: str = None, diff --git a/sdcm/utils/get_username.py b/sdcm/utils/get_username.py index c0dc06ab5b5..368bd5c6b21 100644 --- a/sdcm/utils/get_username.py +++ b/sdcm/utils/get_username.py @@ -11,8 +11,8 @@ # # Copyright (c) 2020 ScyllaDB -import os import getpass +import os import subprocess @@ -54,7 +54,7 @@ def get_username() -> str: # pylint: disable=too-many-return-statements # noqa return get_email_user(res.stdout) # We didn't find email, fallback to current user with unknown email user identifier - return "linux_user={}".format(current_linux_user) + return f"linux_user={current_linux_user}" if __name__ == "__main__": diff --git a/sdcm/utils/git.py b/sdcm/utils/git.py index fec53e1689c..2a2eac8b436 100644 --- a/sdcm/utils/git.py +++ b/sdcm/utils/git.py @@ -1,8 +1,8 @@ """ Simple git wrappers that provide useful information about current repository """ -import subprocess import logging +import subprocess from typing import TypedDict LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/health_checker.py b/sdcm/utils/health_checker.py index 8908634d29a..d97b3b4499b 100644 --- a/sdcm/utils/health_checker.py +++ b/sdcm/utils/health_checker.py @@ -11,14 +11,13 @@ # # Copyright (c) 2020 ScyllaDB -import time import logging -from typing import Generator +import time +from collections.abc import Generator from sdcm.sct_events import Severity from sdcm.sct_events.health import ClusterHealthValidatorEvent - CHECK_NODE_HEALTH_RETRIES = 3 CHECK_NODE_HEALTH_RETRY_DELAY = 45 diff --git a/sdcm/utils/housekeeping.py b/sdcm/utils/housekeeping.py index 61a97d1fb27..a462c3031b4 100644 --- a/sdcm/utils/housekeeping.py +++ b/sdcm/utils/housekeeping.py @@ -12,13 +12,13 @@ # Copyright (c) 2020 ScyllaDB import logging -from typing import Sequence, Optional, Any +from collections.abc import Sequence +from typing import Any import mysql.connector from sdcm.keystore import KeyStore - DB_NAME = "housekeeping" LOGGER = logging.getLogger(__name__) @@ -52,7 +52,7 @@ def connect(self, db_name: str = DB_NAME) -> None: database=db_name) self._cursor = self._connection.cursor() - def execute(self, query: str, args: Optional[Sequence[Any]] = None) -> Sequence[Row]: + def execute(self, query: str, args: Sequence[Any] | None = None) -> Sequence[Row]: LOGGER.debug("Query: `%s', Args: %s", query, args) self._connection.reconnect() self._cursor.execute(query, args) @@ -70,11 +70,11 @@ def close(self) -> None: self._connection.close() self._connection = None - def get_most_recent_record(self, query: str, args: Optional[Sequence[Any]] = None) -> Optional[Row]: + def get_most_recent_record(self, query: str, args: Sequence[Any] | None = None) -> Row | None: result = self.execute(query + " ORDER BY -dt LIMIT 1", args) LOGGER.debug("Housekeeping DB saved info, query '%s': %s", query, result) return result[0] if result else None - def get_new_records(self, query: str, args: Optional[Sequence[Any]] = None, last_id: int = 0) -> Sequence[Row]: + def get_new_records(self, query: str, args: Sequence[Any] | None = None, last_id: int = 0) -> Sequence[Row]: args = (tuple(args) if args else ()) + (last_id, ) return self.execute(query + " id > ", args) diff --git a/sdcm/utils/jepsen.py b/sdcm/utils/jepsen.py index 81461e25028..0c36d9f0f49 100644 --- a/sdcm/utils/jepsen.py +++ b/sdcm/utils/jepsen.py @@ -11,14 +11,12 @@ # # Copyright (c) 2021 ScyllaDB -import uuid import logging -from typing import Optional +import uuid from functools import cached_property -from sdcm.utils.common import list_logs_by_test_id, get_free_port -from sdcm.utils.docker_utils import ContainerManager, Container, DockerException - +from sdcm.utils.common import get_free_port, list_logs_by_test_id +from sdcm.utils.docker_utils import Container, ContainerManager, DockerException JEPSEN_IMAGE = "tjake/jepsen" JEPSEN_RESULTS_PORT = 8080 @@ -53,7 +51,7 @@ def runcmd(self, command: str, detach: bool = False) -> None: raise DockerException(f"{self._jepsen_container}: {res.output.decode('utf-8')}") @property - def jepsen_results_port(self) -> Optional[int]: + def jepsen_results_port(self) -> int | None: return ContainerManager.get_container_port(self, "jepsen", JEPSEN_RESULTS_PORT) @staticmethod diff --git a/sdcm/utils/k8s/__init__.py b/sdcm/utils/k8s/__init__.py index 75e0b399b8d..f4db30c7915 100644 --- a/sdcm/utils/k8s/__init__.py +++ b/sdcm/utils/k8s/__init__.py @@ -13,40 +13,39 @@ # pylint: disable=too-many-arguments,too-many-lines import abc -import getpass +import contextlib import json +import logging +import multiprocessing import os -import time import queue -import logging import re import threading -import multiprocessing -import contextlib -from tempfile import NamedTemporaryFile -from typing import Iterator, Optional, Union, Callable, List +import time +from collections.abc import Callable, Iterator from functools import cached_property, partialmethod from pathlib import Path +from tempfile import NamedTemporaryFile import kubernetes as k8s import yaml from paramiko.config import invoke -from urllib3.util.retry import Retry from urllib3.exceptions import ( IncompleteRead, ProtocolError, ReadTimeoutError, ) +from urllib3.util.retry import Retry -from sdcm.log import SDCMAdapter from sdcm import sct_abs_path +from sdcm.log import SDCMAdapter from sdcm.remote import LOCALRUNNER from sdcm.utils.common import walk_thru_data -from sdcm.utils.decorators import timeout as timeout_decor, retrying -from sdcm.utils.docker_utils import ContainerManager, DockerException, Container +from sdcm.utils.decorators import retrying +from sdcm.utils.decorators import timeout as timeout_decor +from sdcm.utils.docker_utils import Container, ContainerManager from sdcm.wait import wait_for - KUBECTL_BIN = "kubectl" HELM_IMAGE = "alpine/helm:3.8.0" @@ -300,16 +299,16 @@ def kubectl_cmd(kluster, *command, namespace=None, ignore_k8s_server_url=False): return " ".join(cmd) @classmethod - def kubectl(cls, kluster, *command, namespace: Optional[str] = None, timeout: int = KUBECTL_TIMEOUT, # noqa: PLR0913 - remoter: Optional['KubernetesCmdRunner'] = None, ignore_status: bool = False, verbose: bool = True): + def kubectl(cls, kluster, *command, namespace: str | None = None, timeout: int = KUBECTL_TIMEOUT, # noqa: PLR0913 + remoter: 'KubernetesCmdRunner' | None = None, ignore_status: bool = False, verbose: bool = True): cmd = cls.kubectl_cmd(kluster, *command, namespace=namespace, ignore_k8s_server_url=bool(remoter)) if remoter is None: remoter = LOCALRUNNER return remoter.run(cmd, timeout=timeout, ignore_status=ignore_status, verbose=verbose) @classmethod - def kubectl_multi_cmd(cls, kluster, *command, namespace: Optional[str] = None, timeout: int = KUBECTL_TIMEOUT, # noqa: PLR0913 - remoter: Optional['KubernetesCmdRunner'] = None, ignore_status: bool = False, + def kubectl_multi_cmd(cls, kluster, *command, namespace: str | None = None, timeout: int = KUBECTL_TIMEOUT, # noqa: PLR0913 + remoter: 'KubernetesCmdRunner' | None = None, ignore_status: bool = False, verbose: bool = True): total_command = ' '.join(command) final_command = [] @@ -327,7 +326,7 @@ def kubectl_multi_cmd(cls, kluster, *command, namespace: Optional[str] = None, t @classmethod def apply_file(cls, kluster, config_path, namespace=None, # pylint: disable=too-many-locals,too-many-branches # noqa: PLR0913 timeout=KUBECTL_TIMEOUT, environ=None, envsubst=True, - modifiers: List[Callable] = None, server_side=False): + modifiers: list[Callable] = None, server_side=False): if environ: environ_str = (' '.join([f'{name}="{value}"' for name, value in environ.items()])) + ' ' else: @@ -421,11 +420,11 @@ def patch_kubectl_auth_config(cls, config, auth_type, cmd: str, args: list): raise ValueError(f'Unknown auth-type {auth_type}') @staticmethod - def wait_for_pods_with_condition(kluster, condition: str, total_pods: Union[int, Callable], # noqa: PLR0913 + def wait_for_pods_with_condition(kluster, condition: str, total_pods: int | Callable, # noqa: PLR0913 timeout: float, namespace: str, selector: str = '', sleep_between_retries: int = 10): - assert isinstance(total_pods, (int, float)) or callable(total_pods), ( + assert isinstance(total_pods, int | float) or callable(total_pods), ( "total_pods should be number or callable") waiter_message = ( f"{kluster.region_name}: Wait for the '{total_pods}' pod(s) " @@ -441,7 +440,7 @@ def wait_for_condition(kluster, condition, total_pods, timeout, namespace, selec namespace=namespace, timeout=timeout * 60 // 5 + 10) count = result.stdout.count('condition met') - if isinstance(total_pods, (int, float)): + if isinstance(total_pods, int | float): if total_pods != count: raise RuntimeError('Not all pods reported') elif callable(total_pods): @@ -453,7 +452,7 @@ def wait_for_condition(kluster, condition, total_pods, timeout, namespace, selec wait_for_condition(kluster, condition, total_pods, timeout, namespace, selector) @staticmethod - def wait_for_pods_readiness(kluster, total_pods: Union[int, Callable], readiness_timeout: float, # noqa: PLR0913 + def wait_for_pods_readiness(kluster, total_pods: int | Callable, readiness_timeout: float, # noqa: PLR0913 namespace: str, selector: str = '', sleep_between_retries: int = 10): KubernetesOps.wait_for_pods_with_condition(kluster, condition='condition=Ready', @@ -464,7 +463,7 @@ def wait_for_pods_readiness(kluster, total_pods: Union[int, Callable], readiness sleep_between_retries=sleep_between_retries) @staticmethod - def wait_for_pods_running(kluster, total_pods: Union[int, Callable], timeout: float, # noqa: PLR0913 + def wait_for_pods_running(kluster, total_pods: int | Callable, timeout: float, # noqa: PLR0913 namespace: str, selector: str = '', sleep_between_retries: int = 10): KubernetesOps.wait_for_pods_with_condition(kluster, condition="jsonpath='{.status.phase}'=Running", @@ -597,8 +596,7 @@ def gather_k8s_logs(cls, logdir_path, kubectl=None, namespaces=None) -> None: # if not namespaces: # Read all the namespaces from already saved file - with open(logdir / cluster_scope_dir / "namespaces.wide", - mode="r", encoding="utf-8") as namespaces_file: + with open(logdir / cluster_scope_dir / "namespaces.wide", encoding="utf-8") as namespaces_file: # Reverse order of namespaces because preferred ones are there namespaces = [n.split()[0] for n in namespaces_file.readlines()[1:]][::-1] elif isinstance(namespaces, str): @@ -692,7 +690,7 @@ def helm_container_run_args(self) -> dict: def _helm_container(self) -> Container: return ContainerManager.run_container(self, "helm") - def helm(self, kluster, *command: str, namespace: Optional[str] = None, values: 'HelmValues' = None, prepend_command=None) -> str: # pylint: disable=no-self-use + def helm(self, kluster, *command: str, namespace: str | None = None, values: 'HelmValues' = None, prepend_command=None) -> str: # pylint: disable=no-self-use cmd = [ f"HELM_CONFIG_HOME={kluster.helm_dir_path}", "helm", @@ -739,9 +737,9 @@ def _helm_install_or_upgrade(self, # noqa: PLR0913 use_devel: bool = False, debug: bool = True, values: 'HelmValues' = None, - namespace: Optional[str] = None, + namespace: str | None = None, atomic: bool = False, - timeout: Optional[str] = None) -> str: + timeout: str | None = None) -> str: command = [operation_type, target_chart_name, source_chart_name] prepend_command = [] if version: @@ -956,7 +954,7 @@ def _process_line(self, line: str) -> None: # pylint: disable=too-many-branches # NOTE: following condition allows to skip processing of all the # load-balancer/headless endpoints that do not refer to any Scylla pod. labels = metadata.get('labels', {}) - if not all((key in labels.keys() for key in self.SCYLLA_PODS_EXPECTED_LABEL_KEYS)): + if not all(key in labels.keys() for key in self.SCYLLA_PODS_EXPECTED_LABEL_KEYS): return None if name := metadata.get('name'): @@ -1019,7 +1017,7 @@ def process_callback(callback, namespace, pod_name=None, add_pod_name_as_kwarg=F callback=callback[:-1], namespace=namespace, pod_name=pod_name, add_pod_name_as_kwarg=callback[-1]) - def register_callbacks(self, callbacks: Union[Callable, list[Callable]], + def register_callbacks(self, callbacks: Callable | list[Callable], namespace: str, pod_name: str = '__each__', add_pod_name_as_kwarg: bool = False) -> None: """Register callbacks to be called after a Scylla pod IP change. @@ -1071,10 +1069,10 @@ def register_callbacks(self, callbacks: Union[Callable, list[Callable]], for callback in callbacks: if callable(callback): callback = [callback, [], {}] # noqa: PLW2901 - if (isinstance(callback, (tuple, list)) + if (isinstance(callback, tuple | list) and len(callback) == 3 # noqa: PLR2004 and callable(callback[0]) - and isinstance(callback[1], (tuple, list)) + and isinstance(callback[1], tuple | list) and isinstance(callback[2], dict)): self.mapper_dict[namespace][pod_name]['callbacks'].append( (callback[0], callback[1], callback[2], add_pod_name_as_kwarg)) @@ -1083,14 +1081,14 @@ def register_callbacks(self, callbacks: Union[Callable, list[Callable]], "Unexpected type (%s) of the callback: %s. Skipping", type(callback), callback) -def convert_cpu_units_to_k8s_value(cpu: Union[float, int]) -> str: +def convert_cpu_units_to_k8s_value(cpu: float | int) -> str: if isinstance(cpu, float): if not cpu.is_integer(): return f'{int(cpu * 1000)}m' return f'{int(cpu)}' -def convert_memory_units_to_k8s_value(memory: Union[float, int]) -> str: +def convert_memory_units_to_k8s_value(memory: float | int) -> str: if isinstance(memory, float): if not memory.is_integer(): return f'{int(memory * 1024)}Mi' @@ -1266,7 +1264,7 @@ def delete(self, path): def as_dict(self): return self._data - def __eq__(self, other: Union['HelmValues', dict]): + def __eq__(self, other: 'HelmValues' | dict): if isinstance(other, HelmValues): return self._data == other._data return self._data == other diff --git a/sdcm/utils/k8s/chaos_mesh.py b/sdcm/utils/k8s/chaos_mesh.py index 1649a66a64d..bd1c2e460ec 100644 --- a/sdcm/utils/k8s/chaos_mesh.py +++ b/sdcm/utils/k8s/chaos_mesh.py @@ -14,18 +14,18 @@ import ast import json import logging +import time from datetime import datetime from enum import Enum from json import JSONDecodeError from tempfile import NamedTemporaryFile -import time from typing import Literal import yaml from botocore.utils import deep_merge -from sdcm.log import SDCMAdapter from sdcm import sct_abs_path +from sdcm.log import SDCMAdapter from sdcm.utils.common import time_period_str_to_seconds from sdcm.utils.k8s import HelmValues, get_helm_pool_affinity_values diff --git a/sdcm/utils/ldap.py b/sdcm/utils/ldap.py index 54f01b43e2f..8fbad773cd3 100644 --- a/sdcm/utils/ldap.py +++ b/sdcm/utils/ldap.py @@ -11,17 +11,16 @@ # # Copyright (c) 2020 ScyllaDB -from time import sleep from enum import Enum +from time import sleep from cassandra import Unauthorized -from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES +from ldap3 import ALL, ALL_ATTRIBUTES, Connection, Server from ldap3.core.exceptions import LDAPSocketOpenError from sdcm.keystore import KeyStore from sdcm.utils.decorators import retrying - LDAP_IMAGE = "osixia/openldap:1.4.0" LDAP_PORT = 389 LDAP_SSL_PORT = 636 diff --git a/sdcm/utils/loader_utils.py b/sdcm/utils/loader_utils.py index 332b13740af..d30b928a43a 100644 --- a/sdcm/utils/loader_utils.py +++ b/sdcm/utils/loader_utils.py @@ -100,11 +100,11 @@ def _run_all_stress_cmds(self, stress_queue, params): if 'compression' in stress_cmd: if 'keyspace_name' not in stress_params: compression_prefix = re.search('compression=(.*)Compressor', stress_cmd).group(1) - keyspace_name = "keyspace_{}".format(compression_prefix.lower()) + keyspace_name = f"keyspace_{compression_prefix.lower()}" stress_params.update({'keyspace_name': keyspace_name}) # Run all stress commands - self.log.debug('stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'stress cmd: {stress_cmd}') if stress_cmd.startswith('scylla-bench'): stress_queue.append(self.run_stress_thread(stress_cmd=stress_cmd, stats_aggregate_cmds=False, @@ -163,7 +163,7 @@ def run_cs_user_profiles(self, cs_profiles: str | int | list, duration_per_cs_pr round_robin = self.params.get('round_robin') for i, cs_profile in enumerate(cs_profiles): - assert os.path.exists(cs_profile), 'File not found: {}'.format(cs_profile) + assert os.path.exists(cs_profile), f'File not found: {cs_profile}' self.log.debug('Run stress with user profile %s', cs_profile) profile_dst = os.path.join('/tmp', os.path.basename(cs_profile)) with open(cs_profile, encoding="utf-8") as file: @@ -174,7 +174,7 @@ def run_cs_user_profiles(self, cs_profiles: str | int | list, duration_per_cs_pr params = {'stress_cmd': stress_cmd, 'profile': cs_profile, 'round_robin': round_robin} stress_params = dict(params) - self.log.debug('stress cmd: {}'.format(stress_cmd)) + self.log.debug(f'stress cmd: {stress_cmd}') stress_queue.append(self.run_stress_thread(**stress_params)) return stress_queue @@ -191,7 +191,7 @@ def run_stress_and_verify_threads(self, params=None): @staticmethod def _get_keyspace_name(ks_number, keyspace_pref='keyspace'): - return '{}{}'.format(keyspace_pref, ks_number) + return f'{keyspace_pref}{ks_number}' def _run_cql_commands(self, cmds, node=None): node = node if node else self.db_cluster.nodes[0] @@ -287,15 +287,15 @@ def _set_credentials_to_cmd(cmd): if roles and "' was " - f"not found. Fix the command and re-run the test. Command: {cmd}") + raise OSError("Stress command is defined wrong. Expected pattern '' was " + f"not found. Fix the command and re-run the test. Command: {cmd}") sla_role_name = roles[role_index].name.replace('"', '') sla_role_password = roles[role_index].password return re.sub(r'', f'user={sla_role_name} password={sla_role_password}', cmd) @@ -322,6 +322,6 @@ def _set_credentials_to_cmd(cmd): stress_cmds.append(_set_credentials_to_cmd(cmd=stress_cmd)) params[stress_op] = stress_cmds - except EnvironmentError as error_message: + except OSError as error_message: TestFrameworkEvent(source=parent_class_name, message=error_message, severity=Severity.CRITICAL).publish() raise diff --git a/sdcm/utils/log.py b/sdcm/utils/log.py index c0f04788abd..953fcc47bd2 100644 --- a/sdcm/utils/log.py +++ b/sdcm/utils/log.py @@ -1,6 +1,6 @@ -import sys import logging import logging.config +import sys import warnings import urllib3 diff --git a/sdcm/utils/microbenchmarking/perf_simple_query_reporter.py b/sdcm/utils/microbenchmarking/perf_simple_query_reporter.py index 3c305a9c3a3..51dd163e573 100644 --- a/sdcm/utils/microbenchmarking/perf_simple_query_reporter.py +++ b/sdcm/utils/microbenchmarking/perf_simple_query_reporter.py @@ -1,7 +1,7 @@ -import os import json +import os -from sdcm.results_analyze import BaseResultsAnalyzer, collections, TestConfig +from sdcm.results_analyze import BaseResultsAnalyzer, TestConfig, collections def keys_exists(element, *keys): @@ -50,7 +50,7 @@ def get_sorted_results_as_list(results): def check_regression(self, test_id, mad_deviation_limit=0.02, regression_limit=0.05, is_gce=False): # pylint: disable=too-many-locals,too-many-statements # noqa: PLR0915 doc = self.get_test_by_id(test_id) if not doc: - self.log.error('Cannot find test by id: {}!'.format(test_id)) + self.log.error(f'Cannot find test by id: {test_id}!') return False test_stats = self._get_perf_simple_query_result(doc) diff --git a/sdcm/utils/nemesis.py b/sdcm/utils/nemesis.py index f436772bf9f..d60e2440a6f 100644 --- a/sdcm/utils/nemesis.py +++ b/sdcm/utils/nemesis.py @@ -1,6 +1,6 @@ import logging -from functools import cached_property from difflib import unified_diff +from functools import cached_property from pathlib import Path import yaml @@ -8,7 +8,6 @@ import sdcm.utils.nemesis_jobs_configs as config - DEFAULT_JOB_NAME = "longevity-5gb-1h" TEST_CASE_TEMPLATE_DIR = "test_config" JOB_PIPELINE_TEMPLATE_DIR = "job_pipeline" @@ -88,7 +87,7 @@ def create_test_cases_from_template(self): old_config = yaml.safe_load(content) new_keys = set(new_config.keys()) old_keys = set(old_config.keys()) - if len((key_diff := new_keys.symmetric_difference(old_keys))) != 0: + if len(key_diff := new_keys.symmetric_difference(old_keys)) != 0: diff = unified_diff( content.splitlines(keepends=True), nemesis_config_body.splitlines(keepends=True), diff --git a/sdcm/utils/net.py b/sdcm/utils/net.py index 15ae0116e4f..8a83a05b906 100644 --- a/sdcm/utils/net.py +++ b/sdcm/utils/net.py @@ -2,9 +2,9 @@ Utility functions related to network information """ import ipaddress - import os import socket + import requests diff --git a/sdcm/utils/operations_thread.py b/sdcm/utils/operations_thread.py index 25c8dd03538..9f730bfa582 100644 --- a/sdcm/utils/operations_thread.py +++ b/sdcm/utils/operations_thread.py @@ -7,14 +7,14 @@ import time import traceback from dataclasses import dataclass, fields -from typing import Literal, TYPE_CHECKING, get_type_hints, get_origin +from typing import TYPE_CHECKING, Literal, get_origin, get_type_hints from prettytable import PrettyTable from sdcm import wait if TYPE_CHECKING: - from sdcm.cluster import BaseScyllaCluster, BaseCluster + from sdcm.cluster import BaseCluster, BaseScyllaCluster # pylint: disable=too-many-instance-attributes diff --git a/sdcm/utils/operator/multitenant_common.py b/sdcm/utils/operator/multitenant_common.py index ad77dcee125..26396f61063 100644 --- a/sdcm/utils/operator/multitenant_common.py +++ b/sdcm/utils/operator/multitenant_common.py @@ -17,8 +17,8 @@ import logging import time -from sdcm.utils.common import ParallelObject from sdcm.tester import silence +from sdcm.utils.common import ParallelObject from sdcm.utils.database_query_utils import PartitionsValidationAttributes LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/pricing.py b/sdcm/utils/pricing.py index 0871cf70d1b..dbe2def3de8 100644 --- a/sdcm/utils/pricing.py +++ b/sdcm/utils/pricing.py @@ -1,12 +1,13 @@ import json from datetime import datetime, timedelta -from functools import lru_cache +from functools import cache, lru_cache from logging import getLogger + import boto3 import requests from mypy_boto3_pricing import PricingClient -from sdcm.utils.cloud_monitor.common import InstanceLifecycle +from sdcm.utils.cloud_monitor.common import InstanceLifecycle LOGGER = getLogger(__name__) @@ -17,7 +18,7 @@ class AWSPricing: def __init__(self): self.pricing_client: PricingClient = boto3.client('pricing', region_name='us-east-1') - @lru_cache(maxsize=None) + @cache def get_on_demand_instance_price(self, region_name: str, instance_type: str): regions_names_map = { "af-south-1": "Africa (Cape Town)", @@ -62,15 +63,14 @@ def get_on_demand_instance_price(self, region_name: str, instance_type: str): ], MaxResults=10 ) - assert response['PriceList'], "failed to get price for {instance_type} in {region_name}".format( - region_name=region_name, instance_type=instance_type) + assert response['PriceList'], f"failed to get price for {instance_type} in {region_name}" price = response['PriceList'][0] price_dimensions = next(iter(json.loads(price)['terms']['OnDemand'].values()))['priceDimensions'] instance_price = next(iter(price_dimensions.values()))['pricePerUnit']['USD'] return float(instance_price) @staticmethod - @lru_cache(maxsize=None) + @cache def get_spot_instance_price(region_name, instance_type): """currently doesn't take AZ into consideration""" client = boto3.client('ec2', region_name=region_name) @@ -285,7 +285,7 @@ def get_instance_price(self, region, instance_type, state, lifecycle): return 0 @staticmethod - @lru_cache(maxsize=None) + @cache def _get_sku_prices(instance_type: str, region): resp = requests.get( f"https://prices.azure.com/api/retail/prices?$filter=serviceName eq 'Virtual Machines' " diff --git a/sdcm/utils/profiler.py b/sdcm/utils/profiler.py index 25481bf24ad..479231d44a3 100644 --- a/sdcm/utils/profiler.py +++ b/sdcm/utils/profiler.py @@ -12,17 +12,15 @@ # Copyright (c) 2020 ScyllaDB # -import os -import threading +import atexit +import cProfile +import marshal import multiprocessing +import os import pstats -import cProfile import shutil -import atexit -import marshal +import threading import time -from typing import List - TIMERS = { 'perf': time.perf_counter, @@ -283,7 +281,7 @@ def fork(self) -> 'ProcessProfile': def send_stats_to_main_process(self, stats): self._queue.put(stats) - def get_stats_holders(self) -> List[StatsHolder]: + def get_stats_holders(self) -> list[StatsHolder]: """ Get slave's stats via queue, make list of StatsHolder of them and return it. """ diff --git a/sdcm/utils/properties.py b/sdcm/utils/properties.py index a1c4f7ffe32..05a537d6fd9 100644 --- a/sdcm/utils/properties.py +++ b/sdcm/utils/properties.py @@ -1,4 +1,4 @@ -from typing import Union, TextIO +from typing import TextIO class PropertiesDict(dict): @@ -27,7 +27,7 @@ def values(self): yield value -def serialize(data: Union[dict, PropertiesDict]) -> str: +def serialize(data: dict | PropertiesDict) -> str: output = [] items = data.all_items() if isinstance(data, PropertiesDict) else data.items() for key, value in items: @@ -41,7 +41,7 @@ def serialize(data: Union[dict, PropertiesDict]) -> str: return '\n'.join(output) -def deserialize(data: Union[str, TextIO]) -> PropertiesDict: +def deserialize(data: str | TextIO) -> PropertiesDict: if not isinstance(data, str): data = data.read() output = PropertiesDict() diff --git a/sdcm/utils/quota.py b/sdcm/utils/quota.py index 0de44db9631..d334710ca52 100644 --- a/sdcm/utils/quota.py +++ b/sdcm/utils/quota.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, annotations +from __future__ import annotations import logging import time @@ -7,13 +7,12 @@ from sdcm.exceptions import QuotaConfigurationFailure from sdcm.wait import wait_for - LOGGER = logging.getLogger(__name__) def enable_quota_on_node(node): if not is_quota_enabled_on_node(node): - LOGGER.info('Enabling quota on node {}'.format(node.name)) + LOGGER.info(f'Enabling quota on node {node.name}') change_default_grub_cmd = 'sed -i s\'/GRUB_CMDLINE_LINUX_DEFAULT="quiet splash"/' \ 'GRUB_CMDLINE_LINUX_DEFAULT="quiet splash rootflags=uquota,pquota"/\' ' \ '/etc/default/grub' @@ -24,14 +23,14 @@ def enable_quota_on_node(node): node.remoter.sudo(cmd=change_default_grub_cmd) node.remoter.sudo(cmd=mk_grub_cmd) node.remoter.sudo(cmd=enable_uquota_on_mount_cmd) - LOGGER.debug('Rebooting node: "{}"'.format(node.name)) + LOGGER.debug(f'Rebooting node: "{node.name}"') node.reboot(hard=False) node.wait_node_fully_start() def is_quota_enabled_on_node(node): """ Verify usrquota is enabled on scylla user. """ - LOGGER.info("Verifying quota is configured on the node {}".format(node.name)) + LOGGER.info(f"Verifying quota is configured on the node {node.name}") verify_usrquot_in_mount_cmd = "mount | grep /var/lib/scylla | awk {'print $6'} | cut -d ',' -f 8" result = node.remoter.run(cmd=verify_usrquot_in_mount_cmd).stdout.rstrip().replace(")", "") if not result == "usrquota": @@ -48,7 +47,7 @@ def get_currently_used_space_by_scylla_user(node) -> int: def get_quota_size_to_configure(node): used_space = get_currently_used_space_by_scylla_user(node) - LOGGER.debug('Currently used space by scylla user is: {}K'.format(used_space)) + LOGGER.debug(f'Currently used space by scylla user is: {used_space}K') return used_space + 3000000 @@ -61,7 +60,7 @@ def configure_quota_on_node_for_scylla_user_context(node): Remove quota configuration """ quota_size = get_quota_size_to_configure(node) - LOGGER.info("Configuring quota for scylla user with quota size: {}K on node {}".format(quota_size, node)) + LOGGER.info(f"Configuring quota for scylla user with quota size: {quota_size}K on node {node}") conf_quota_cmd = f"xfs_quota -x -c 'limit bsoft={quota_size}K bhard={quota_size}K scylla' /var/lib/scylla" node.remoter.sudo(conf_quota_cmd, verbose=True) verify_quota_is_configured_for_scylla_user(node) @@ -79,8 +78,8 @@ def verify_quota_is_configured_for_scylla_user(node): if quota_value == "0": LOGGER.error("Failed to configure quota on user") raise QuotaConfigurationFailure( - "Quota was not configured for scylla user. The value for hard quota is: {}".format(quota_value)) - LOGGER.info("Quota was configured for scylla user with value: {}".format(quota_value)) + f"Quota was not configured for scylla user. The value for hard quota is: {quota_value}") + LOGGER.info(f"Quota was configured for scylla user with value: {quota_value}") def remove_quota_configuration_from_scylla_user_on_node(node): @@ -102,11 +101,11 @@ def approach_end_of_quota(): occupy_space_size = int((quota_size - currently_used_space) * 90 / 100) occupy_space_cmd = f"su - scylla -c 'fallocate -l {occupy_space_size}K /var/lib/scylla/occupy_90percent.{time.time()}'" try: - LOGGER.debug('Cost 90% free space on /var/lib/scylla/ by {}'.format(occupy_space_cmd)) + LOGGER.debug(f'Cost 90% free space on /var/lib/scylla/ by {occupy_space_cmd}') node.remoter.sudo(occupy_space_cmd, Verbose=True) except Exception as exc: # pylint: disable=broad-except # noqa: BLE001 LOGGER.warning("We should have reached the expected I/O error and quota has exceeded\n" - "Message: {}".format(str(exc))) + f"Message: {str(exc)}") return bool(list(quota_exceeded_appearances)) try: diff --git a/sdcm/utils/raft/__init__.py b/sdcm/utils/raft/__init__.py index 96e2a7b306a..085dc77e110 100644 --- a/sdcm/utils/raft/__init__.py +++ b/sdcm/utils/raft/__init__.py @@ -1,15 +1,14 @@ import contextlib import logging import random - -from enum import Enum -from typing import Protocol, NamedTuple, Mapping, Iterable from collections import namedtuple +from collections.abc import Iterable, Mapping +from enum import Enum +from typing import NamedTuple, Protocol +from sdcm.sct_events import Severity from sdcm.sct_events.database import DatabaseLogEvent from sdcm.sct_events.filters import EventsSeverityChangerFilter -from sdcm.sct_events import Severity - LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/raft/common.py b/sdcm/utils/raft/common.py index edfe03eec64..dc2063e5ec1 100644 --- a/sdcm/utils/raft/common.py +++ b/sdcm/utils/raft/common.py @@ -1,16 +1,18 @@ -import logging import contextlib - -from typing import Iterable, Callable +import logging +from collections.abc import Callable, Iterable from functools import partial +from sdcm.cluster import BaseMonitorSet, BaseNode, BaseScyllaCluster from sdcm.exceptions import BootstrapStreamErrorFailure -from sdcm.cluster import BaseNode, BaseScyllaCluster, BaseMonitorSet -from sdcm.wait import wait_for -from sdcm.sct_events.group_common_events import decorate_with_context, \ - ignore_stream_mutation_fragments_errors, ignore_ycsb_connection_refused +from sdcm.sct_events.group_common_events import ( + decorate_with_context, + ignore_stream_mutation_fragments_errors, + ignore_ycsb_connection_refused, +) from sdcm.utils.adaptive_timeouts import Operations, adaptive_timeout from sdcm.utils.common import ParallelObject +from sdcm.wait import wait_for LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/remote_logger.py b/sdcm/utils/remote_logger.py index 8fcb9f910d7..5eedf4bf516 100644 --- a/sdcm/utils/remote_logger.py +++ b/sdcm/utils/remote_logger.py @@ -13,17 +13,18 @@ from __future__ import annotations -import time -import socket import logging +import socket import subprocess -from abc import abstractmethod, ABCMeta +import time +from abc import ABCMeta, abstractmethod from datetime import datetime from functools import cached_property -from threading import Thread, Event as ThreadEvent -from typing import TYPE_CHECKING -from multiprocessing import Process, Event +from multiprocessing import Event, Process from textwrap import dedent +from threading import Event as ThreadEvent +from threading import Thread +from typing import TYPE_CHECKING import kubernetes as k8s from dateutil.parser import isoparse @@ -37,11 +38,11 @@ from sdcm.sct_events import Severity from sdcm.sct_events.decorators import raise_event_on_failure from sdcm.sct_events.system import TestFrameworkEvent -from sdcm.utils.k8s import KubernetesOps from sdcm.utils.decorators import retrying +from sdcm.utils.k8s import KubernetesOps if TYPE_CHECKING: - from typing import Generator + from collections.abc import Generator from sdcm.cluster import BaseNode @@ -306,7 +307,7 @@ class K8sClientLogger(LoggerBase): # pylint: disable=too-many-instance-attribut RECONNECT_DELAY = 30 CHUNK_SIZE = 64 - def __init__(self, pod: "sdcm.cluster_k8s.BasePodContainer", target_log_file: str): + def __init__(self, pod: sdcm.cluster_k8s.BasePodContainer, target_log_file: str): """ Reads logs from a k8s pod using k8s client API and forwards it to a file. diff --git a/sdcm/utils/remotewebbrowser.py b/sdcm/utils/remotewebbrowser.py index 33c5a14df5a..debc4b0c22a 100644 --- a/sdcm/utils/remotewebbrowser.py +++ b/sdcm/utils/remotewebbrowser.py @@ -11,21 +11,19 @@ # # Copyright (c) 2020 ScyllaDB -import time import logging import os -from typing import Optional +import time from functools import cached_property import paramiko -from docker import DockerClient # pylint: disable=wrong-import-order -from selenium.webdriver import Remote, ChromeOptions +from selenium.webdriver import ChromeOptions, Remote -from sdcm.utils.docker_utils import ContainerManager, DOCKER_API_CALL_TIMEOUT -from sdcm.utils.common import get_free_port, wait_for_port, normalize_ipv6_url +from docker import DockerClient # pylint: disable=wrong-import-order +from sdcm.utils.common import get_free_port, normalize_ipv6_url, wait_for_port +from sdcm.utils.docker_utils import DOCKER_API_CALL_TIMEOUT, ContainerManager from sdcm.utils.ssh_agent import SSHAgent - WEB_DRIVER_IMAGE = "selenium/standalone-chrome:3.141.59-20210713" WEB_DRIVER_REMOTE_PORT = 4444 WEB_DRIVER_CONTAINER_START_DELAY = 30 # seconds @@ -42,7 +40,7 @@ def web_driver_container_run_args(self) -> dict: volumes={"/dev/shm": {"bind": "/dev/shm"}, }) @property - def web_driver_docker_client(self) -> Optional[DockerClient]: + def web_driver_docker_client(self) -> DockerClient | None: if not self.ssh_login_info or self.ssh_login_info["key_file"] is None: # running with docker backend, no real monitor node, fallback to use local docker return None diff --git a/sdcm/utils/replication_strategy_utils.py b/sdcm/utils/replication_strategy_utils.py index 2b4fbd51229..2b626833685 100644 --- a/sdcm/utils/replication_strategy_utils.py +++ b/sdcm/utils/replication_strategy_utils.py @@ -1,8 +1,7 @@ import ast import re - +from collections.abc import Callable from contextlib import ContextDecorator -from typing import Callable, Dict from sdcm.cluster import BaseNode @@ -76,7 +75,7 @@ class temporary_replication_strategy_setter(ContextDecorator): # pylint: disabl def __init__(self, node: BaseNode) -> None: self.node = node - self.preserved: Dict[str, ReplicationStrategy] = {} + self.preserved: dict[str, ReplicationStrategy] = {} def __enter__(self) -> Callable[..., None]: return self diff --git a/sdcm/utils/rsyslog.py b/sdcm/utils/rsyslog.py index 339decc226a..751848a2b67 100644 --- a/sdcm/utils/rsyslog.py +++ b/sdcm/utils/rsyslog.py @@ -11,12 +11,11 @@ # # Copyright (c) 2020 ScyllaDB -import os -import logging import getpass +import logging +import os from functools import cached_property from tempfile import mkstemp -from typing import Optional from sdcm.utils.docker_utils import ContainerManager @@ -33,7 +32,7 @@ def rsyslog_confpath(self) -> str: return generate_rsyslog_conf_file() @property - def rsyslog_port(self) -> Optional[int]: + def rsyslog_port(self) -> int | None: return ContainerManager.get_container_port(self, "rsyslog", RSYSLOG_PORT) @property diff --git a/sdcm/utils/s3_remote_uploader.py b/sdcm/utils/s3_remote_uploader.py index 6f4175bb7ae..ea3e450fe03 100644 --- a/sdcm/utils/s3_remote_uploader.py +++ b/sdcm/utils/s3_remote_uploader.py @@ -13,7 +13,6 @@ import logging import socket from os.path import expanduser -from typing import List import boto3 from botocore.response import StreamingBody @@ -40,7 +39,7 @@ def read(self, amt=1024): return buff -def upload_remote_files_directly_to_s3(ssh_info: dict[str, str], files: List[str], # pylint: disable=too-many-arguments # noqa: PLR0913 +def upload_remote_files_directly_to_s3(ssh_info: dict[str, str], files: list[str], # pylint: disable=too-many-arguments # noqa: PLR0913 s3_bucket: str, s3_key: str, max_size_gb: int = 400, public_read_acl: bool = False): """Streams given remote files/directories straight to S3 as tar.gz file. Returns download link.""" diff --git a/sdcm/utils/sct_cmd_helpers.py b/sdcm/utils/sct_cmd_helpers.py index d0213f92aaf..5303c2672cf 100644 --- a/sdcm/utils/sct_cmd_helpers.py +++ b/sdcm/utils/sct_cmd_helpers.py @@ -11,17 +11,15 @@ # # Copyright (c) 2022 ScyllaDB -import os import logging -from typing import Optional +import os import click import __main__ - from sdcm.cluster import TestConfig -from sdcm.utils.azure_utils import AzureService from sdcm.utils.azure_region import region_name_to_location +from sdcm.utils.azure_utils import AzureService from sdcm.utils.common import ( all_aws_regions, get_all_gce_regions, @@ -42,7 +40,7 @@ def get_all_regions(cloud_provider: str) -> list[str] | None: class CloudRegion(click.ParamType): name = "cloud_region" - def __init__(self, cloud_provider: Optional[str] = None): + def __init__(self, cloud_provider: str | None = None): super().__init__() self.cloud_provider = cloud_provider diff --git a/sdcm/utils/scylla_args.py b/sdcm/utils/scylla_args.py index fbfa16999bd..9746d44cf90 100644 --- a/sdcm/utils/scylla_args.py +++ b/sdcm/utils/scylla_args.py @@ -11,11 +11,11 @@ # # Copyright (c) 2021 ScyllaDB -import re -import logging import argparse -from typing import Text, NoReturn, Callable - +import logging +import re +from collections.abc import Callable +from typing import NoReturn, Text # Regexp for parsing arguments from the output of `scylla --help' command: # $ scylla --help @@ -40,12 +40,12 @@ class ScyllaArgParser(argparse.ArgumentParser): def __init__(self, prog: str) -> None: super().__init__(prog=prog, argument_default=argparse.SUPPRESS, add_help=False) - def error(self, message: Text) -> NoReturn: + def error(self, message: str) -> NoReturn: LOGGER.error(message) raise ScyllaArgError(message) @classmethod - def from_scylla_help(cls, help_text: Text, duplicate_cb: Callable = None) -> "ScyllaArgParser": + def from_scylla_help(cls, help_text: str, duplicate_cb: Callable = None) -> "ScyllaArgParser": parser = cls(prog="scylla") duplicates = set() for *args, val in SCYLLA_ARG.findall(help_text): diff --git a/sdcm/utils/ssh_agent.py b/sdcm/utils/ssh_agent.py index cc8b4aeb86d..747abb7aa8f 100644 --- a/sdcm/utils/ssh_agent.py +++ b/sdcm/utils/ssh_agent.py @@ -11,15 +11,14 @@ # # Copyright (c) 2020 ScyllaDB -import os -import json import atexit +import json import logging -from typing import Iterable +import os +from collections.abc import Iterable from sdcm.remote import LOCALRUNNER - LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/sstable/load_utils.py b/sdcm/utils/sstable/load_utils.py index e08b07fb927..dbfd0ce20dd 100644 --- a/sdcm/utils/sstable/load_utils.py +++ b/sdcm/utils/sstable/load_utils.py @@ -2,15 +2,22 @@ import random import re from collections import namedtuple -from typing import Any, List, Iterable +from collections.abc import Iterable +from typing import Any from sdcm.keystore import KeyStore from sdcm.remote import LocalCmdRunner -from sdcm.utils.common import remote_get_file, LOGGER, RemoteTemporaryFolder +from sdcm.utils.common import LOGGER, RemoteTemporaryFolder, remote_get_file from sdcm.utils.decorators import timeout as timeout_decor -from sdcm.utils.sstable.load_inventory import (TestDataInventory, BIG_SSTABLE_COLUMN_1_DATA, COLUMN_1_DATA, - MULTI_NODE_DATA, BIG_SSTABLE_MULTI_COLUMNS_DATA, MULTI_COLUMNS_DATA) from sdcm.utils.node import RequestMethods, build_node_api_command +from sdcm.utils.sstable.load_inventory import ( + BIG_SSTABLE_COLUMN_1_DATA, + BIG_SSTABLE_MULTI_COLUMNS_DATA, + COLUMN_1_DATA, + MULTI_COLUMNS_DATA, + MULTI_NODE_DATA, + TestDataInventory, +) from sdcm.wait import wait_for_log_lines LOCAL_CMD_RUNNER = LocalCmdRunner() @@ -39,7 +46,7 @@ def get_random_item(items: list, pop: bool = False) -> Any: return items[random.randint(0, len(items) - 1)] @classmethod - def distribute_test_files_to_cluster_nodes(cls, nodes, test_data: List[TestDataInventory]) -> List: + def distribute_test_files_to_cluster_nodes(cls, nodes, test_data: list[TestDataInventory]) -> list: """ Distribute test sstables over cluster nodes for `load-and-stream` test: the feature allow loading arbitrary sstables that do not belong to a node into the cluster. @@ -181,7 +188,7 @@ def validate_resharding_after_refresh(node, system_log_follower): @classmethod def get_load_test_data_inventory(cls, column_number: int, big_sstable: bool, - load_and_stream: bool) -> List[TestDataInventory]: + load_and_stream: bool) -> list[TestDataInventory]: if column_number == 1: # Use special schema (one column) for refresh before https://github.com/scylladb/scylla/issues/6617 is fixed if big_sstable: @@ -206,7 +213,7 @@ def get_load_test_data_inventory(cls, column_number: int, big_sstable: bool, def create_keyspace(cls, node, keyspace_name: str = "keyspace1", strategy: str = 'NetworkTopologyStrategy', replication_factor: int = 1): node.run_cqlsh( - "CREATE KEYSPACE %s WITH replication = {'class': '%s', 'replication_factor': %s}" % ( + "CREATE KEYSPACE {} WITH replication = {{'class': '{}', 'replication_factor': {}}}".format( keyspace_name, strategy, replication_factor)) @classmethod diff --git a/sdcm/utils/sstable/sstable_utils.py b/sdcm/utils/sstable/sstable_utils.py index e741f9f4462..1abe1f8d3d8 100644 --- a/sdcm/utils/sstable/sstable_utils.py +++ b/sdcm/utils/sstable/sstable_utils.py @@ -3,9 +3,9 @@ import logging import random +from sdcm.exceptions import SstablesNotFound from sdcm.paths import SCYLLA_YAML_PATH from sdcm.utils.version_utils import ComparableScyllaVersion -from sdcm.exceptions import SstablesNotFound class NonDeletedTombstonesFound(Exception): diff --git a/sdcm/utils/syslogng.py b/sdcm/utils/syslogng.py index 475f4f30315..34e079e94e4 100644 --- a/sdcm/utils/syslogng.py +++ b/sdcm/utils/syslogng.py @@ -11,13 +11,12 @@ # # Copyright (c) 2021 ScyllaDB -import os -import logging import getpass +import logging +import os from functools import cached_property from pwd import getpwnam from tempfile import mkstemp -from typing import Optional from sdcm.utils.docker_utils import ContainerManager @@ -34,11 +33,11 @@ def syslogng_confpath(self) -> str: return generate_syslogng_conf_file() @property - def syslogng_port(self) -> Optional[int]: + def syslogng_port(self) -> int | None: return ContainerManager.get_container_port(self, "syslogng", SYSLOGNG_PORT) @property - def syslogng_log_dir(self) -> Optional[str]: + def syslogng_log_dir(self) -> str | None: # This should work after process that had run container was terminated and another one starts # that is why it is not using variable to store log directory instead return ContainerManager.get_host_volume_path( diff --git a/sdcm/utils/threads_and_processes_alive.py b/sdcm/utils/threads_and_processes_alive.py index e939b76312c..a96e089bc19 100644 --- a/sdcm/utils/threads_and_processes_alive.py +++ b/sdcm/utils/threads_and_processes_alive.py @@ -6,7 +6,6 @@ import traceback from typing import Any - LOGGER = logging.getLogger(__name__) diff --git a/sdcm/utils/toppartition_util.py b/sdcm/utils/toppartition_util.py index 2f5cd36fb12..d715e68c260 100644 --- a/sdcm/utils/toppartition_util.py +++ b/sdcm/utils/toppartition_util.py @@ -1,9 +1,7 @@ -import random import logging +import random import re - -from typing import List, Tuple -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod from collections import OrderedDict LOGGER = logging.getLogger(__name__) @@ -11,7 +9,7 @@ class TopPartitionCmd(ABC): - def __init__(self, ks_cf_list: List[str]): + def __init__(self, ks_cf_list: list[str]): self.ks_cf_list = ks_cf_list self._built_args = {} @@ -85,16 +83,16 @@ def verify_output(self, output: str): toppartition_result = self._parse_toppartitions_output(output) for _sampler in self._built_args['samplers'].split(','): sampler = _sampler.upper() - assert sampler in toppartition_result, "{} sampler not found in result".format(sampler) + assert sampler in toppartition_result, f"{sampler} sampler not found in result" assert toppartition_result[sampler]['toppartitions'] == self._built_args['toppartition'], \ - "Wrong expected and actual top partitions number for {} sampler".format(sampler) + f"Wrong expected and actual top partitions number for {sampler} sampler" assert toppartition_result[sampler]['capacity'] == self._built_args['capacity'], \ - "Wrong expected and actual capacity number for {} sampler".format(sampler) + f"Wrong expected and actual capacity number for {sampler} sampler" assert len(toppartition_result[sampler]['partitions'].keys()) <= int(self._built_args['toppartition']), \ - "Wrong number of requested and expected toppartitions for {} sampler".format(sampler) + f"Wrong number of requested and expected toppartitions for {sampler} sampler" @abstractmethod - def _filter_ks_cf(self) -> List[Tuple[str, str]]: + def _filter_ks_cf(self) -> list[tuple[str, str]]: ... @abstractmethod @@ -119,7 +117,7 @@ def generate_cmd_arg_values(self): 'cf_filters': ','.join([f"{ks}:{cf}" for ks, cf in filtered_ks_cf]) } - def _filter_ks_cf(self) -> List[Tuple[str, str]]: + def _filter_ks_cf(self) -> list[tuple[str, str]]: filter_ks_cf = [] try: for _ in range(random.randint(1, len(self.ks_cf_list))): @@ -128,7 +126,7 @@ def _filter_ks_cf(self) -> List[Tuple[str, str]]: except IndexError as details: LOGGER.error('Keyspace and ColumnFamily are not found %s.', self.ks_cf_list) LOGGER.debug('Error during choosing keyspace and column family %s', details) - raise Exception('Keyspace and ColumnFamily are not found. \n{}'.format(details)) from IndexError + raise Exception(f'Keyspace and ColumnFamily are not found. \n{details}') from IndexError return filter_ks_cf @@ -150,14 +148,14 @@ def generate_cmd_arg_values(self): 'cf': cf, } - def _filter_ks_cf(self) -> List[Tuple[str, str]]: + def _filter_ks_cf(self) -> list[tuple[str, str]]: try: keyspace, table_name = random.choice(self.ks_cf_list).split('.', maxsplit=1) return [(keyspace, table_name)] except IndexError as details: LOGGER.error('Keyspace and ColumnFamily are not found %s.', self.ks_cf_list) LOGGER.debug('Error during choosing keyspace and column family %s', details) - raise Exception('Keyspace and ColumnFamily are not found. \n{}'.format(details)) from IndexError + raise Exception(f'Keyspace and ColumnFamily are not found. \n{details}') from IndexError def get_cmd_args(self) -> str: return "{ks} {cf} {duration} -s {capacity} -k {toppartition} -a {samplers}".format(**self._built_args) diff --git a/sdcm/utils/uda.py b/sdcm/utils/uda.py index 506c43f6e97..c777860e57a 100644 --- a/sdcm/utils/uda.py +++ b/sdcm/utils/uda.py @@ -11,8 +11,8 @@ # # Copyright (c) 2022 ScyllaDB from __future__ import annotations + from pathlib import Path -from typing import Optional import yaml from pydantic import BaseModel @@ -63,7 +63,7 @@ class UDA(BaseModel): args: str return_type: str accumulator_udf: UDF - reduce_udf: Optional[UDF] + reduce_udf: UDF | None final_udf: UDF initial_condition: str @@ -82,9 +82,9 @@ def get_create_query_string(self, ks: str) -> str: def from_yaml(cls, uda_yaml_file_path: str) -> UDA: with Path(uda_yaml_file_path).open(mode="r", encoding="utf-8") as uda_yaml: input_yaml = yaml.safe_load(uda_yaml) - accumulator_udf = UDFS.get((input_yaml.get("accumulator_udf_name", None))) - reduce_udf = UDFS.get((input_yaml.get("reduce_udf_name", None))) - final_udf = UDFS.get((input_yaml.get("final_udf_name", None))) + accumulator_udf = UDFS.get(input_yaml.get("accumulator_udf_name", None)) + reduce_udf = UDFS.get(input_yaml.get("reduce_udf_name", None)) + final_udf = UDFS.get(input_yaml.get("final_udf_name", None)) return UDA( accumulator_udf=accumulator_udf, reduce_udf=reduce_udf, diff --git a/sdcm/utils/udf.py b/sdcm/utils/udf.py index 74a5de3d2f1..7092b21f679 100644 --- a/sdcm/utils/udf.py +++ b/sdcm/utils/udf.py @@ -11,6 +11,7 @@ # # Copyright (c) 2022 ScyllaDB from __future__ import annotations + from pathlib import Path from typing import Literal diff --git a/sdcm/utils/user_profile.py b/sdcm/utils/user_profile.py index 7fdaed29f6e..9d35b1f8a0c 100644 --- a/sdcm/utils/user_profile.py +++ b/sdcm/utils/user_profile.py @@ -12,7 +12,7 @@ # Copyright (c) 2023 ScyllaDB """ This module keeps utilities that help to extract schema definition and stress commands from user profile """ -from __future__ import absolute_import, annotations +from __future__ import annotations import re diff --git a/sdcm/utils/version_utils.py b/sdcm/utils/version_utils.py index 4726b7a5e83..a95568bc138 100644 --- a/sdcm/utils/version_utils.py +++ b/sdcm/utils/version_utils.py @@ -11,29 +11,29 @@ # # Copyright (c) 2020 ScyllaDB -import re -import os import logging +import os +import re +from collections import defaultdict, namedtuple from enum import Enum, auto -from string import Template -from typing import List, Optional, Literal -from collections import namedtuple, defaultdict -from urllib.parse import urlparse from functools import lru_cache, wraps from itertools import count +from string import Template +from typing import Literal +from urllib.parse import urlparse -import yaml import boto3 -import requests import dateutil.parser -from mypy_boto3_s3 import S3Client +import requests +import yaml from botocore import UNSIGNED from botocore.client import Config +from mypy_boto3_s3 import S3Client from repodataParser.RepoParser import Parser from sdcm.remote import LOCALRUNNER -from sdcm.utils.common import ParallelObject, DEFAULT_AWS_REGION from sdcm.sct_events.system import ScyllaRepoEvent +from sdcm.utils.common import DEFAULT_AWS_REGION, ParallelObject from sdcm.utils.decorators import retrying # Examples of ScyllaDB version strings: @@ -406,7 +406,7 @@ def is_enterprise(scylla_version): return bool(re.search(r"^20[0-9]{2}.*", scylla_version)) -def assume_version(params: dict[str], scylla_version: Optional[str] = None) -> tuple[bool, str]: +def assume_version(params: dict[str], scylla_version: str | None = None) -> tuple[bool, str]: # Try to get the major version from the branch name, it will only be used when scylla_version isn't assigned. # It can be switched to RELEASE_BRANCH from upstream job git_branch = os.environ.get('GIT_BRANCH') # origin/branch-4.5 @@ -438,7 +438,7 @@ def get_gemini_version(output: str): return None -def get_node_supported_sstable_versions(node_system_log) -> List[str]: +def get_node_supported_sstable_versions(node_system_log) -> list[str]: output = [] with open(node_system_log, encoding="utf-8") as file: for line in file.readlines(): @@ -501,7 +501,7 @@ def get_docker_image_by_version(scylla_version: str): return default_image -def _list_repo_file_etag(s3_client: S3Client, prefix: str) -> Optional[dict]: +def _list_repo_file_etag(s3_client: S3Client, prefix: str) -> str | None: repo_file = s3_client.list_objects_v2(Bucket=SCYLLA_REPO_BUCKET, Prefix=prefix) if repo_file["KeyCount"] != 1: LOGGER.debug("No such file `%s' in %s bucket", prefix, SCYLLA_REPO_BUCKET) @@ -696,7 +696,7 @@ def __call__(self, func): if (func.__name__, func.__code__.co_filename) not in self.VERSIONS: self.VERSIONS[(func.__name__, func.__code__.co_filename)] = {} for min_v, max_v in self.min_max_version_pairs: - scylla_type = "enterprise" if any((is_enterprise(v) for v in (min_v, max_v) if v)) else "oss" + scylla_type = "enterprise" if any(is_enterprise(v) for v in (min_v, max_v) if v) else "oss" min_v = min_v or ("3.0.0" if scylla_type == "oss" else "2019.1.rc0") # noqa: PLW2901 max_v = max_v or ("99.99.99" if scylla_type == "oss" else "2099.99.99") # noqa: PLW2901 if max_v.count(".") == 1: @@ -748,7 +748,7 @@ def get_relocatable_pkg_url(scylla_version: str) -> str: _S3_SCYLLA_REPOS_CACHE = defaultdict(dict) -def get_s3_scylla_repos_mapping(dist_type='centos', dist_version=None): +def get_s3_scylla_repos_mapping(dist_type: Literal["centos", "ubuntu", "debian"] = 'centos', dist_version: str | None = None): """ get the mapping from version prefixes to rpm .repo or deb .list files locations @@ -778,28 +778,28 @@ def get_s3_scylla_repos_mapping(dist_type='centos', dist_version=None): path=repo_file['Key']) elif dist_type in ('ubuntu', 'debian'): - response = s3_client.list_objects(Bucket=bucket, Prefix='deb/{}/'.format(dist_type), Delimiter='/') + response = s3_client.list_objects(Bucket=bucket, Prefix=f'deb/{dist_type}/', Delimiter='/') for repo_file in response['Contents']: filename = os.path.basename(repo_file['Key']) # only if path look like 'deb/debian/scylla-3.0-jessie.list', we deem it formal one repo_regex = re.compile(r'\d+\.\d\.list') if filename.startswith('scylla-') and ( - filename.endswith('-{}.list'.format(dist_version)) or + filename.endswith(f'-{dist_version}.list') or repo_regex.search(filename)): version_prefix = \ - filename.replace('-{}.list'.format(dist_version), '').replace('.list', '').split('-')[-1] + filename.replace(f'-{dist_version}.list', '').replace('.list', '').split('-')[-1] _S3_SCYLLA_REPOS_CACHE[( dist_type, dist_version)][version_prefix] = "https://s3.amazonaws.com/{bucket}/{path}".format( bucket=bucket, path=repo_file['Key']) else: - raise NotImplementedError("[{}] is not yet supported".format(dist_type)) + raise NotImplementedError(f"[{dist_type}] is not yet supported") return _S3_SCYLLA_REPOS_CACHE[(dist_type, dist_version)] -def find_scylla_repo(scylla_version, dist_type='centos', dist_version=None): +def find_scylla_repo(scylla_version, dist_type: Literal["centos", "ubuntu", "debian"] = 'centos', dist_version: str | None = None): """ Get a repo/list of scylla, based on scylla version match @@ -827,7 +827,7 @@ def find_scylla_repo(scylla_version, dist_type='centos', dist_version=None): def get_branched_repo(scylla_version: str, dist_type: Literal["centos", "ubuntu", "debian"] = "centos", - bucket: str = "downloads.scylladb.com") -> Optional[str]: + bucket: str = "downloads.scylladb.com") -> str | None: """ Get a repo/list of scylla, based on scylla version match diff --git a/sdcm/wait.py b/sdcm/wait.py index 582722d4131..27c0f072261 100644 --- a/sdcm/wait.py +++ b/sdcm/wait.py @@ -14,15 +14,15 @@ """ Wait functions appropriate for tests that have high timing variance. """ -import time import logging +import time +from collections.abc import Callable from contextlib import contextmanager from typing import TypeVar, cast -from collections.abc import Callable import tenacity -from sdcm.exceptions import WaitForTimeoutError, ExitByEventError +from sdcm.exceptions import ExitByEventError, WaitForTimeoutError LOGGER = logging.getLogger("sdcm.wait") diff --git a/sdcm/ycsb_thread.py b/sdcm/ycsb_thread.py index 0a7512e1407..515ed5c5ccb 100644 --- a/sdcm/ycsb_thread.py +++ b/sdcm/ycsb_thread.py @@ -11,22 +11,21 @@ # # Copyright (c) 2020 ScyllaDB +import logging import os import re +import tempfile import time import uuid -import tempfile -import logging from textwrap import dedent from sdcm.prometheus import nemesis_metrics_obj -from sdcm.sct_events.loaders import YcsbStressEvent from sdcm.remote import FailuresWatcher +from sdcm.sct_events.loaders import YcsbStressEvent +from sdcm.stress.base import DockerBasedStressThread, format_stress_cmd_error from sdcm.utils import alternator -from sdcm.utils.common import FileFollowerThread +from sdcm.utils.common import FileFollowerThread, generate_random_string from sdcm.utils.docker_remote import RemoteDocker -from sdcm.utils.common import generate_random_string -from sdcm.stress.base import format_stress_cmd_error, DockerBasedStressThread LOGGER = logging.getLogger(__name__) @@ -129,7 +128,7 @@ def copy_template(self, docker): dynamodb_teample = dedent(''' measurementtype=hdrhistogram dynamodb.awsCredentialsFile = /tmp/aws_empty_file - dynamodb.endpoint = http://{0}:{1} + dynamodb.endpoint = http://{}:{} dynamodb.connectMax = 200 requestdistribution = uniform dynamodb.consistentReads = true @@ -238,8 +237,7 @@ def _run_stress(self, loader, loader_idx, cpu_idx): if not os.path.exists(loader.logdir): os.makedirs(loader.logdir, exist_ok=True) - log_file_name = os.path.join(loader.logdir, 'ycsb-l%s-c%s-%s.log' % - (loader_idx, cpu_idx, uuid.uuid4())) + log_file_name = os.path.join(loader.logdir, f'ycsb-l{loader_idx}-c{cpu_idx}-{uuid.uuid4()}.log') LOGGER.debug('ycsb-stress local log: %s', log_file_name) def raise_event_callback(sentinel, line): # pylint: disable=unused-argument @@ -248,7 +246,7 @@ def raise_event_callback(sentinel, line): # pylint: disable=unused-argument LOGGER.debug("running: %s", stress_cmd) - node_cmd = 'cd /YCSB && {}'.format(stress_cmd) + node_cmd = f'cd /YCSB && {stress_cmd}' YcsbStressEvent.start(node=loader, stress_cmd=stress_cmd).publish() diff --git a/sla_per_user_system_test.py b/sla_per_user_system_test.py index 2151494d206..a4f6bf7af26 100644 --- a/sla_per_user_system_test.py +++ b/sla_per_user_system_test.py @@ -21,7 +21,7 @@ from sdcm.es import ES from sdcm.sct_events import Severity from sdcm.sct_events.workload_prioritisation import WorkloadPrioritisationEvent -from test_lib.sla import ServiceLevel, Role, User +from test_lib.sla import Role, ServiceLevel, User # pylint: disable=too-many-public-methods @@ -101,14 +101,14 @@ def validate_scheduler_runtime(self, start_time, end_time, read_users, expected_ for node_ip in self.db_cluster.get_node_private_ips(): # Temporary solution scheduler_shares = self.prometheus_stats.get_scylla_scheduler_shares_per_sla(start_time, end_time, node_ip) - self.log.debug('SCHEDULERS SHARES FROM PROMETHEUS: {}'.format(scheduler_shares)) + self.log.debug(f'SCHEDULERS SHARES FROM PROMETHEUS: {scheduler_shares}') # this default scheduler that is not under test - ignore it if 'sl:default' in scheduler_shares: scheduler_shares.pop('sl:default') test_users_to_sg = self.role_to_scheduler_group(test_users=roles_with_shares, scheduler_shares=scheduler_shares) - self.log.debug('ROLE - SERVICE LEVEL - SCHEDULER: {}'.format(test_users_to_sg)) + self.log.debug(f'ROLE - SERVICE LEVEL - SCHEDULER: {test_users_to_sg}') # End Temporary solution # Query 'scylla_scheduler_runtime_ms' from prometheus. If no data returned, try to increase the step time @@ -137,7 +137,7 @@ def validate_scheduler_runtime(self, start_time, end_time, read_users, expected_ len(shards_time_per_sla[node_ip][val[1]]) else: runtime_per_role[rolename] = 0 - self.log.debug('RUN TIME PER ROLE: {}'.format(runtime_per_role)) + self.log.debug(f'RUN TIME PER ROLE: {runtime_per_role}') actual_shares_ratio = self.calculate_metrics_ratio_per_user(two_users_list=read_users, metrics=runtime_per_role) self.validate_deviation(expected_ratio=expected_ratio, actual_ratio=actual_shares_ratio, @@ -833,7 +833,7 @@ def _compare_workloads_c_s_metrics(self, workloads_queue: list) -> dict: # noqa: PLR2004 assert len(workloads_results) == 2, \ - "Expected workload_results length to be 2, got: %s. workload_results: %s" % ( # noqa: PLR2004 + "Expected workload_results length to be 2, got: {}. workload_results: {}".format( # noqa: PLR2004 len(workloads_results), workloads_results) comparison_results = {} try: @@ -902,7 +902,7 @@ def get_email_data(self): def get_test_status(self) -> str: if self._comparison_results: try: - if all((item["within_margin"] for item in self._comparison_results.values())): + if all(item["within_margin"] for item in self._comparison_results.values()): return "SUCCESS" else: return "FAILED" @@ -943,8 +943,8 @@ def __get_stat_for_user(read, user_name): grafana_screenshots = grafana_dataset.get('screenshots', []) grafana_snapshots = grafana_dataset.get('snapshots', []) - self.log.debug('GRAFANA SCREENSHOTS: {}'.format(grafana_screenshots)) - self.log.debug('GRAFANA SNAPSHOTS: {}'.format(grafana_snapshots)) + self.log.debug(f'GRAFANA SCREENSHOTS: {grafana_screenshots}') + self.log.debug(f'GRAFANA SNAPSHOTS: {grafana_snapshots}') # Compare latency of two runs self.log.debug('Test results:\n---------------------\n') @@ -961,9 +961,9 @@ def __get_stat_for_user(read, user_name): result_print_str = '\nTest results:\n---------------------\n' result_print_str += '\nWorkload | Latency 99%' result_print_str += '\n========================= | =================' - result_print_str += '\nLatency only | {}'.format(latency_99_latency_workload) - result_print_str += '\nLatency and throughput | {}'.format(latency_99_mixed_workload) + result_print_str += f'\nLatency only | {latency_99_latency_workload}' + result_print_str += f'\nLatency and throughput | {latency_99_mixed_workload}' result_print_str += '\n------------------------- | -----------------' - result_print_str += '\nLatency 99 is {} in {}%'.format(latency_change, deviation) + result_print_str += f'\nLatency 99 is {latency_change} in {deviation}%' return latency_99_latency_workload, latency_99_mixed_workload, result_print_str diff --git a/stop_compaction_test.py b/stop_compaction_test.py index 6774c2c0739..1dc3bb527dd 100644 --- a/stop_compaction_test.py +++ b/stop_compaction_test.py @@ -15,19 +15,24 @@ import functools import logging import re +from collections.abc import Callable from functools import partial -from typing import Callable from sdcm.cluster import BaseNode -from sdcm.nemesis import StartStopMajorCompaction, StartStopScrubCompaction, StartStopCleanupCompaction, \ - StartStopValidationCompaction +from sdcm.nemesis import ( + StartStopCleanupCompaction, + StartStopMajorCompaction, + StartStopScrubCompaction, + StartStopValidationCompaction, +) from sdcm.rest.compaction_manager_client import CompactionManagerClient from sdcm.rest.storage_service_client import StorageServiceClient from sdcm.sct_events.group_common_events import ignore_compaction_stopped_exceptions from sdcm.send_email import FunctionalEmailReporter from sdcm.tester import ClusterTester from sdcm.utils.common import ParallelObject -from sdcm.utils.compaction_ops import CompactionOps, COMPACTION_TYPES +from sdcm.utils.compaction_ops import COMPACTION_TYPES, CompactionOps + LOGGER = logging.getLogger(__name__) diff --git a/test_add_remove_ldap_role_permission.py b/test_add_remove_ldap_role_permission.py index a20eff6a302..4b5bc70cc7e 100644 --- a/test_add_remove_ldap_role_permission.py +++ b/test_add_remove_ldap_role_permission.py @@ -6,7 +6,7 @@ from longevity_test import LongevityTest from sdcm.cluster import BaseNode from sdcm.sct_events.system import InfoEvent -from sdcm.utils.ldap import LdapServerType, LDAP_USERS, LDAP_PASSWORD, LdapUtilsMixin +from sdcm.utils.ldap import LDAP_PASSWORD, LDAP_USERS, LdapServerType, LdapUtilsMixin class AddRemoveLdapRolePermissionTest(LongevityTest, LdapUtilsMixin): diff --git a/test_lib/compaction.py b/test_lib/compaction.py index da623fd6fdd..d15d1bccdc2 100644 --- a/test_lib/compaction.py +++ b/test_lib/compaction.py @@ -27,7 +27,7 @@ def from_str(cls, output_str): try: return CompactionStrategy[CompactionStrategy(output_str).name] except AttributeError as attr_err: - err_msg = "Could not recognize compaction strategy value: {} - {}".format(output_str, attr_err) + err_msg = f"Could not recognize compaction strategy value: {output_str} - {attr_err}" raise ValueError(err_msg) from attr_err @@ -81,7 +81,7 @@ def get_compaction_strategy(node, keyspace, table): compaction = CompactionStrategy.from_str(output_str=dict_compaction_values['class']) break - LOGGER.debug("Query result for {}.{} compaction is: {}".format(keyspace, table, compaction)) + LOGGER.debug(f"Query result for {keyspace}.{table} compaction is: {compaction}") return compaction diff --git a/test_lib/cql_types.py b/test_lib/cql_types.py index d2339835fcb..196aad0e92d 100644 --- a/test_lib/cql_types.py +++ b/test_lib/cql_types.py @@ -1,6 +1,5 @@ import random - __ALL__ = ['CQLTypeBuilder', 'ALL_COLUMN_TYPES', 'NOT_EMBEDDABLE_COLUMN_TYPES', 'COLLECTION_COLUMN_TYPES'] diff --git a/test_lib/sla.py b/test_lib/sla.py index 3eb1ee34a09..4284b845e85 100644 --- a/test_lib/sla.py +++ b/test_lib/sla.py @@ -3,12 +3,13 @@ import logging from dataclasses import dataclass, field, fields -from typing import Optional from sdcm.utils.decorators import retrying -from sdcm.utils.loader_utils import (STRESS_ROLE_NAME_TEMPLATE, - STRESS_ROLE_PASSWORD_TEMPLATE, - SERVICE_LEVEL_NAME_TEMPLATE) +from sdcm.utils.loader_utils import ( + SERVICE_LEVEL_NAME_TEMPLATE, + STRESS_ROLE_NAME_TEMPLATE, + STRESS_ROLE_PASSWORD_TEMPLATE, +) LOGGER = logging.getLogger(__name__) @@ -65,7 +66,7 @@ class ServiceLevel: # pylint: disable=too-many-arguments def __init__(self, session, # noqa: PLR0913 name: str, - shares: Optional[int] = 1000, + shares: int | None = 1000, timeout: str = None, workload_type: str = None): self.session = session @@ -133,7 +134,7 @@ def __eq__(self, other): return (self.name == other.name) and self._sl_attributes == other._sl_attributes def __repr__(self): - return "%s: name: %s, attributes: %s" % (self.__class__.__name__, self.name, self._sl_attributes) + return f"{self.__class__.__name__}: name: {self.name}, attributes: {self._sl_attributes}" def create(self, if_not_exists=True) -> ServiceLevel: query = "CREATE SERVICE_LEVEL{if_not_exists} {service_level_name}{query_attributes}"\ @@ -149,9 +150,7 @@ def create(self, if_not_exists=True) -> ServiceLevel: def alter(self, new_shares: int = None, new_timeout: str = None, new_workload_type: str = None): sla = ServiceLevelAttributes(shares=new_shares, timeout=new_timeout, workload_type=new_workload_type) - query = 'ALTER SERVICE_LEVEL {service_level_name} {query_string}'\ - .format(service_level_name=self.name, - query_string=sla.query_string) + query = f'ALTER SERVICE_LEVEL {self.name} {sla.query_string}' if self.verbose: LOGGER.debug('Change service level query: %s', query) self.session.execute(query) @@ -170,7 +169,7 @@ def drop(self, if_exists=True): self.created = False def list_service_level(self) -> ServiceLevel | None: - query = 'LIST SERVICE_LEVEL {}'.format(self.name) + query = f'LIST SERVICE_LEVEL {self.name}' if self.verbose: LOGGER.debug('List service level query: %s', query) res = self.session.execute(query).all() @@ -277,9 +276,6 @@ def attach_service_level(self, service_level: ServiceLevel): self._attached_scheduler_group_name = service_level.scheduler_group_name def detach_service_level(self): - """ - :param auth_name: it may be role name or user name - """ query = f'DETACH SERVICE_LEVEL FROM {self.name}' if self.verbose: LOGGER.debug('Detach service level query: %s', query) @@ -348,11 +344,11 @@ def create(self, if_not_exists=True) -> Role: if hasattr(self, opt): value = getattr(self, opt) if value: - role_options[opt.replace('_dict', '')] = '\'{}\''.format(value) \ + role_options[opt.replace('_dict', '')] = f'\'{value}\'' \ if opt == 'password' else value - role_options_str = ' AND '.join(['{} = {}'.format(opt, val) for opt, val in role_options.items()]) + role_options_str = ' AND '.join([f'{opt} = {val}' for opt, val in role_options.items()]) if role_options_str: - role_options_str = ' WITH {}'.format(role_options_str) + role_options_str = f' WITH {role_options_str}' query = f"CREATE ROLE{' IF NOT EXISTS' if if_not_exists else ''} {self.name}{role_options_str}" if self.verbose: @@ -408,7 +404,7 @@ def create(self) -> User: superuser='' if self.superuser is None else ' SUPERUSER' if self.superuser else ' NOSUPERUSER') if user_options_str: - user_options_str = ' WITH {}'.format(user_options_str) + user_options_str = f' WITH {user_options_str}' query = f'CREATE USER {self.name}{user_options_str}' if self.verbose: diff --git a/test_lib/utils.py b/test_lib/utils.py index 064d3dc1961..5c9462b4658 100644 --- a/test_lib/utils.py +++ b/test_lib/utils.py @@ -1,6 +1,6 @@ -import typing -import functools import collections +import functools +import typing class __DEFAULT__: # pylint: disable=invalid-name,too-few-public-methods @@ -12,7 +12,7 @@ class __DEFAULT2__: # pylint: disable=invalid-name,too-few-public-methods def get_data_by_path( - data: typing.Union[list, dict, object, typing.Type[object]], + data: list | dict | object | type[object], data_path: str, default: typing.Any = __DEFAULT__): """ @@ -50,7 +50,7 @@ def get_data_by_path( def get_class_by_path( - cls: typing.Type[object], + cls: type[object], class_path: str, default: typing.Any = __DEFAULT__): """ @@ -78,7 +78,7 @@ def get_class_by_path( return current -GroupByType = typing.Dict[typing.Any, typing.Union['GroupByType', 'MagicList']] +GroupByType = dict[typing.Any, 'GroupByType | MagicList'] class MagicList(list): diff --git a/throughput_limit_test.py b/throughput_limit_test.py index 7e9b926990c..895c03035e2 100644 --- a/throughput_limit_test.py +++ b/throughput_limit_test.py @@ -11,8 +11,9 @@ # # Copyright (c) 2022 ScyllaDB import logging -from dataclasses import dataclass import time +from dataclasses import dataclass + import yaml from sdcm.cluster import BaseNode @@ -21,7 +22,6 @@ from sdcm.stress_thread import CassandraStressThread from sdcm.tester import ClusterTester - LOGGER = logging.getLogger(__name__) diff --git a/uda_udf_test.py b/uda_udf_test.py index 87344a72eca..3e0b6f68640 100644 --- a/uda_udf_test.py +++ b/uda_udf_test.py @@ -1,11 +1,12 @@ -from typing import NamedTuple, Callable +from collections.abc import Callable +from typing import NamedTuple from sdcm.cluster import BaseNode from sdcm.send_email import LongevityEmailReporter from sdcm.stress_thread import CassandraStressThread from sdcm.tester import ClusterTester -from sdcm.utils.udf import UDFS from sdcm.utils.uda import UDAS +from sdcm.utils.udf import UDFS class UDVerification(NamedTuple): @@ -104,7 +105,7 @@ def _verify_udf_functions(self): c7_value = row_result.c7_text assert row_query.verifier_func(c2_value, c3_value, c7_value), \ "Expected row values to not be None, at least one them was. " \ - "c2_value: %s, c3_value: %s, c7 value: %s" % (c2_value, c3_value, c7_value) + f"c2_value: {c2_value}, c3_value: {c3_value}, c7 value: {c7_value}" for verification in verifications: self.log.info("Running UDF verification: %s; query: %s", verification.name, verification.query) @@ -126,5 +127,5 @@ def _verify_uda_aggregates(self): verification_query_result = session.execute(uda_verification.query).one() assert uda_verification.verifier_func(verification_query_result, avg_result), \ - "UDA verifivation failed. UDA result: %s, Builtin AVG result: %s" % (verification_query_result, avg_result) + f"UDA verifivation failed. UDA result: {verification_query_result}, Builtin AVG result: {avg_result}" self.log.info("Finished running UDA verifications.") diff --git a/unit_tests/conftest.py b/unit_tests/conftest.py index 4f39d2c8af6..e8a9a41f587 100644 --- a/unit_tests/conftest.py +++ b/unit_tests/conftest.py @@ -11,25 +11,23 @@ # # Copyright (c) 2020 ScyllaDB -import os -import logging import collections +import logging +import os from pathlib import Path import pytest -from sdcm import wait, sct_config +from sdcm import sct_config, wait from sdcm.cluster import BaseNode from sdcm.prometheus import start_metrics_server from sdcm.provision import provisioner_factory from sdcm.remote import RemoteCmdRunnerBase from sdcm.sct_events.continuous_event import ContinuousEventsRegistry from sdcm.sct_provision import region_definition_builder -from sdcm.utils.docker_remote import RemoteDocker from sdcm.utils.common import update_certificates - +from sdcm.utils.docker_remote import RemoteDocker from unit_tests.dummy_remote import LocalNode, LocalScyllaClusterDummy - from unit_tests.lib.events_utils import EventsUtilsMixin from unit_tests.lib.fake_provisioner import FakeProvisioner from unit_tests.lib.fake_region_definition_builder import FakeDefinitionBuilder diff --git a/unit_tests/dummy_remote.py b/unit_tests/dummy_remote.py index a543540d7da..598322f68f1 100644 --- a/unit_tests/dummy_remote.py +++ b/unit_tests/dummy_remote.py @@ -13,12 +13,12 @@ # pylint: disable=too-few-public-methods +import logging import os import shutil -import logging +from sdcm.cluster import BaseCluster, BaseNode, BaseScyllaCluster from sdcm.remote import LocalCmdRunner -from sdcm.cluster import BaseNode, BaseCluster, BaseScyllaCluster class DummyOutput: diff --git a/unit_tests/lib/data_pickle.py b/unit_tests/lib/data_pickle.py index f40ca7c7d36..5fbd641e98c 100644 --- a/unit_tests/lib/data_pickle.py +++ b/unit_tests/lib/data_pickle.py @@ -11,9 +11,9 @@ # # Copyright (c) 2020 ScyllaDB -from typing import Optional, Any, Dict -import json import enum +import json +from typing import Any # pylint: disable=too-few-public-methods @@ -88,7 +88,7 @@ def import_and_return(path): return lib @classmethod - def _get_attrs_by_class(cls, instance_class: str, attr_type=None) -> Optional[Dict[str, Any]]: + def _get_attrs_by_class(cls, instance_class: str, attr_type=None) -> dict[str, Any] | None: info = cls.class_info.get(instance_class, None) if info is None: return None @@ -132,7 +132,7 @@ def _to_data_list(cls, obj: list) -> list: @classmethod def to_data(cls, obj): - if isinstance(obj, (str, type(None), int, float)): + if isinstance(obj, str | type(None) | int | float): return obj if isinstance(obj, dict): return cls._to_data_dict(obj) @@ -146,7 +146,7 @@ def to_data(cls, obj): @classmethod def from_data(cls, data): - if isinstance(data, (str, type(None), int, float)): + if isinstance(data, str | type(None) | int | float): return data if isinstance(data, dict): return cls._from_data_dict(data) diff --git a/unit_tests/lib/events_utils.py b/unit_tests/lib/events_utils.py index 75e53633e88..f1813b8575d 100644 --- a/unit_tests/lib/events_utils.py +++ b/unit_tests/lib/events_utils.py @@ -11,17 +11,24 @@ # # Copyright (c) 2020 ScyllaDB -import time import shutil import tempfile +import time import unittest.mock from contextlib import contextmanager -from sdcm.sct_events.setup import EVENTS_DEVICE_START_DELAY, start_events_device, stop_events_device -from sdcm.sct_events.events_device import start_events_main_device, get_events_main_device -from sdcm.sct_events.file_logger import get_events_logger -from sdcm.sct_events.events_processes import EventsProcessesRegistry from sdcm.sct_events.event_counter import get_events_counter +from sdcm.sct_events.events_device import ( + get_events_main_device, + start_events_main_device, +) +from sdcm.sct_events.events_processes import EventsProcessesRegistry +from sdcm.sct_events.file_logger import get_events_logger +from sdcm.sct_events.setup import ( + EVENTS_DEVICE_START_DELAY, + start_events_device, + stop_events_device, +) class EventsUtilsMixin: diff --git a/unit_tests/lib/fake_provisioner.py b/unit_tests/lib/fake_provisioner.py index 028a40085c7..aa4c5317ee4 100644 --- a/unit_tests/lib/fake_provisioner.py +++ b/unit_tests/lib/fake_provisioner.py @@ -12,11 +12,15 @@ # Copyright (c) 2022 ScyllaDB import datetime import subprocess -from typing import List, Dict from invoke import Result -from sdcm.provision.provisioner import Provisioner, VmInstance, InstanceDefinition, PricingModel +from sdcm.provision.provisioner import ( + InstanceDefinition, + PricingModel, + Provisioner, + VmInstance, +) class FakeProvisioner(Provisioner): @@ -33,7 +37,7 @@ def __new__(cls, test_id: str, region: str, availability_zone: str, **_) -> Prov def __init__(self, test_id: str, region: str, availability_zone: str, **_) -> None: super().__init__(test_id, region, availability_zone) - self._instances: Dict[str, VmInstance] = getattr(self, "_instances", {}) + self._instances: dict[str, VmInstance] = getattr(self, "_instances", {}) def get_or_create_instance(self, definition: InstanceDefinition, @@ -48,21 +52,21 @@ def get_or_create_instance(self, private_ip_address='10.10.10.10', tags=definition.tags, pricing_model=pricing_model, image=definition.image_id, - creation_time=datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc), + creation_time=datetime.datetime.utcnow().replace(tzinfo=datetime.UTC), _provisioner=self) self._instances[definition.name] = v_m return v_m def get_or_create_instances(self, - definitions: List[InstanceDefinition], + definitions: list[InstanceDefinition], pricing_model: PricingModel = PricingModel.SPOT - ) -> List[VmInstance]: + ) -> list[VmInstance]: return [self.get_or_create_instance(definition, pricing_model) for definition in definitions] - def list_instances(self) -> List[VmInstance]: + def list_instances(self) -> list[VmInstance]: return list(self._instances.values()) - def add_instance_tags(self, name: str, tags: Dict[str, str]) -> None: + def add_instance_tags(self, name: str, tags: dict[str, str]) -> None: self._instances[name].tags.update(tags) def terminate_instance(self, name: str, wait: bool = False) -> None: @@ -79,5 +83,5 @@ def run_command(self, name: str, command: str) -> Result: return subprocess.run(command, shell=True, capture_output=True, text=True, check=False) # pylint: disable=subprocess-run-check @classmethod - def discover_regions(cls, test_id: str, **kwargs) -> List[Provisioner]: # pylint: disable=unused-argument + def discover_regions(cls, test_id: str, **kwargs) -> list[Provisioner]: # pylint: disable=unused-argument return list(cls._provisioners.get(test_id, {}).values()) diff --git a/unit_tests/lib/fake_region_definition_builder.py b/unit_tests/lib/fake_region_definition_builder.py index c444bf8223e..e1a5234027a 100644 --- a/unit_tests/lib/fake_region_definition_builder.py +++ b/unit_tests/lib/fake_region_definition_builder.py @@ -10,11 +10,13 @@ # See LICENSE for more details. # # Copyright (c) 2022 ScyllaDB -from typing import Dict from sdcm.keystore import SSHKey from sdcm.sct_provision.common.types import NodeTypeType -from sdcm.sct_provision.region_definition_builder import ConfigParamsMap, DefinitionBuilder +from sdcm.sct_provision.region_definition_builder import ( + ConfigParamsMap, + DefinitionBuilder, +) db_map = ConfigParamsMap(image_id="fake_image_db", type="fake_instance_type_db", @@ -31,7 +33,7 @@ user_name="ami_monitor_user", root_disk_size="root_disk_size_monitor") -mapper: Dict[NodeTypeType, ConfigParamsMap] = {"scylla-db": db_map, +mapper: dict[NodeTypeType, ConfigParamsMap] = {"scylla-db": db_map, "loader": loader_map, "monitor": monitor_map} diff --git a/unit_tests/lib/fake_remoter.py b/unit_tests/lib/fake_remoter.py index 0dea801647e..39d50168034 100644 --- a/unit_tests/lib/fake_remoter.py +++ b/unit_tests/lib/fake_remoter.py @@ -12,7 +12,7 @@ # Copyright (c) 2022 ScyllaDB import re -from typing import Dict, Pattern +from re import Pattern from invoke import Result @@ -22,7 +22,7 @@ class FakeRemoter(RemoteCmdRunnerBase): """Fake remoter that responds to commands as described in `result_map` class attribute.""" - result_map: Dict[Pattern, Result] = {} + result_map: dict[Pattern, Result] = {} def run(self, # pylint: disable=too-many-arguments # noqa: PLR0913 cmd: str, diff --git a/unit_tests/lib/mock_remoter.py b/unit_tests/lib/mock_remoter.py index b322bf29b3d..0afab3169e9 100644 --- a/unit_tests/lib/mock_remoter.py +++ b/unit_tests/lib/mock_remoter.py @@ -11,8 +11,7 @@ # # Copyright (c) 2020 ScyllaDB -from typing import Optional, List, Union -from sdcm.remote.remote_base import StreamWatcher, Result, CommandRunner +from sdcm.remote.remote_base import CommandRunner, Result, StreamWatcher from unit_tests.lib.data_pickle import Pickler @@ -30,7 +29,7 @@ class MockRemoter: """ user = 'scylla-test' - def __init__(self, responses: Union[dict, str] = None): + def __init__(self, responses: dict | str = None): self.command_counter = {} if isinstance(responses, str): self.responses = Pickler.load_from_file(responses) @@ -48,9 +47,9 @@ def _process_response(self, response): # pylint: disable=no-self-use return None # pylint: disable=too-many-arguments,unused-argument - def run(self, cmd: str, timeout: Optional[float] = None, # noqa: PLR0913 + def run(self, cmd: str, timeout: float | None = None, # noqa: PLR0913 ignore_status: bool = False, verbose: bool = True, new_session: bool = False, - log_file: Optional[str] = None, retry: int = 1, watchers: Optional[List[StreamWatcher]] = None, + log_file: str | None = None, retry: int = 1, watchers: List[StreamWatcher] | None = None, change_context: bool = False) -> Result: response = self.responses.get(cmd) if response is None: diff --git a/unit_tests/lib/remoter_recorder.py b/unit_tests/lib/remoter_recorder.py index fd56df4834f..1e933110fbf 100644 --- a/unit_tests/lib/remoter_recorder.py +++ b/unit_tests/lib/remoter_recorder.py @@ -12,8 +12,8 @@ # Copyright (c) 2020 ScyllaDB import json -from typing import Optional, List -from sdcm.remote.remote_base import StreamWatcher, Result + +from sdcm.remote.remote_base import Result, StreamWatcher from sdcm.remote.remote_cmd_runner import RemoteCmdRunner from unit_tests.lib.data_pickle import Pickler @@ -29,9 +29,9 @@ class RemoterRecorder(RemoteCmdRunner): """ responses = {} - def run(self, cmd: str, timeout: Optional[float] = None, # pylint: disable=too-many-arguments # noqa: PLR0913 + def run(self, cmd: str, timeout: float | None = None, # pylint: disable=too-many-arguments # noqa: PLR0913 ignore_status: bool = False, verbose: bool = True, new_session: bool = False, - log_file: Optional[str] = None, retry: int = 1, watchers: Optional[List[StreamWatcher]] = None, + log_file: str | None = None, retry: int = 1, watchers: List[StreamWatcher] | None = None, change_context: bool = False) -> Result: try: output = super().run(cmd, timeout, ignore_status, verbose, new_session, log_file, retry, watchers, change_context) diff --git a/unit_tests/lib/test_profiler/lib.py b/unit_tests/lib/test_profiler/lib.py index 81a418ac75c..515d99e678d 100644 --- a/unit_tests/lib/test_profiler/lib.py +++ b/unit_tests/lib/test_profiler/lib.py @@ -12,14 +12,13 @@ # Copyright (c) 2020 ScyllaDB # -import time -import threading -from threading import Thread as LibThread -from threading import Thread # pylint: disable=reimported import multiprocessing -from multiprocessing import Process as LibProcess +import threading +import time from multiprocessing import Process # pylint: disable=reimported - +from multiprocessing import Process as LibProcess +from threading import Thread # pylint: disable=reimported +from threading import Thread as LibThread __all__ = [ 'LibMultiprocessingProcessCustomClass', 'LibProcessCustomClass', 'LibMultiprocessingProcessCustomClassWithRun', diff --git a/unit_tests/lib/test_profiler/lib2.py b/unit_tests/lib/test_profiler/lib2.py index 9f446114c7b..857c3779ae0 100644 --- a/unit_tests/lib/test_profiler/lib2.py +++ b/unit_tests/lib/test_profiler/lib2.py @@ -13,8 +13,9 @@ # import time -from sdcm.utils.profiler import ProfileableProcess as LibProfileableProcess, ProfileableThread as LibProfileableThread +from sdcm.utils.profiler import ProfileableProcess as LibProfileableProcess +from sdcm.utils.profiler import ProfileableThread as LibProfileableThread __all__ = [ 'LibProfileableProcessCustomClass', 'LibProfileableProcessCustomClassWithRun', diff --git a/unit_tests/provisioner/conftest.py b/unit_tests/provisioner/conftest.py index 73e1102baef..661196c4b78 100644 --- a/unit_tests/provisioner/conftest.py +++ b/unit_tests/provisioner/conftest.py @@ -22,7 +22,9 @@ from sdcm.sct_config import SCTConfiguration from sdcm.test_config import TestConfig from sdcm.utils.azure_utils import AzureService # pylint: disable=import-error -from unit_tests.provisioner.fake_azure_service import FakeAzureService # pylint: disable=import-error +from unit_tests.provisioner.fake_azure_service import ( + FakeAzureService, # pylint: disable=import-error +) @pytest.fixture(scope="session") diff --git a/unit_tests/provisioner/fake_azure_service.py b/unit_tests/provisioner/fake_azure_service.py index 4fdf13de060..26b1fab5608 100644 --- a/unit_tests/provisioner/fake_azure_service.py +++ b/unit_tests/provisioner/fake_azure_service.py @@ -15,14 +15,24 @@ import os import shutil import subprocess - from pathlib import Path -from typing import List, Any, Dict - -from azure.core.exceptions import ResourceNotFoundError, AzureError -from azure.mgmt.network.models import (NetworkSecurityGroup, Subnet, PublicIPAddress, NetworkInterface, VirtualNetwork) +from typing import Any + +from azure.core.exceptions import AzureError, ResourceNotFoundError +from azure.mgmt.compute.models import ( + Image, + InstanceViewStatus, + RunCommandResult, + VirtualMachine, +) +from azure.mgmt.network.models import ( + NetworkInterface, + NetworkSecurityGroup, + PublicIPAddress, + Subnet, + VirtualNetwork, +) from azure.mgmt.resource.resources.models import ResourceGroup -from azure.mgmt.compute.models import VirtualMachine, Image, InstanceViewStatus, RunCommandResult def snake_case_to_camel_case(string): @@ -32,8 +42,8 @@ def snake_case_to_camel_case(string): def dict_keys_to_camel_case(dct): if isinstance(dct, list): - return [dict_keys_to_camel_case(i) if isinstance(i, (dict, list)) else i for i in dct] - return {snake_case_to_camel_case(a): dict_keys_to_camel_case(b) if isinstance(b, (dict, list)) + return [dict_keys_to_camel_case(i) if isinstance(i, dict | list) else i for i in dct] + return {snake_case_to_camel_case(a): dict_keys_to_camel_case(b) if isinstance(b, dict | list) else b for a, b in dct.items()} @@ -62,7 +72,7 @@ class FakeResourceGroups: def __init__(self, path: Path): self.path = path - def create_or_update(self, resource_group_name: str, parameters: Dict[str, Any]) -> ResourceGroup: + def create_or_update(self, resource_group_name: str, parameters: dict[str, Any]) -> ResourceGroup: res_group = { "id": f"/subscriptions/6c268694-47ab-43ab-b306-3c5514bc4112/resourceGroups/{resource_group_name}", "name": resource_group_name, @@ -79,16 +89,16 @@ def create_or_update(self, resource_group_name: str, parameters: Dict[str, Any]) def get(self, name) -> ResourceGroup: try: - with open(self.path / name / "resource_group.json", "r", encoding="utf-8") as file: + with open(self.path / name / "resource_group.json", encoding="utf-8") as file: return ResourceGroup.deserialize(json.load(file)) except FileNotFoundError: raise ResourceNotFoundError("Resource group not found") from None - def list(self) -> List[ResourceGroup]: + def list(self) -> list[ResourceGroup]: rgs = [] for name in os.listdir(self.path): try: - with open(self.path / str(name) / "resource_group.json", "r", encoding="utf-8") as file: + with open(self.path / str(name) / "resource_group.json", encoding="utf-8") as file: rgs.append(ResourceGroup.deserialize(json.load(file))) except FileNotFoundError: raise ResourceNotFoundError("Resource group not found") from None @@ -103,19 +113,19 @@ class FakeNetworkSecurityGroup: def __init__(self, path: Path) -> None: self.path = path - def list(self, resource_group_name: str) -> List[NetworkSecurityGroup]: + def list(self, resource_group_name: str) -> list[NetworkSecurityGroup]: try: files = [file for file in os.listdir(self.path / resource_group_name) if file.startswith("nsg-")] except FileNotFoundError: raise ResourceNotFoundError("No resource group") from None elements = [] for file_name in files: - with open(self.path / resource_group_name / file_name, "r", encoding="utf-8") as file_obj: + with open(self.path / resource_group_name / file_name, encoding="utf-8") as file_obj: elements.append(NetworkSecurityGroup.deserialize(json.load(file_obj))) return elements def begin_create_or_update(self, resource_group_name: str, network_security_group_name: str, - parameters: Dict[str, Any]) -> WaitableObject: + parameters: dict[str, Any]) -> WaitableObject: base = { "id": f"/subscriptions/6c268694-47ab-43ab-b306-3c5514bc4112/resourceGroups/{resource_group_name}/providers" f"/Microsoft.Network/networkSecurityGroups/default", @@ -162,7 +172,7 @@ def begin_create_or_update(self, resource_group_name: str, network_security_grou def get(self, resource_group_name: str, network_security_group_name: str) -> NetworkSecurityGroup: try: - with open(self.path / resource_group_name / f"nsg-{network_security_group_name}.json", "r", + with open(self.path / resource_group_name / f"nsg-{network_security_group_name}.json", encoding="utf-8") as file: return NetworkSecurityGroup.deserialize(json.load(file)) except FileNotFoundError: @@ -173,18 +183,18 @@ class FakeVirtualNetwork: def __init__(self, path: Path) -> None: self.path = path - def list(self, resource_group_name: str) -> List[VirtualNetwork]: + def list(self, resource_group_name: str) -> list[VirtualNetwork]: try: files = [file for file in os.listdir(self.path / resource_group_name) if file.startswith("vnet-")] except FileNotFoundError: raise ResourceNotFoundError("No resource group") from None elements = [] for file_name in files: - with open(self.path / resource_group_name / file_name, "r", encoding="utf-8") as file_obj: + with open(self.path / resource_group_name / file_name, encoding="utf-8") as file_obj: elements.append(VirtualNetwork.deserialize(json.load(file_obj))) return elements - def begin_create_or_update(self, resource_group_name: str, virtual_network_name: str, parameters: Dict[str, Any] + def begin_create_or_update(self, resource_group_name: str, virtual_network_name: str, parameters: dict[str, Any] ) -> WaitableObject: base = { "id": f"/subscriptions/6c268694-47ab-43ab-b306-3c5514bc4112/resourceGroups/{resource_group_name}/providers" @@ -213,7 +223,7 @@ def begin_create_or_update(self, resource_group_name: str, virtual_network_name: def get(self, resource_group_name: str, virtual_network_name: str) -> NetworkSecurityGroup: try: - with open(self.path / resource_group_name / f"vnet-{virtual_network_name}.json", "r", encoding="utf-8") as file: + with open(self.path / resource_group_name / f"vnet-{virtual_network_name}.json", encoding="utf-8") as file: return VirtualNetwork.deserialize(json.load(file)) except FileNotFoundError: raise ResourceNotFoundError("Network security group not found") from None @@ -223,7 +233,7 @@ class FakeSubnet: def __init__(self, path: Path) -> None: self.path = path - def list(self, resource_group_name: str, virtual_network_name: str) -> List[Subnet]: + def list(self, resource_group_name: str, virtual_network_name: str) -> list[Subnet]: try: files = [file for file in os.listdir(self.path / resource_group_name) if file.startswith(f"subnet-{virtual_network_name}")] @@ -231,12 +241,12 @@ def list(self, resource_group_name: str, virtual_network_name: str) -> List[Subn raise ResourceNotFoundError("No resource group") from None elements = [] for file_name in files: - with open(self.path / resource_group_name / file_name, "r", encoding="utf-8") as file_obj: + with open(self.path / resource_group_name / file_name, encoding="utf-8") as file_obj: elements.append(Subnet.deserialize(json.load(file_obj))) return elements def begin_create_or_update(self, resource_group_name: str, virtual_network_name: str, subnet_name: str, - subnet_parameters: Dict[str, Any]) -> WaitableObject: + subnet_parameters: dict[str, Any]) -> WaitableObject: base = { "id": f"/subscriptions/6c268694-47ab-43ab-b306-3c5514bc4112/resourceGroups/{resource_group_name}/providers" f"/Microsoft.Network/virtualNetworks/{virtual_network_name}/subnets/{subnet_name}", @@ -263,7 +273,7 @@ def begin_create_or_update(self, resource_group_name: str, virtual_network_name: def get(self, resource_group_name: str, virtual_network_name: str, subnet_name: str) -> NetworkSecurityGroup: try: - with open(self.path / resource_group_name / f"subnet-{virtual_network_name}-{subnet_name}.json", "r", + with open(self.path / resource_group_name / f"subnet-{virtual_network_name}-{subnet_name}.json", encoding="utf-8") as file: return Subnet.deserialize(json.load(file)) except FileNotFoundError: @@ -274,18 +284,18 @@ class FakeIpAddress: def __init__(self, path: Path) -> None: self.path = path - def list(self, resource_group_name: str) -> List[PublicIPAddress]: + def list(self, resource_group_name: str) -> list[PublicIPAddress]: try: files = [file for file in os.listdir(self.path / resource_group_name) if file.startswith("ip-")] except FileNotFoundError: raise ResourceNotFoundError("No resource group") from None elements = [] for file_name in files: - with open(self.path / resource_group_name / file_name, "r", encoding="utf-8") as file_obj: + with open(self.path / resource_group_name / file_name, encoding="utf-8") as file_obj: elements.append(PublicIPAddress.deserialize(json.load(file_obj))) return elements - def begin_create_or_update(self, resource_group_name: str, public_ip_address_name: str, parameters: Dict[str, Any] + def begin_create_or_update(self, resource_group_name: str, public_ip_address_name: str, parameters: dict[str, Any] ) -> WaitableObject: base = { "id": f"/subscriptions/6c268694-47ab-43ab-b306-3c5514bc4112/resourceGroups/{resource_group_name}/providers" @@ -316,7 +326,7 @@ def begin_create_or_update(self, resource_group_name: str, public_ip_address_nam def get(self, resource_group_name: str, public_ip_address_name: str) -> PublicIPAddress: try: - with open(self.path / resource_group_name / f"ip-{public_ip_address_name}.json", "r", + with open(self.path / resource_group_name / f"ip-{public_ip_address_name}.json", encoding="utf-8") as file: return PublicIPAddress.deserialize(json.load(file)) except FileNotFoundError: @@ -331,18 +341,18 @@ class FakeNetworkInterface: def __init__(self, path: Path) -> None: self.path = path - def list(self, resource_group_name: str) -> List[NetworkInterface]: + def list(self, resource_group_name: str) -> list[NetworkInterface]: try: files = [file for file in os.listdir(self.path / resource_group_name) if file.startswith("nic-")] except FileNotFoundError: raise ResourceNotFoundError("No resource group") from None elements = [] for file_name in files: - with open(self.path / resource_group_name / file_name, "r", encoding="utf-8") as file_obj: + with open(self.path / resource_group_name / file_name, encoding="utf-8") as file_obj: elements.append(NetworkInterface.deserialize(json.load(file_obj))) return elements - def begin_create_or_update(self, resource_group_name: str, network_interface_name: str, parameters: Dict[str, Any] + def begin_create_or_update(self, resource_group_name: str, network_interface_name: str, parameters: dict[str, Any] ) -> WaitableObject: base = { "id": f"/subscriptions/6c268694-47ab-43ab-b306-3c5514bc4112/resourceGroups/{resource_group_name}/providers" @@ -419,7 +429,7 @@ def begin_create_or_update(self, resource_group_name: str, network_interface_nam def get(self, resource_group_name: str, network_interface_name: str) -> NetworkInterface: try: - with open(self.path / resource_group_name / f"nic-{network_interface_name}.json", "r", + with open(self.path / resource_group_name / f"nic-{network_interface_name}.json", encoding="utf-8") as file: return NetworkInterface.deserialize(json.load(file)) except FileNotFoundError: @@ -445,18 +455,18 @@ class FakeVirtualMachines: def __init__(self, path: Path) -> None: self.path = path - def list(self, resource_group_name: str) -> List[VirtualMachine]: + def list(self, resource_group_name: str) -> list[VirtualMachine]: try: files = [file for file in os.listdir(self.path / resource_group_name) if file.startswith("vm-")] except FileNotFoundError: raise ResourceNotFoundError("No resource group") from None elements = [] for fle in files: - with open(self.path / resource_group_name / fle, "r", encoding="utf-8") as file: + with open(self.path / resource_group_name / fle, encoding="utf-8") as file: elements.append(VirtualMachine.deserialize(json.load(file))) return elements - def begin_create_or_update(self, resource_group_name: str, vm_name: str, parameters: Dict[str, Any] + def begin_create_or_update(self, resource_group_name: str, vm_name: str, parameters: dict[str, Any] ) -> WaitableObject: tags = parameters.pop("tags") if "tags" in parameters else {} parameters = dict_keys_to_camel_case(parameters) @@ -544,9 +554,9 @@ def begin_create_or_update(self, resource_group_name: str, vm_name: str, paramet json.dump(base, fp=file, indent=2) return WaitableObject() - def begin_update(self, resource_group_name: str, vm_name: str, parameters: Dict[str, Any]) -> WaitableObject: + def begin_update(self, resource_group_name: str, vm_name: str, parameters: dict[str, Any]) -> WaitableObject: try: - with open(self.path / resource_group_name / f"vm-{vm_name}.json", "r", encoding="utf-8") as file: + with open(self.path / resource_group_name / f"vm-{vm_name}.json", encoding="utf-8") as file: v_m = json.loads(file.read()) except FileNotFoundError: raise ResourceNotFoundError("Virtual Machine not found") from None @@ -572,7 +582,7 @@ def begin_delete(self, resource_group_name: str, vm_name: str) -> WaitableObject def get(self, resource_group_name: str, vm_name: str) -> VirtualMachine: try: - with open(self.path / resource_group_name / f"vm-{vm_name}.json", "r", encoding="utf-8") as file: + with open(self.path / resource_group_name / f"vm-{vm_name}.json", encoding="utf-8") as file: return VirtualMachine.deserialize(json.load(file)) except FileNotFoundError: raise ResourceNotFoundError("Virtual Machine not found") from None @@ -592,7 +602,7 @@ class FakeImages: # pylint: disable=too-few-public-methods def __init__(self, path: Path) -> None: self.path = path - def list_by_resource_group(self, resource_group_name) -> List[Image]: + def list_by_resource_group(self, resource_group_name) -> list[Image]: with open(self.path / resource_group_name / "azure_images_list.json", encoding="utf-8") as file: return [Image.deserialize(image) for image in json.load(file)] diff --git a/unit_tests/provisioner/test_azure_region_definition_builder.py b/unit_tests/provisioner/test_azure_region_definition_builder.py index 8885c44e296..4aadc131396 100644 --- a/unit_tests/provisioner/test_azure_region_definition_builder.py +++ b/unit_tests/provisioner/test_azure_region_definition_builder.py @@ -14,7 +14,6 @@ from collections import namedtuple from pathlib import Path - from sdcm.keystore import KeyStore from sdcm.provision.provisioner import InstanceDefinition from sdcm.sct_config import SCTConfiguration diff --git a/unit_tests/provisioner/test_provisioner.py b/unit_tests/provisioner/test_provisioner.py index 2a6722fc45d..d50cad29228 100644 --- a/unit_tests/provisioner/test_provisioner.py +++ b/unit_tests/provisioner/test_provisioner.py @@ -12,15 +12,15 @@ # Copyright (c) 2022 ScyllaDB # pylint: disable=redefined-outer-name import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime, timezone import pytest from sdcm.keystore import KeyStore from sdcm.provision.provisioner import ( InstanceDefinition, - provisioner_factory, ProvisionerError, + provisioner_factory, ) from sdcm.provision.user_data import UserDataObject @@ -89,7 +89,7 @@ def provisioner(backend, provisioner_params): def test_can_provision_scylla_vm(region, definition, provisioner, backend, provisioner_params): - creation_time = datetime.utcnow().replace(microsecond=0).replace(tzinfo=timezone.utc) + creation_time = datetime.utcnow().replace(microsecond=0).replace(tzinfo=UTC) v_m = provisioner.get_or_create_instances(definitions=[definition])[0] assert v_m.name == definition.name assert v_m.region == region diff --git a/unit_tests/provisioner/test_user_data_builder.py b/unit_tests/provisioner/test_user_data_builder.py index fe5c3c807b3..d242d649310 100644 --- a/unit_tests/provisioner/test_user_data_builder.py +++ b/unit_tests/provisioner/test_user_data_builder.py @@ -13,7 +13,7 @@ import yaml -from sdcm.provision.user_data import UserDataObject, UserDataBuilder +from sdcm.provision.user_data import UserDataBuilder, UserDataObject class ExampleUserDataObject(UserDataObject): diff --git a/unit_tests/test_adaptive_timeouts.py b/unit_tests/test_adaptive_timeouts.py index d3edaf91a1e..b8d3b853146 100644 --- a/unit_tests/test_adaptive_timeouts.py +++ b/unit_tests/test_adaptive_timeouts.py @@ -21,14 +21,14 @@ from invoke import Result from sdcm.remote import RemoteCmdRunnerBase -from sdcm.utils.adaptive_timeouts.load_info_store import AdaptiveTimeoutStore from sdcm.utils.adaptive_timeouts import Operations, adaptive_timeout +from sdcm.utils.adaptive_timeouts.load_info_store import AdaptiveTimeoutStore from unit_tests.lib.fake_remoter import FakeRemoter LOGGER = logging.getLogger(__name__) -class FakeNode(): +class FakeNode: def __init__(self, name: str, remoter): self.name = name diff --git a/unit_tests/test_alternator_streams_kcl.py b/unit_tests/test_alternator_streams_kcl.py index 939052eefed..60442741a24 100644 --- a/unit_tests/test_alternator_streams_kcl.py +++ b/unit_tests/test_alternator_streams_kcl.py @@ -11,12 +11,13 @@ # # Copyright (c) 2021 ScyllaDB -import time import logging +import time + import pytest +from sdcm.kcl_thread import CompareTablesSizesThread, KclStressThread from sdcm.ycsb_thread import YcsbStressThread -from sdcm.kcl_thread import KclStressThread, CompareTablesSizesThread from unit_tests.dummy_remote import LocalLoaderSetDummy from unit_tests.lib.alternator_utils import TEST_PARAMS diff --git a/unit_tests/test_base_version.py b/unit_tests/test_base_version.py index dcd8e55cd50..f445762aece 100644 --- a/unit_tests/test_base_version.py +++ b/unit_tests/test_base_version.py @@ -14,7 +14,9 @@ import unittest -from utils.get_supported_scylla_base_versions import UpgradeBaseVersion # pylint: disable=no-name-in-module, import-error +from utils.get_supported_scylla_base_versions import ( + UpgradeBaseVersion, # pylint: disable=no-name-in-module, import-error +) def general_test(scylla_repo='', linux_distro='', cloud_provider=None): diff --git a/unit_tests/test_chaos_mesh.py b/unit_tests/test_chaos_mesh.py index 338d10c242c..772c97f1c8b 100644 --- a/unit_tests/test_chaos_mesh.py +++ b/unit_tests/test_chaos_mesh.py @@ -15,24 +15,28 @@ # pylint: skip-file from dataclasses import dataclass, field -from typing import Dict +import pytest import yaml from invoke import Result -import pytest -from sdcm.utils.k8s.chaos_mesh import PodFailureExperiment, ExperimentStatus, ChaosMeshTimeout, ChaosMeshExperimentException, \ - MemoryStressExperiment +from sdcm.utils.k8s.chaos_mesh import ( + ChaosMeshExperimentException, + ChaosMeshTimeout, + ExperimentStatus, + MemoryStressExperiment, + PodFailureExperiment, +) @dataclass class DummyK8sCluster: - _commands: Dict[str, Result] = field(default_factory=dict) + _commands: dict[str, Result] = field(default_factory=dict) region_name: str = 'fake-region-1' def apply_file(self, config_path: str): """Parses file content to yaml and prints it.""" - with open(config_path, "r", encoding="utf-8") as config_file: + with open(config_path, encoding="utf-8") as config_file: print(yaml.safe_load(config_file)) def kubectl(self, command, *args, **kwargs): diff --git a/unit_tests/test_clean_cloud_resources_func.py b/unit_tests/test_clean_cloud_resources_func.py index 4f038587aca..ef57ef724d7 100644 --- a/unit_tests/test_clean_cloud_resources_func.py +++ b/unit_tests/test_clean_cloud_resources_func.py @@ -15,12 +15,16 @@ from unittest.mock import MagicMock, patch from sdcm.sct_config import SCTConfiguration -from sdcm.utils.common import \ - clean_cloud_resources, \ - clean_instances_aws, clean_elastic_ips_aws, clean_clusters_gke, clean_instances_gce, clean_resources_docker +from sdcm.utils.common import ( + clean_cloud_resources, + clean_clusters_gke, + clean_elastic_ips_aws, + clean_instances_aws, + clean_instances_gce, + clean_resources_docker, +) from sdcm.utils.context_managers import environment - SCT_RUNNER_AWS = { "Tags": [{"Key": "NodeType", "Value": "sct-runner"}], "InstanceId": "i-1111", diff --git a/unit_tests/test_cluster.py b/unit_tests/test_cluster.py index 623eefbf553..e2fc1ace26f 100644 --- a/unit_tests/test_cluster.py +++ b/unit_tests/test_cluster.py @@ -14,30 +14,28 @@ # pylint: disable=too-few-public-methods import json -import time -import shutil import logging import os.path +import shutil import tempfile +import time import unittest from datetime import datetime from functools import cached_property -from typing import List from weakref import proxy as weakproxy import pytest from invoke import Result -from sdcm.cluster import BaseNode, BaseCluster, BaseMonitorSet, BaseScyllaCluster +from sdcm.cluster import BaseCluster, BaseMonitorSet, BaseNode, BaseScyllaCluster from sdcm.db_log_reader import DbLogReader from sdcm.sct_events import Severity from sdcm.sct_events.database import SYSTEM_ERROR_EVENTS_PATTERNS -from sdcm.sct_events.group_common_events import ignore_upgrade_schema_errors from sdcm.sct_events.filters import DbEventsFilter +from sdcm.sct_events.group_common_events import ignore_upgrade_schema_errors from sdcm.sct_events.system import InstanceStatusEvent from sdcm.utils.common import get_keyspace_partition_ranges, keyspace_min_max_tokens from sdcm.utils.distro import Distro - from unit_tests.dummy_remote import DummyRemote from unit_tests.lib.events_utils import EventsUtilsMixin from unit_tests.test_utils_common import DummyNode @@ -443,7 +441,7 @@ def run_nodetool(self, *args, **kwargs): # pylint: disable=unused-argument class DummyScyllaCluster(BaseScyllaCluster, BaseCluster): # pylint: disable=abstract-method - nodes: List['NodetoolDummyNode'] + nodes: list['NodetoolDummyNode'] def __init__(self, params): # pylint: disable=super-init-not-called self.nodes = params diff --git a/unit_tests/test_config.py b/unit_tests/test_config.py index 460068da30d..474aac8dae7 100644 --- a/unit_tests/test_config.py +++ b/unit_tests/test_config.py @@ -11,16 +11,18 @@ # # Copyright (c) 2020 ScyllaDB -import os -import logging import itertools +import logging +import os import unittest from collections import namedtuple + import pytest + from sdcm import sct_config from sdcm.utils import loader_utils -from sdcm.utils.common import get_latest_scylla_release from sdcm.utils.aws_utils import get_ssm_ami +from sdcm.utils.common import get_latest_scylla_release RPM_URL = 'https://s3.amazonaws.com/downloads.scylladb.com/enterprise/rpm/unstable/centos/' \ '9f724fedb93b4734fcfaec1156806921ff46e956-2bdfa9f7ef592edaf15e028faf3b7f695f39ebc1' \ diff --git a/unit_tests/test_config_get_version_based_on_conf.py b/unit_tests/test_config_get_version_based_on_conf.py index 885a751a4ce..493aeeff448 100644 --- a/unit_tests/test_config_get_version_based_on_conf.py +++ b/unit_tests/test_config_get_version_based_on_conf.py @@ -11,8 +11,9 @@ # # Copyright (c) 2023 ScyllaDB -import os import logging +import os + import pytest from sdcm import sct_config diff --git a/unit_tests/test_coredump.py b/unit_tests/test_coredump.py index 85aec47de99..cce5196c6fc 100644 --- a/unit_tests/test_coredump.py +++ b/unit_tests/test_coredump.py @@ -1,11 +1,16 @@ -import unittest import os -import time import tempfile +import time +import unittest from abc import abstractmethod from sdcm.cluster import BaseNode -from sdcm.coredump import CoredumpExportSystemdThread, CoreDumpInfo, CoredumpExportFileThread, CoredumpThreadBase +from sdcm.coredump import ( + CoredumpExportFileThread, + CoredumpExportSystemdThread, + CoreDumpInfo, + CoredumpThreadBase, +) from unit_tests.lib.data_pickle import Pickler from unit_tests.lib.mock_remoter import MockRemoter diff --git a/unit_tests/test_decode_backtrace.py b/unit_tests/test_decode_backtrace.py index cf626bd8100..979ec0b6ae6 100644 --- a/unit_tests/test_decode_backtrace.py +++ b/unit_tests/test_decode_backtrace.py @@ -11,19 +11,18 @@ # # Copyright (c) 2020 ScyllaDB -import os import json -from multiprocessing import Queue +import os import unittest from functools import cached_property +from multiprocessing import Queue from sdcm.cluster import TestConfig from sdcm.db_log_reader import DbLogReader from sdcm.sct_events.database import SYSTEM_ERROR_EVENTS_PATTERNS - from unit_tests.dummy_remote import DummyRemote -from unit_tests.test_utils_common import DummyNode from unit_tests.lib.events_utils import EventsUtilsMixin +from unit_tests.test_utils_common import DummyNode class DecodeDummyNode(DummyNode): # pylint: disable=abstract-method diff --git a/unit_tests/test_events.py b/unit_tests/test_events.py index dac440e8133..3f762dc63a6 100644 --- a/unit_tests/test_events.py +++ b/unit_tests/test_events.py @@ -11,30 +11,33 @@ # # Copyright (c) 2020 ScyllaDB -import time import logging -import unittest import multiprocessing -from pathlib import Path +import time +import unittest from datetime import datetime +from pathlib import Path from unittest import mock import pytest from parameterized import parameterized -from sdcm.exceptions import UnsupportedNemesis, KillNemesis +from sdcm.exceptions import KillNemesis, UnsupportedNemesis from sdcm.prometheus import start_metrics_server -from sdcm.sct_events.nodetool import NodetoolEvent -from sdcm.utils.decorators import timeout from sdcm.sct_events import Severity -from sdcm.sct_events.system import CoreDumpEvent, TestFrameworkEvent, SoftTimeoutEvent -from sdcm.sct_events.filters import DbEventsFilter, EventsFilter, EventsSeverityChangerFilter -from sdcm.sct_events.loaders import YcsbStressEvent -from sdcm.sct_events.nemesis import DisruptionEvent from sdcm.sct_events.database import DatabaseLogEvent -from sdcm.sct_events.file_logger import get_logger_event_summary from sdcm.sct_events.event_counter import EventCounterContextManager - +from sdcm.sct_events.file_logger import get_logger_event_summary +from sdcm.sct_events.filters import ( + DbEventsFilter, + EventsFilter, + EventsSeverityChangerFilter, +) +from sdcm.sct_events.loaders import YcsbStressEvent +from sdcm.sct_events.nemesis import DisruptionEvent +from sdcm.sct_events.nodetool import NodetoolEvent +from sdcm.sct_events.system import CoreDumpEvent, SoftTimeoutEvent, TestFrameworkEvent +from sdcm.utils.decorators import timeout from unit_tests.lib.events_utils import EventsUtilsMixin LOGGER = logging.getLogger(__name__) diff --git a/unit_tests/test_hydra_sh.py b/unit_tests/test_hydra_sh.py index e4c84f62fc3..73ae7946e87 100644 --- a/unit_tests/test_hydra_sh.py +++ b/unit_tests/test_hydra_sh.py @@ -4,8 +4,8 @@ import re import tempfile import unittest +from collections.abc import Iterable, Sequence from functools import cached_property -from typing import Dict, Union, Tuple, Iterable, Sequence, List from parameterized import parameterized @@ -57,10 +57,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): class HydraTestCaseParams: name: str cmd: str - expected: List[Union[str, re.Pattern]] - not_expected: List[Union[str, re.Pattern]] + expected: list[str | re.Pattern] + not_expected: list[str | re.Pattern] return_code: int - env: Dict[str, str] + env: dict[str, str] def __str__(self): return self.name @@ -219,7 +219,7 @@ def is_gce_or_gke(self) -> bool: return 'gce' in self.backend or 'gke' in self.backend @property - def get_longevity_env(self) -> Dict[str, str]: + def get_longevity_env(self) -> dict[str, str]: longevity_end = { 'SCT_TEST_ID': self.test_id, 'HOME': self.test_home_dir, @@ -354,7 +354,7 @@ def test_case_send_email(self): ), self.test_tmp_dir @property - def hydra_test_cases(self) -> Iterable[Tuple[HydraTestCaseParams, HydraTestCaseTmpDir]]: + def hydra_test_cases(self) -> Iterable[tuple[HydraTestCaseParams, HydraTestCaseTmpDir]]: """ Creates list of test case parameters that represent steps in longevity pipeline steps """ @@ -387,8 +387,8 @@ def environ(): def validate_result( result, expected_status: int, - expected: Sequence[Union[str, re.Pattern]], - not_expected: Sequence[Union[str, re.Pattern]], + expected: Sequence[str | re.Pattern], + not_expected: Sequence[str | re.Pattern], ): errors = [] if expected_status is not None: diff --git a/unit_tests/test_latte_thread.py b/unit_tests/test_latte_thread.py index ffa8b04b72d..fd6a258f3e1 100644 --- a/unit_tests/test_latte_thread.py +++ b/unit_tests/test_latte_thread.py @@ -16,8 +16,8 @@ import pytest import requests -from sdcm.utils.decorators import timeout from sdcm.stress.latte_thread import LatteStressThread +from sdcm.utils.decorators import timeout from unit_tests.dummy_remote import LocalLoaderSetDummy pytestmark = [ @@ -85,7 +85,7 @@ def cleanup_thread(): @timeout(timeout=60) def check_metrics(): - output = requests.get("http://{}/metrics".format(prom_address)).text + output = requests.get(f"http://{prom_address}/metrics").text regex = re.compile(r"^sct_latte_run_gauge.*?([0-9\.]*?)$", re.MULTILINE) assert "sct_latte_run_gauge" in output diff --git a/unit_tests/test_lib_utils.py b/unit_tests/test_lib_utils.py index cda9120e6b8..5c049a9b63b 100644 --- a/unit_tests/test_lib_utils.py +++ b/unit_tests/test_lib_utils.py @@ -14,8 +14,7 @@ import logging import unittest -from test_lib.utils import get_data_by_path, MagicList - +from test_lib.utils import MagicList, get_data_by_path logging.basicConfig(level=logging.DEBUG) diff --git a/unit_tests/test_manual.py b/unit_tests/test_manual.py index 16334f545fb..2d3e4248f22 100644 --- a/unit_tests/test_manual.py +++ b/unit_tests/test_manual.py @@ -13,21 +13,20 @@ from __future__ import annotations +import logging +import tempfile import time import unittest -import tempfile -import logging from typing import TYPE_CHECKING import boto3 import requests -from sdcm.prometheus import start_metrics_server, nemesis_metrics_obj +from sdcm.loader import CassandraStressExporter, CassandraStressHDRExporter +from sdcm.prometheus import nemesis_metrics_obj, start_metrics_server from sdcm.remote import RemoteCmdRunnerBase from sdcm.sct_events.setup import start_events_device, stop_events_device - -from sdcm.stress_thread import CassandraStressThread, CassandraStressEventsPublisher -from sdcm.loader import CassandraStressExporter, CassandraStressHDRExporter +from sdcm.stress_thread import CassandraStressEventsPublisher, CassandraStressThread from sdcm.ycsb_thread import YcsbStressThread if TYPE_CHECKING: @@ -121,7 +120,7 @@ def test_01(self): tmp_file.file.flush() time.sleep(2) - output = requests.get("http://{}/metrics".format(self.prom_address)).text + output = requests.get(f"http://{self.prom_address}/metrics").text assert 'sct_cassandra_stress_write_gauge{cassandra_stress_write="0",cpu_idx="0",instance="127.0.0.1",loader_idx="1",type="ops"} 70178.0' in output time.sleep(1) @@ -154,7 +153,7 @@ def test_01_mixed(self): tmp_file.file.flush() time.sleep(2) - output = requests.get("http://{}/metrics".format(self.prom_address)).text + output = requests.get(f"http://{self.prom_address}/metrics").text expected_lines = output.split("\n")[-12:] assert 'collectd_cassandra_stress_hdr_mixed_gauge{cassandra_stress_hdr_mixed="WRITE",cpu_idx="0",instance="127.0.0.1",keyspace="",loader_idx="1",type="lat_perc_50"} 0.62' in expected_lines @@ -176,7 +175,7 @@ def test_02_write(self): tmp_file.file.flush() time.sleep(2) - output = requests.get("http://{}/metrics".format(self.prom_address)).text + output = requests.get(f"http://{self.prom_address}/metrics").text expected_lines = output.split("\n")[-6:] assert 'collectd_cassandra_stress_hdr_write_gauge{cassandra_stress_hdr_write="WRITE",cpu_idx="0",instance="127.0.0.1",keyspace="",loader_idx="1",type="lat_perc_50"} 0.62' in expected_lines time.sleep(1) @@ -196,7 +195,7 @@ def test_03_read(self): tmp_file.file.flush() time.sleep(2) - output = requests.get("http://{}/metrics".format(self.prom_address)).text + output = requests.get(f"http://{self.prom_address}/metrics").text expected_lines = output.split("\n")[-6:] assert 'collectd_cassandra_stress_hdr_read_gauge{cassandra_stress_hdr_read="READ",cpu_idx="0",instance="127.0.0.1",keyspace="",loader_idx="1",type="lat_perc_999"} 36.54' in expected_lines time.sleep(1) @@ -221,7 +220,7 @@ def test_04_mixed_only_write(self): tmp_file.file.flush() time.sleep(2) - output = requests.get("http://{}/metrics".format(self.prom_address)).text + output = requests.get(f"http://{self.prom_address}/metrics").text expected_lines = output.split("\n")[-6:] logging.getLogger(__file__).info(output) logging.getLogger(__file__).info(expected_lines) diff --git a/unit_tests/test_microbenchmarking.py b/unit_tests/test_microbenchmarking.py index e1bd03ed2b9..62917e3aac2 100644 --- a/unit_tests/test_microbenchmarking.py +++ b/unit_tests/test_microbenchmarking.py @@ -1,16 +1,18 @@ -from __future__ import absolute_import -import unittest -import os.path -import subprocess +import hashlib +import json import logging +import os.path import pickle +import subprocess import tempfile -import json +import unittest import zipfile -import hashlib - -from sdcm.microbenchmarking import MicroBenchmarkingResultsAnalyzer, LargeNumberOfDatasetsException, EmptyResultFolder +from sdcm.microbenchmarking import ( + EmptyResultFolder, + LargeNumberOfDatasetsException, + MicroBenchmarkingResultsAnalyzer, +) LOGGER = logging.getLogger("microbenchmarking-tests") diff --git a/unit_tests/test_nemesis.py b/unit_tests/test_nemesis.py index 6d86cdf060d..62f9d48c7eb 100644 --- a/unit_tests/test_nemesis.py +++ b/unit_tests/test_nemesis.py @@ -5,17 +5,16 @@ import pytest import sdcm.utils.cloud_monitor # pylint: disable=unused-import # import only to avoid cyclic dependency -from sdcm.nemesis import Nemesis, CategoricalMonkey, SisyphusMonkey, ToggleGcModeMonkey from sdcm.cluster import BaseScyllaCluster -from sdcm.cluster_k8s.mini_k8s import LocalMinimalScyllaPodCluster -from sdcm.cluster_k8s.gke import GkeScyllaPodCluster -from sdcm.cluster_k8s.eks import EksScyllaPodCluster -from sdcm.cluster_gce import ScyllaGCECluster from sdcm.cluster_aws import ScyllaAWSCluster from sdcm.cluster_docker import ScyllaDockerCluster +from sdcm.cluster_gce import ScyllaGCECluster +from sdcm.cluster_k8s.eks import EksScyllaPodCluster +from sdcm.cluster_k8s.gke import GkeScyllaPodCluster +from sdcm.cluster_k8s.mini_k8s import LocalMinimalScyllaPodCluster +from sdcm.nemesis import CategoricalMonkey, Nemesis, SisyphusMonkey, ToggleGcModeMonkey from unit_tests.test_tester import ClusterTesterForTests - PARAMS = dict(nemesis_interval=1, nemesis_filter_seeds=False) LOGGER = logging.getLogger(__name__) diff --git a/unit_tests/test_nemesis_sisyphus.py b/unit_tests/test_nemesis_sisyphus.py index 5e18160f311..79fa27ad264 100644 --- a/unit_tests/test_nemesis_sisyphus.py +++ b/unit_tests/test_nemesis_sisyphus.py @@ -1,8 +1,9 @@ from dataclasses import dataclass, field + import yaml -from sdcm.nemesis import Nemesis, SisyphusMonkey from sdcm.cluster import BaseScyllaCluster +from sdcm.nemesis import Nemesis, SisyphusMonkey PARAMS = dict(nemesis_interval=1, nemesis_filter_seeds=False) @@ -80,7 +81,7 @@ def test_list_all_available_nemesis(generate_file=True): with open('data_dir/nemesis_classes.yml', 'w', encoding="utf-8") as outfile2: yaml.dump(disruption_classes, outfile2, default_flow_style=False) - with open('data_dir/nemesis.yml', 'r', encoding="utf-8") as nemesis_file: + with open('data_dir/nemesis.yml', encoding="utf-8") as nemesis_file: static_nemesis_list = yaml.safe_load(nemesis_file) assert static_nemesis_list == disruptions_dict diff --git a/unit_tests/test_parallelobject.py b/unit_tests/test_parallelobject.py index 395a4ea1d20..e4e5266d8e6 100644 --- a/unit_tests/test_parallelobject.py +++ b/unit_tests/test_parallelobject.py @@ -1,9 +1,8 @@ -import unittest -import time +import concurrent.futures import logging import random -import concurrent.futures - +import time +import unittest from sdcm.utils.common import ParallelObject, ParallelObjectException diff --git a/unit_tests/test_profiler.py b/unit_tests/test_profiler.py index e52a9e43372..025eb76ef4f 100644 --- a/unit_tests/test_profiler.py +++ b/unit_tests/test_profiler.py @@ -12,25 +12,40 @@ # Copyright (c) 2020 ScyllaDB # -import time -import threading -from threading import Thread import multiprocessing -from multiprocessing import Process -import unittest import os.path import shutil -from sdcm.utils.common import generate_random_string -from sdcm.prometheus import PrometheusAlertManagerListener -from unit_tests.lib.test_profiler.lib import LibMultiprocessingProcessCustomClass, LibProcessCustomClass, \ - LibMultiprocessingProcessCustomClassWithRun, LibProcessCustomClassWithRun, LibThreadCustomClass, \ - LibThreadCustomClassWithRun, LibThreadingThreadCustomClass, LibThreadingThreadCustomClassWithRun, LibThread, LibProcess - -from unit_tests.lib.test_profiler.lib2 import LibProfileableProcessCustomClass, \ - LibProfileableProcessCustomClassWithRun, LibProfileableThreadCustomClass, LibProfileableThreadCustomClassWithRun, \ - LibProfileableThread, LibProfileableProcess +import threading +import time +import unittest +from multiprocessing import Process +from threading import Thread -from sdcm.utils.profiler import ProfilerFactory, ProfileableProcess as pp, ProfileableThread as pt # pylint: disable=ungrouped-imports +from sdcm.prometheus import PrometheusAlertManagerListener +from sdcm.utils.common import generate_random_string +from sdcm.utils.profiler import ProfileableProcess as pp +from sdcm.utils.profiler import ProfileableThread as pt +from sdcm.utils.profiler import ProfilerFactory # pylint: disable=ungrouped-imports +from unit_tests.lib.test_profiler.lib import ( + LibMultiprocessingProcessCustomClass, + LibMultiprocessingProcessCustomClassWithRun, + LibProcess, + LibProcessCustomClass, + LibProcessCustomClassWithRun, + LibThread, + LibThreadCustomClass, + LibThreadCustomClassWithRun, + LibThreadingThreadCustomClass, + LibThreadingThreadCustomClassWithRun, +) +from unit_tests.lib.test_profiler.lib2 import ( + LibProfileableProcess, + LibProfileableProcessCustomClass, + LibProfileableProcessCustomClassWithRun, + LibProfileableThread, + LibProfileableThreadCustomClass, + LibProfileableThreadCustomClassWithRun, +) def function_sleep(): diff --git a/unit_tests/test_prometheus.py b/unit_tests/test_prometheus.py index ba14b3eef04..53c733f4a6d 100644 --- a/unit_tests/test_prometheus.py +++ b/unit_tests/test_prometheus.py @@ -11,8 +11,8 @@ # # Copyright (c) 2020 ScyllaDB -import os import json +import os import unittest from sdcm.prometheus import PrometheusAlertManagerListener diff --git a/unit_tests/test_python_driver.py b/unit_tests/test_python_driver.py index 48f10474d0d..209047a02f7 100644 --- a/unit_tests/test_python_driver.py +++ b/unit_tests/test_python_driver.py @@ -4,7 +4,6 @@ from unit_tests.test_cluster import DummyDbCluster, DummyNode, DummyRemote - log = logging.getLogger(__name__) diff --git a/unit_tests/test_remoter.py b/unit_tests/test_remoter.py index 9551d74032a..a082fb4ad2c 100644 --- a/unit_tests/test_remoter.py +++ b/unit_tests/test_remoter.py @@ -11,21 +11,26 @@ # # Copyright (c) 2020 ScyllaDB -import os import getpass -import unittest +import os import threading -from typing import Union, Optional +import unittest from logging import getLogger -# from parameterized import parameterized +from sdcm.cluster_k8s import KubernetesCluster -from sdcm.remote import RemoteLibSSH2CmdRunner, RemoteCmdRunner, LocalCmdRunner, RetryableNetworkException, \ - SSHConnectTimeoutError, shell_script_cmd -from sdcm.remote.kubernetes_cmd_runner import KubernetesCmdRunner +# from parameterized import parameterized +from sdcm.remote import ( + LocalCmdRunner, + RemoteCmdRunner, + RemoteLibSSH2CmdRunner, + RetryableNetworkException, + SSHConnectTimeoutError, + shell_script_cmd, +) from sdcm.remote.base import CommandRunner, Result +from sdcm.remote.kubernetes_cmd_runner import KubernetesCmdRunner from sdcm.remote.remote_file import remote_file -from sdcm.cluster_k8s import KubernetesCluster class FakeKluster(KubernetesCluster): @@ -89,7 +94,7 @@ class TestRemoteCmdRunners(unittest.TestCase): @staticmethod def _create_and_run_twice_in_same_thread(remoter_type, key_file, stmt, kwargs, paramiko_thread_results): - if issubclass(remoter_type, (RemoteCmdRunner, RemoteLibSSH2CmdRunner)): + if issubclass(remoter_type, RemoteCmdRunner | RemoteLibSSH2CmdRunner): remoter = remoter_type(hostname='127.0.0.1', user=getpass.getuser(), key_file=key_file) else: remoter = KubernetesCmdRunner( @@ -111,7 +116,7 @@ def _create_and_run_twice_in_same_thread(remoter_type, key_file, stmt, kwargs, p # pylint: disable=too-many-arguments @staticmethod def _create_and_run_in_same_thread(remoter_type, host, key_file, stmt, kwargs, paramiko_thread_results): # noqa: PLR0913 - if issubclass(remoter_type, (RemoteCmdRunner, RemoteLibSSH2CmdRunner)): + if issubclass(remoter_type, RemoteCmdRunner | RemoteLibSSH2CmdRunner): remoter = remoter_type(hostname=host, user=getpass.getuser(), key_file=key_file) else: remoter = KubernetesCmdRunner( @@ -202,7 +207,7 @@ def _run_parallel(thread_count, thread_body, args, kwargs): @unittest.skip('To be ran manually') def test_run_in_mainthread( # pylint: disable=too-many-arguments # noqa: PLR0913 self, remoter_type, host: str, stmt: str, verbose: bool, ignore_status: bool, new_session: bool, retry: int, - timeout: Union[float, None]): + timeout: float | None): kwargs = { 'verbose': verbose, 'ignore_status': ignore_status, @@ -214,7 +219,7 @@ def test_run_in_mainthread( # pylint: disable=too-many-arguments # noqa: PLR09 except Exception as exc: # pylint: disable=broad-except # noqa: BLE001 expected = exc - if issubclass(remoter_type, (RemoteCmdRunner, RemoteLibSSH2CmdRunner)): + if issubclass(remoter_type, RemoteCmdRunner | RemoteLibSSH2CmdRunner): remoter = remoter_type(hostname=host, user=getpass.getuser(), key_file=self.key_file) else: remoter = KubernetesCmdRunner( @@ -239,7 +244,7 @@ def test_run_in_mainthread( # pylint: disable=too-many-arguments # noqa: PLR09 @unittest.skip('To be ran manually') def test_create_and_run_in_same_thread( # pylint: disable=too-many-arguments,too-many-locals # noqa: PLR0913 self, remoter_type, host: str, stmt: str, verbose: bool, ignore_status: bool, new_session: bool, - retry: int, timeout: Union[float, None]): + retry: int, timeout: float | None): kwargs = { 'verbose': verbose, 'ignore_status': ignore_status, @@ -266,7 +271,7 @@ def test_create_and_run_in_same_thread( # pylint: disable=too-many-arguments,to @unittest.skip('To be ran manually') def test_create_and_run_in_separate_thread( # pylint: disable=too-many-arguments # noqa: PLR0913 self, remoter_type, host: str, stmt: str, verbose: bool, ignore_status: bool, - new_session: bool, retry: int, timeout: Union[float, None]): + new_session: bool, retry: int, timeout: float | None): kwargs = { 'verbose': verbose, 'ignore_status': ignore_status, @@ -281,7 +286,7 @@ def test_create_and_run_in_separate_thread( # pylint: disable=too-many-argument # Paramiko fails too often when it is invoked like that, that is why it is not in the test - if issubclass(remoter_type, (RemoteCmdRunner, RemoteLibSSH2CmdRunner)): + if issubclass(remoter_type, RemoteCmdRunner | RemoteLibSSH2CmdRunner): remoter = remoter_type(hostname=host, user=getpass.getuser(), key_file=self.key_file) else: remoter = KubernetesCmdRunner( @@ -390,7 +395,7 @@ def run(self, cmd, *_, **__): def _create_connection(self): pass - def is_up(self, timeout: Optional[float] = None) -> bool: + def is_up(self, timeout: float | None = None) -> bool: return True cls.remoter_cls = _Runner diff --git a/unit_tests/test_replication_strategy_utils.py b/unit_tests/test_replication_strategy_utils.py index dfcbfa60a60..d00b4e1d91f 100644 --- a/unit_tests/test_replication_strategy_utils.py +++ b/unit_tests/test_replication_strategy_utils.py @@ -1,8 +1,14 @@ from collections import namedtuple + import pytest -from sdcm.utils.replication_strategy_utils import temporary_replication_strategy_setter, \ - SimpleReplicationStrategy, NetworkTopologyReplicationStrategy, ReplicationStrategy, LocalReplicationStrategy +from sdcm.utils.replication_strategy_utils import ( + LocalReplicationStrategy, + NetworkTopologyReplicationStrategy, + ReplicationStrategy, + SimpleReplicationStrategy, + temporary_replication_strategy_setter, +) class TestReplicationStrategies: @@ -75,7 +81,7 @@ def cql_connection_patient(node): return Cluster.Session() -class Node(): # pylint: disable=too-few-public-methods +class Node: # pylint: disable=too-few-public-methods def __init__(self): self.parent_cluster = Cluster() diff --git a/unit_tests/test_scan_operation_thread.py b/unit_tests/test_scan_operation_thread.py index ef922ba3a31..28835e9aaa4 100644 --- a/unit_tests/test_scan_operation_thread.py +++ b/unit_tests/test_scan_operation_thread.py @@ -6,19 +6,25 @@ test_scan_negative_operation_timed_out - getting operation_timed_out in scan execution test_scan_negative_exception - getting operation_timed_out in scan execution (with and without nemesis) """ -from pathlib import Path import os -from threading import Event from importlib import reload +from pathlib import Path +from threading import Event from unittest.mock import MagicMock, patch + import pytest from cassandra import OperationTimedOut, ReadTimeout +import sdcm.scan_operation_thread +from sdcm.scan_operation_thread import ( + PrometheusDBStats, + ScanOperationThread, + ThreadParams, +) +from sdcm.utils.decorators import Retry, retrying + # from sdcm.utils.operations_thread import ThreadParams from unit_tests.test_cluster import DummyDbCluster, DummyNode -from sdcm.utils.decorators import retrying, Retry -import sdcm.scan_operation_thread -from sdcm.scan_operation_thread import ScanOperationThread, ThreadParams, PrometheusDBStats def mock_retrying_decorator(*args, **kwargs): # pylint: disable=unused-argument diff --git a/unit_tests/test_sct_events_base.py b/unit_tests/test_sct_events_base.py index c7045006af5..f0cfb1800f7 100644 --- a/unit_tests/test_sct_events_base.py +++ b/unit_tests/test_sct_events_base.py @@ -15,13 +15,17 @@ import pickle import tempfile import unittest -from typing import Optional, Type, Protocol, runtime_checkable +from typing import Protocol, runtime_checkable from unittest.mock import patch -from sdcm.sct_events import Severity, SctEventProtocol -from sdcm.sct_events.base import \ - SctEvent, SctEventTypesRegistry, BaseFilter, LogEvent, LogEventProtocol - +from sdcm.sct_events import SctEventProtocol, Severity +from sdcm.sct_events.base import ( + BaseFilter, + LogEvent, + LogEventProtocol, + SctEvent, + SctEventTypesRegistry, +) Y = None # define a global name for pickle. @@ -40,7 +44,7 @@ class SctEventTestCase(unittest.TestCase): b"Y.T.S: CRITICAL\n" b"Z.T.S: NORMAL\n" b"W.T.S: NORMAL\n") - severities_conf: Optional[str] = None + severities_conf: str | None = None _registry_bu = None @classmethod @@ -210,10 +214,10 @@ class Y(SctEvent): def test_add_subevent_type_all_levels_not_abstract(self): @runtime_checkable class YT(SctEventProtocol, Protocol): - S: Type[SctEventProtocol] + S: type[SctEventProtocol] class Y(SctEvent): - T: Type[YT] + T: type[YT] Y.add_subevent_type("T") self.assertFalse(Y.T.is_abstract()) @@ -233,10 +237,10 @@ class Y(SctEvent): def test_add_subevent_type_abstract_level_in_the_middle(self): @runtime_checkable class ZT(SctEventProtocol, Protocol): - S: Type[SctEventProtocol] + S: type[SctEventProtocol] class Z(SctEvent): - T: Type[ZT] + T: type[ZT] self.assertRaisesRegex(ValueError, "no max severity", Z.add_subevent_type, "T") Z.add_subevent_type("T", abstract=True) @@ -257,10 +261,10 @@ class Z(SctEvent): def test_add_subevent_type_subtype_is_not_abstract_only(self): @runtime_checkable class WT(SctEventProtocol, Protocol): - S: Type[SctEventProtocol] + S: type[SctEventProtocol] class W(SctEvent, abstract=True): - T: Type[WT] + T: type[WT] self.assertRaisesRegex(ValueError, "no max severity", W.add_subevent_type, "T") W.add_subevent_type("T", abstract=True) @@ -281,10 +285,10 @@ class W(SctEvent, abstract=True): def test_add_subevent_type_assertions(self): @runtime_checkable class YT(SctEventProtocol, Protocol): - S: Type[SctEventProtocol] + S: type[SctEventProtocol] class Y(SctEvent): - T: Type[YT] + T: type[YT] self.assertRaisesRegex(AssertionError, "valid Python identifier", Y.add_subevent_type, "123") self.assertRaisesRegex(AssertionError, "valid Python identifier", Y.add_subevent_type, "T.S") @@ -300,10 +304,10 @@ class Y(SctEvent): def test_add_subevent_type_mixin_with_init(self): @runtime_checkable class YT(SctEventProtocol, Protocol): - S: Type[SctEventProtocol] + S: type[SctEventProtocol] class Y(SctEvent): - T: Type[YT] + T: type[YT] # pylint: disable=too-few-public-methods class Mixin: @@ -354,7 +358,7 @@ class YT(SctEventProtocol, Protocol): attr1: str class Y(SctEvent): - T: Type[YT] + T: type[YT] def __init__(self, attr1): self.attr1 = attr1 @@ -375,7 +379,7 @@ def test_add_subevent_type_pickle(self): global Y # pylint: disable=global-variable-not-assigned; assigned by class definition # noqa: PLW0603 class Y(SctEvent): - T: Type[SctEvent] + T: type[SctEvent] Y.add_subevent_type("T") @@ -557,7 +561,7 @@ class Y(LogEvent): def test_msgfmt(self): class Y(LogEvent): - T: Type[LogEventProtocol] + T: type[LogEventProtocol] Y.add_subevent_type("T", regex="r1") diff --git a/unit_tests/test_sct_events_continuous_events_registry.py b/unit_tests/test_sct_events_continuous_events_registry.py index 0a9700c4d74..12df328870c 100644 --- a/unit_tests/test_sct_events_continuous_events_registry.py +++ b/unit_tests/test_sct_events_continuous_events_registry.py @@ -1,15 +1,23 @@ # pylint: disable=no-self-use import uuid +from collections.abc import Generator from pathlib import Path -from typing import Generator import pytest from sdcm.sct_events import Severity -from sdcm.sct_events.continuous_event import ContinuousEventsRegistry, ContinuousEventRegistryException -from sdcm.sct_events.database import get_pattern_to_event_to_func_mapping, CompactionEvent, \ - IndexSpecialColumnErrorEvent, ScyllaServerStatusEvent, DatabaseLogEvent +from sdcm.sct_events.continuous_event import ( + ContinuousEventRegistryException, + ContinuousEventsRegistry, +) +from sdcm.sct_events.database import ( + CompactionEvent, + DatabaseLogEvent, + IndexSpecialColumnErrorEvent, + ScyllaServerStatusEvent, + get_pattern_to_event_to_func_mapping, +) from sdcm.sct_events.loaders import GeminiStressEvent from sdcm.sct_events.nemesis import DisruptionEvent from sdcm.sct_events.nodetool import NodetoolEvent diff --git a/unit_tests/test_sct_events_database.py b/unit_tests/test_sct_events_database.py index 3a4745e597d..39951733903 100644 --- a/unit_tests/test_sct_events_database.py +++ b/unit_tests/test_sct_events_database.py @@ -11,13 +11,18 @@ # # Copyright (c) 2020 ScyllaDB -import unittest import re +import unittest from sdcm.sct_events import Severity from sdcm.sct_events.base import LogEvent -from sdcm.sct_events.database import \ - DatabaseLogEvent, FullScanEvent, IndexSpecialColumnErrorEvent, TOLERABLE_REACTOR_STALL, SYSTEM_ERROR_EVENTS +from sdcm.sct_events.database import ( + SYSTEM_ERROR_EVENTS, + TOLERABLE_REACTOR_STALL, + DatabaseLogEvent, + FullScanEvent, + IndexSpecialColumnErrorEvent, +) class TestDatabaseLogEvent(unittest.TestCase): diff --git a/unit_tests/test_sct_events_events_analyzer.py b/unit_tests/test_sct_events_events_analyzer.py index d3b400f431a..8962f497920 100644 --- a/unit_tests/test_sct_events_events_analyzer.py +++ b/unit_tests/test_sct_events_events_analyzer.py @@ -15,11 +15,10 @@ import unittest import unittest.mock -from sdcm.sct_events.system import InfoEvent, SpotTerminationEvent -from sdcm.sct_events.setup import EVENTS_SUBSCRIBERS_START_DELAY from sdcm.sct_events.events_analyzer import EventsAnalyzer, start_events_analyzer from sdcm.sct_events.events_processes import EVENTS_ANALYZER_ID, get_events_process - +from sdcm.sct_events.setup import EVENTS_SUBSCRIBERS_START_DELAY +from sdcm.sct_events.system import InfoEvent, SpotTerminationEvent from unit_tests.lib.events_utils import EventsUtilsMixin diff --git a/unit_tests/test_sct_events_events_device.py b/unit_tests/test_sct_events_events_device.py index 883429cc457..0d2173500ad 100644 --- a/unit_tests/test_sct_events_events_device.py +++ b/unit_tests/test_sct_events_events_device.py @@ -12,15 +12,19 @@ # Copyright (c) 2020 ScyllaDB import ctypes +import multiprocessing import shutil import tempfile -import unittest import threading -import multiprocessing +import unittest -from sdcm.sct_events.health import ClusterHealthValidatorEvent -from sdcm.sct_events.events_device import EventsDevice, start_events_main_device, get_events_main_device +from sdcm.sct_events.events_device import ( + EventsDevice, + get_events_main_device, + start_events_main_device, +) from sdcm.sct_events.events_processes import EventsProcessesRegistry +from sdcm.sct_events.health import ClusterHealthValidatorEvent from sdcm.wait import wait_for diff --git a/unit_tests/test_sct_events_events_handler.py b/unit_tests/test_sct_events_events_handler.py index 8a0d7527997..48d1188d708 100644 --- a/unit_tests/test_sct_events_events_handler.py +++ b/unit_tests/test_sct_events_events_handler.py @@ -14,9 +14,8 @@ import unittest import unittest.mock - from sdcm.sct_events.event_handler import start_events_handler -from sdcm.sct_events.events_processes import get_events_process, EVENTS_HANDLER_ID +from sdcm.sct_events.events_processes import EVENTS_HANDLER_ID, get_events_process from sdcm.sct_events.loaders import CassandraStressLogEvent from sdcm.sct_events.setup import EVENTS_SUBSCRIBERS_START_DELAY from sdcm.test_config import TestConfig diff --git a/unit_tests/test_sct_events_events_processes.py b/unit_tests/test_sct_events_events_processes.py index fca009bada5..32cff5554be 100644 --- a/unit_tests/test_sct_events_events_processes.py +++ b/unit_tests/test_sct_events_events_processes.py @@ -15,8 +15,11 @@ import unittest.mock from pathlib import Path -from sdcm.sct_events.events_processes import \ - EventsProcessesRegistry, create_default_events_process_registry, get_default_events_process_registry +from sdcm.sct_events.events_processes import ( + EventsProcessesRegistry, + create_default_events_process_registry, + get_default_events_process_registry, +) class FakeProcess: # pylint: disable=too-few-public-methods diff --git a/unit_tests/test_sct_events_file_logger.py b/unit_tests/test_sct_events_file_logger.py index 1b1e5d7f608..cc34966d4cf 100644 --- a/unit_tests/test_sct_events_file_logger.py +++ b/unit_tests/test_sct_events_file_logger.py @@ -15,11 +15,15 @@ import unittest from sdcm.sct_events import Severity -from sdcm.sct_events.system import SpotTerminationEvent +from sdcm.sct_events.file_logger import ( + EventsFileLogger, + get_events_grouped_by_category, + get_events_logger, + get_logger_event_summary, + start_events_logger, +) from sdcm.sct_events.setup import EVENTS_SUBSCRIBERS_START_DELAY -from sdcm.sct_events.file_logger import \ - EventsFileLogger, start_events_logger, get_events_logger, get_events_grouped_by_category, get_logger_event_summary - +from sdcm.sct_events.system import SpotTerminationEvent from unit_tests.lib.events_utils import EventsUtilsMixin diff --git a/unit_tests/test_sct_events_filters.py b/unit_tests/test_sct_events_filters.py index a6d12ffd91d..7baddadc62b 100644 --- a/unit_tests/test_sct_events_filters.py +++ b/unit_tests/test_sct_events_filters.py @@ -11,13 +11,17 @@ # # Copyright (c) 2020 ScyllaDB -import re import pickle +import re import unittest from sdcm.sct_events import Severity -from sdcm.sct_events.filters import DbEventsFilter, EventsFilter, EventsSeverityChangerFilter from sdcm.sct_events.database import DatabaseLogEvent +from sdcm.sct_events.filters import ( + DbEventsFilter, + EventsFilter, + EventsSeverityChangerFilter, +) class TestDbEventsFilter(unittest.TestCase): diff --git a/unit_tests/test_sct_events_grafana.py b/unit_tests/test_sct_events_grafana.py index 333fd1052b4..5a6b4cf2a91 100644 --- a/unit_tests/test_sct_events_grafana.py +++ b/unit_tests/test_sct_events_grafana.py @@ -16,8 +16,11 @@ import unittest.mock from sdcm.sct_events import Severity -from sdcm.sct_events.health import ClusterHealthValidatorEvent -from sdcm.sct_events.setup import EVENTS_SUBSCRIBERS_START_DELAY +from sdcm.sct_events.events_processes import ( + EVENTS_GRAFANA_AGGREGATOR_ID, + EVENTS_GRAFANA_ANNOTATOR_ID, + get_events_process, +) from sdcm.sct_events.grafana import ( GrafanaAnnotator, GrafanaEventAggregator, @@ -27,10 +30,9 @@ start_grafana_pipeline, start_posting_grafana_annotations, ) -from sdcm.sct_events.events_processes import \ - EVENTS_GRAFANA_ANNOTATOR_ID, EVENTS_GRAFANA_AGGREGATOR_ID, get_events_process +from sdcm.sct_events.health import ClusterHealthValidatorEvent +from sdcm.sct_events.setup import EVENTS_SUBSCRIBERS_START_DELAY from sdcm.wait import wait_for - from unit_tests.lib.events_utils import EventsUtilsMixin # pylint: disable=protected-access diff --git a/unit_tests/test_sct_events_health.py b/unit_tests/test_sct_events_health.py index 6c0aba55b39..785704d9d85 100644 --- a/unit_tests/test_sct_events_health.py +++ b/unit_tests/test_sct_events_health.py @@ -13,8 +13,8 @@ import pickle import unittest -from sdcm.sct_events import Severity +from sdcm.sct_events import Severity from sdcm.sct_events.health import ClusterHealthValidatorEvent, DataValidatorEvent diff --git a/unit_tests/test_sct_events_loaders.py b/unit_tests/test_sct_events_loaders.py index 950eda64f14..8d708bfdb66 100644 --- a/unit_tests/test_sct_events_loaders.py +++ b/unit_tests/test_sct_events_loaders.py @@ -13,17 +13,29 @@ import pickle import unittest - from itertools import chain from invoke.runners import Result from sdcm.sct_events import Severity from sdcm.sct_events.base import LogEvent -from sdcm.sct_events.loaders import \ - GeminiStressEvent, CassandraStressEvent, ScyllaBenchEvent, YcsbStressEvent, NdBenchStressEvent, CDCReaderStressEvent, \ - KclStressEvent, CassandraStressLogEvent, ScyllaBenchLogEvent, GeminiStressLogEvent, \ - CS_ERROR_EVENTS, CS_NORMAL_EVENTS, SCYLLA_BENCH_ERROR_EVENTS, CS_ERROR_EVENTS_PATTERNS, CS_NORMAL_EVENTS_PATTERNS +from sdcm.sct_events.loaders import ( + CS_ERROR_EVENTS, + CS_ERROR_EVENTS_PATTERNS, + CS_NORMAL_EVENTS, + CS_NORMAL_EVENTS_PATTERNS, + SCYLLA_BENCH_ERROR_EVENTS, + CassandraStressEvent, + CassandraStressLogEvent, + CDCReaderStressEvent, + GeminiStressEvent, + GeminiStressLogEvent, + KclStressEvent, + NdBenchStressEvent, + ScyllaBenchEvent, + ScyllaBenchLogEvent, + YcsbStressEvent, +) class TestGeminiEvent(unittest.TestCase): diff --git a/unit_tests/test_sct_events_monitors.py b/unit_tests/test_sct_events_monitors.py index 3913468b814..1bcda0c08b6 100644 --- a/unit_tests/test_sct_events_monitors.py +++ b/unit_tests/test_sct_events_monitors.py @@ -17,7 +17,6 @@ from sdcm.sct_events import Severity from sdcm.sct_events.monitors import PrometheusAlertManagerEvent - RAW_ALERT = dict( annotations=dict( description="[10.0.201.178] has been down for more than 30 seconds.", diff --git a/unit_tests/test_sct_events_operator.py b/unit_tests/test_sct_events_operator.py index cf837b3bc27..6282c191203 100644 --- a/unit_tests/test_sct_events_operator.py +++ b/unit_tests/test_sct_events_operator.py @@ -11,10 +11,10 @@ # # Copyright (c) 2020 ScyllaDB +import datetime import pickle import re import unittest -import datetime from sdcm.sct_events.base import LogEvent from sdcm.sct_events.operator import ScyllaOperatorLogEvent @@ -37,7 +37,7 @@ def test_scylla_operator_log_event_tls_handshake_error(self): event.event_id = "9bb2980a-5940-49a7-8b08-d5c323b46aa9" expected_timestamp = datetime.datetime( year=datetime.datetime.now().year, month=6, day=28, hour=15, minute=53, second=2, - microsecond=269804, tzinfo=datetime.timezone.utc).timestamp() + microsecond=269804, tzinfo=datetime.UTC).timestamp() assert event.timestamp == expected_timestamp self.assertEqual( '(ScyllaOperatorLogEvent Severity.WARNING) period_type=one-time ' @@ -63,7 +63,7 @@ def test_scylla_operator_log_event_operator_started_info(self): event.event_id = "9bb2980a-5940-49a7-8b08-d5c323b46aa9" expected_timestamp = datetime.datetime( year=datetime.datetime.now().year, month=6, day=28, hour=16, minute=7, second=43, - microsecond=572294, tzinfo=datetime.timezone.utc).timestamp() + microsecond=572294, tzinfo=datetime.UTC).timestamp() assert event.timestamp == expected_timestamp self.assertEqual( '(ScyllaOperatorLogEvent Severity.NORMAL) period_type=one-time ' diff --git a/unit_tests/test_sct_events_system.py b/unit_tests/test_sct_events_system.py index f8b4e53f0a3..e343a29b811 100644 --- a/unit_tests/test_sct_events_system.py +++ b/unit_tests/test_sct_events_system.py @@ -11,15 +11,24 @@ # # Copyright (c) 2020 ScyllaDB import os - import pickle import unittest from textwrap import dedent from sdcm.sct_events import Severity -from sdcm.sct_events.system import \ - StartupTestEvent, TestFrameworkEvent, ElasticsearchEvent, SpotTerminationEvent, ScyllaRepoEvent, InfoEvent, \ - ThreadFailedEvent, CoreDumpEvent, TestResultEvent, InstanceStatusEvent, INSTANCE_STATUS_EVENTS_PATTERNS +from sdcm.sct_events.system import ( + INSTANCE_STATUS_EVENTS_PATTERNS, + CoreDumpEvent, + ElasticsearchEvent, + InfoEvent, + InstanceStatusEvent, + ScyllaRepoEvent, + SpotTerminationEvent, + StartupTestEvent, + TestFrameworkEvent, + TestResultEvent, + ThreadFailedEvent, +) class TestSystemEvents(unittest.TestCase): diff --git a/unit_tests/test_scylla_yaml.py b/unit_tests/test_scylla_yaml.py index a528924d035..3f58033ea6d 100644 --- a/unit_tests/test_scylla_yaml.py +++ b/unit_tests/test_scylla_yaml.py @@ -15,7 +15,12 @@ import yaml -from sdcm.provision.scylla_yaml import ServerEncryptionOptions, ClientEncryptionOptions, SeedProvider, ScyllaYaml +from sdcm.provision.scylla_yaml import ( + ClientEncryptionOptions, + ScyllaYaml, + SeedProvider, + ServerEncryptionOptions, +) from sdcm.provision.scylla_yaml.auxiliaries import RequestSchedulerOptions diff --git a/unit_tests/test_scylla_yaml_builders.py b/unit_tests/test_scylla_yaml_builders.py index 7fce0c9cf27..28a143a98e3 100644 --- a/unit_tests/test_scylla_yaml_builders.py +++ b/unit_tests/test_scylla_yaml_builders.py @@ -16,20 +16,21 @@ import os import tempfile import unittest -from typing import List from unittest.mock import patch from parameterized import parameterized -from sdcm.cluster import BaseNode, BaseCluster, BaseScyllaCluster -from sdcm.provision.scylla_yaml import ScyllaYamlNodeAttrBuilder, ScyllaYamlClusterAttrBuilder, ScyllaYaml +from sdcm.cluster import BaseCluster, BaseNode, BaseScyllaCluster +from sdcm.provision.scylla_yaml import ( + ScyllaYaml, + ScyllaYamlClusterAttrBuilder, + ScyllaYamlNodeAttrBuilder, +) from sdcm.provision.scylla_yaml.auxiliaries import ScyllaYamlAttrBuilderBase from sdcm.sct_config import SCTConfiguration from sdcm.test_config import TestConfig - from sdcm.utils.distro import Distro - __builtin__ = [].__class__.__module__ @@ -399,7 +400,7 @@ def __getattr__(self, item): class DummyCluster(BaseScyllaCluster, BaseCluster): # pylint: disable=too-few-public-methods - nodes: List['DummyNode'] + nodes: list['DummyNode'] def __init__(self, params): # pylint: disable=super-init-not-called self.nodes = [] diff --git a/unit_tests/test_sdcm_mgmt_common.py b/unit_tests/test_sdcm_mgmt_common.py index 58dd385d12e..4bdcba5dc56 100644 --- a/unit_tests/test_sdcm_mgmt_common.py +++ b/unit_tests/test_sdcm_mgmt_common.py @@ -1,4 +1,4 @@ -from sdcm.mgmt.common import get_manager_scylla_backend, get_manager_repo_from_defaults +from sdcm.mgmt.common import get_manager_repo_from_defaults, get_manager_scylla_backend from sdcm.utils.distro import Distro diff --git a/unit_tests/test_seed_selector.py b/unit_tests/test_seed_selector.py index a0106ed27da..5520fdf7adc 100644 --- a/unit_tests/test_seed_selector.py +++ b/unit_tests/test_seed_selector.py @@ -1,8 +1,8 @@ -import unittest -import tempfile import logging -import shutil import os.path +import shutil +import tempfile +import unittest import sdcm.cluster from sdcm.test_config import TestConfig diff --git a/unit_tests/test_send_email.py b/unit_tests/test_send_email.py index 803e458a113..ae053893911 100644 --- a/unit_tests/test_send_email.py +++ b/unit_tests/test_send_email.py @@ -1,7 +1,7 @@ import os -import unittest -import tempfile import shutil +import tempfile +import unittest import zipfile from sdcm.send_email import LongevityEmailReporter, read_email_data_from_file diff --git a/unit_tests/test_sla_utils.py b/unit_tests/test_sla_utils.py index 79e54360444..087af585565 100644 --- a/unit_tests/test_sla_utils.py +++ b/unit_tests/test_sla_utils.py @@ -4,12 +4,14 @@ from typing import NamedTuple from sdcm.cluster import BaseNode +from sdcm.sla.libs.sla_utils import SchedulerRuntimeUnexpectedValue, SlaUtils from sdcm.utils.distro import Distro -from sdcm.utils.loader_utils import (STRESS_ROLE_NAME_TEMPLATE, - STRESS_ROLE_PASSWORD_TEMPLATE, - SERVICE_LEVEL_NAME_TEMPLATE) -from sdcm.sla.libs.sla_utils import SlaUtils, SchedulerRuntimeUnexpectedValue -from test_lib.sla import Role, UserRoleBase, ServiceLevel +from sdcm.utils.loader_utils import ( + SERVICE_LEVEL_NAME_TEMPLATE, + STRESS_ROLE_NAME_TEMPLATE, + STRESS_ROLE_PASSWORD_TEMPLATE, +) +from test_lib.sla import Role, ServiceLevel, UserRoleBase from unit_tests.test_cluster import DummyDbCluster logging.basicConfig(level=logging.DEBUG) diff --git a/unit_tests/test_test_lib_cql_types.py b/unit_tests/test_test_lib_cql_types.py index df61051cbcf..21de1ca0913 100644 --- a/unit_tests/test_test_lib_cql_types.py +++ b/unit_tests/test_test_lib_cql_types.py @@ -1,5 +1,11 @@ import unittest -from test_lib.cql_types import CQLTypeBuilder, NOT_EMBEDDABLE_COLUMN_TYPES, COLLECTION_COLUMN_TYPES, ALL_COLUMN_TYPES + +from test_lib.cql_types import ( + ALL_COLUMN_TYPES, + COLLECTION_COLUMN_TYPES, + NOT_EMBEDDABLE_COLUMN_TYPES, + CQLTypeBuilder, +) class CQLColumnTypeTest(unittest.TestCase): diff --git a/unit_tests/test_tester.py b/unit_tests/test_tester.py index c93490e7136..70c0efe7f8e 100644 --- a/unit_tests/test_tester.py +++ b/unit_tests/test_tester.py @@ -11,22 +11,22 @@ # # Copyright (c) 2020 ScyllaDB +import logging import os import shutil -import logging import tempfile import time import unittest.mock from time import sleep +from sdcm.sct_config import SCTConfiguration from sdcm.sct_events import Severity +from sdcm.sct_events.events_processes import EventsProcessesRegistry +from sdcm.sct_events.file_logger import get_events_grouped_by_category from sdcm.sct_events.health import ClusterHealthValidatorEvent -from sdcm.tester import ClusterTester, silence, TestResultEvent -from sdcm.sct_config import SCTConfiguration -from sdcm.utils.log import MultilineMessagesFormatter, configure_logging from sdcm.sct_events.system import TestFrameworkEvent -from sdcm.sct_events.file_logger import get_events_grouped_by_category -from sdcm.sct_events.events_processes import EventsProcessesRegistry +from sdcm.tester import ClusterTester, TestResultEvent, silence +from sdcm.utils.log import MultilineMessagesFormatter, configure_logging class FakeSCTConfiguration(SCTConfiguration): diff --git a/unit_tests/test_utils_common.py b/unit_tests/test_utils_common.py index b6e53f7ea33..7ce5ebab83f 100644 --- a/unit_tests/test_utils_common.py +++ b/unit_tests/test_utils_common.py @@ -11,17 +11,17 @@ # # Copyright (c) 2020 ScyllaDB -import os import hashlib -import shutil import logging +import os +import shutil import unittest import unittest.mock from pathlib import Path from sdcm.cluster import BaseNode -from sdcm.utils.distro import Distro from sdcm.utils.common import convert_metric_to_ms, download_dir_from_cloud +from sdcm.utils.distro import Distro from sdcm.utils.sstable import load_inventory from sdcm.utils.sstable.load_utils import SstableLoadUtils diff --git a/unit_tests/test_utils_data_validator.py b/unit_tests/test_utils_data_validator.py index 2e5e6e4d6d3..9810716ecff 100644 --- a/unit_tests/test_utils_data_validator.py +++ b/unit_tests/test_utils_data_validator.py @@ -14,8 +14,8 @@ import pytest -from sdcm.utils.data_validator import LongevityDataValidator from sdcm import sct_config +from sdcm.utils.data_validator import LongevityDataValidator @dataclass diff --git a/unit_tests/test_utils_distro.py b/unit_tests/test_utils_distro.py index 884d9257071..fe7e75aff9d 100644 --- a/unit_tests/test_utils_distro.py +++ b/unit_tests/test_utils_distro.py @@ -11,13 +11,11 @@ # # Copyright (c) 2020 ScyllaDB -from __future__ import absolute_import import unittest from sdcm.utils.distro import Distro, DistroError - DISTROS_OS_RELEASE = { "Debian 10": """\ PRETTY_NAME="Debian GNU/Linux 10 (buster)" diff --git a/unit_tests/test_utils_docker.py b/unit_tests/test_utils_docker.py index 5d0fbf1be19..f1e62263c90 100644 --- a/unit_tests/test_utils_docker.py +++ b/unit_tests/test_utils_docker.py @@ -14,15 +14,22 @@ # pylint: disable=too-few-public-methods,too-many-statements,protected-access,attribute-defined-outside-init # pylint: disable=too-many-public-methods,too-many-instance-attributes -from __future__ import absolute_import import os import unittest -from unittest.mock import Mock, patch, mock_open, sentinel, call from collections import namedtuple - -from sdcm.utils.docker_utils import _Name, ContainerManager, \ - DockerException, NotFound, ImageNotFound, NullResource, Retry, ContainerAlreadyRegistered +from unittest.mock import Mock, call, mock_open, patch, sentinel + +from sdcm.utils.docker_utils import ( + ContainerAlreadyRegistered, + ContainerManager, + DockerException, + ImageNotFound, + NotFound, + NullResource, + Retry, + _Name, +) class DummyDockerClient: diff --git a/unit_tests/test_utils_k8s.py b/unit_tests/test_utils_k8s.py index 3de51a54e2f..36ad0adc919 100644 --- a/unit_tests/test_utils_k8s.py +++ b/unit_tests/test_utils_k8s.py @@ -7,7 +7,6 @@ ScyllaPodsIPChangeTrackerThread, ) - BASE_HELM_VALUES = { "no_nesting_key": "no_nesting_value", "nested_dict": { diff --git a/unit_tests/test_utils_scylla_args.py b/unit_tests/test_utils_scylla_args.py index ad63679fd74..34acd14179a 100644 --- a/unit_tests/test_utils_scylla_args.py +++ b/unit_tests/test_utils_scylla_args.py @@ -13,8 +13,7 @@ import unittest -from sdcm.utils.scylla_args import ScyllaArgParser, ScyllaArgError - +from sdcm.utils.scylla_args import ScyllaArgError, ScyllaArgParser SCYLLA_HELP_EXAMPLE = """\ Scylla version 4.0.0-0.20200505.d95aa77b62 with build-id c9cf14d33523aca01a1ba502a91eb9d400de9c34 starting ... diff --git a/unit_tests/test_version_utils.py b/unit_tests/test_version_utils.py index a8faca8ee60..47cdd22a7d6 100644 --- a/unit_tests/test_version_utils.py +++ b/unit_tests/test_version_utils.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import os import unittest @@ -9,6 +8,13 @@ import sdcm from sdcm.utils.version_utils import ( + SCYLLA_VERSION_GROUPED_RE, + VERSION_NOT_FOUND_ERROR, + ComparableScyllaOperatorVersion, + ComparableScyllaVersion, + MethodVersionNotFound, + RepositoryDetails, + ScyllaFileType, assume_version, get_all_versions, get_branch_version, @@ -19,13 +25,6 @@ get_specific_tag_of_docker_image, is_enterprise, scylla_versions, - ComparableScyllaOperatorVersion, - ComparableScyllaVersion, - MethodVersionNotFound, - RepositoryDetails, - ScyllaFileType, - SCYLLA_VERSION_GROUPED_RE, - VERSION_NOT_FOUND_ERROR, ) BASE_S3_DOWNLOAD_URL = 'https://s3.amazonaws.com/downloads.scylladb.com' diff --git a/unit_tests/test_wait.py b/unit_tests/test_wait.py index 6f089231235..521149d03cc 100644 --- a/unit_tests/test_wait.py +++ b/unit_tests/test_wait.py @@ -12,18 +12,21 @@ # Copyright (c) 2023 ScyllaDB # pylint: disable=all -from __future__ import absolute_import import logging import threading import time import unittest - from concurrent.futures import ThreadPoolExecutor import pytest from sdcm.cluster import BaseNode -from sdcm.wait import wait_for, wait_for_log_lines, WaitForTimeoutError, ExitByEventError +from sdcm.wait import ( + ExitByEventError, + WaitForTimeoutError, + wait_for, + wait_for_log_lines, +) logging.basicConfig(level=logging.DEBUG) diff --git a/unit_tests/test_ycsb_thread.py b/unit_tests/test_ycsb_thread.py index c5c428060e5..d82a12ce83c 100644 --- a/unit_tests/test_ycsb_thread.py +++ b/unit_tests/test_ycsb_thread.py @@ -22,7 +22,7 @@ from sdcm.utils.docker_utils import running_in_docker from sdcm.ycsb_thread import YcsbStressThread from unit_tests.dummy_remote import LocalLoaderSetDummy -from unit_tests.lib.alternator_utils import ALTERNATOR_PORT, ALTERNATOR, TEST_PARAMS +from unit_tests.lib.alternator_utils import ALTERNATOR, ALTERNATOR_PORT, TEST_PARAMS pytestmark = [ pytest.mark.usefixtures("events", "create_table", "create_cql_ks_and_table"), @@ -97,7 +97,7 @@ def cleanup_thread(): @timeout(timeout=60) def check_metrics(): - output = requests.get("http://{}/metrics".format(prom_address)).text + output = requests.get(f"http://{prom_address}/metrics").text regex = re.compile(r"^sct_ycsb_read_gauge.*?([0-9\.]*?)$", re.MULTILINE) assert "sct_ycsb_read_gauge" in output assert "sct_ycsb_update_gauge" in output @@ -156,7 +156,7 @@ def cleanup_thread2(): # 4. wait for expected metrics to be available @timeout(timeout=120) def check_metrics(): - output = requests.get("http://{}/metrics".format(prom_address)).text + output = requests.get(f"http://{prom_address}/metrics").text regex = re.compile(r"^sct_ycsb_verify_gauge.*?([0-9\.]*?)$", re.MULTILINE) assert "sct_ycsb_verify_gauge" in output @@ -199,7 +199,7 @@ def cleanup_thread(): @timeout(timeout=60) def check_metrics(): - output = requests.get("http://{}/metrics".format(prom_address)).text + output = requests.get(f"http://{prom_address}/metrics").text regex = re.compile(r"^sct_ycsb_read_gauge.*?([0-9\.]*?)$", re.MULTILINE) assert "sct_ycsb_read_gauge" in output assert "sct_ycsb_update_gauge" in output diff --git a/upgrade_schema_test.py b/upgrade_schema_test.py index 9954b245d58..7ef9867495d 100644 --- a/upgrade_schema_test.py +++ b/upgrade_schema_test.py @@ -22,11 +22,11 @@ from cassandra.cluster import Cluster from thrift.transport import TSocket +from thrift_bindings.v22.Cassandra import * +from thrift_bindings.v22.ttypes import NotFoundException from sdcm.nemesis import UpgradeNemesisOneNode from sdcm.tester import ClusterTester -from thrift_bindings.v22.Cassandra import * -from thrift_bindings.v22.ttypes import NotFoundException tests = [] ks_name = 'test_upgrade_schema_ks' @@ -303,7 +303,7 @@ def test_upgrade_schema(self): # upgrade all the nodes in random order for i in indexes: self.db_cluster.node_to_upgrade = self.db_cluster.nodes[i] - self.log.info("started upgrade node {0}".format(self.db_cluster.node_to_upgrade)) + self.log.info(f"started upgrade node {self.db_cluster.node_to_upgrade}") self.db_cluster.add_nemesis(nemesis=UpgradeNemesisOneNode, tester_obj=self) self.db_cluster.start_nemesis(interval=15) diff --git a/upgrade_test.py b/upgrade_test.py index 90addb0cc82..dddef4a6600 100644 --- a/upgrade_test.py +++ b/upgrade_test.py @@ -14,16 +14,16 @@ # Copyright (c) 2016 ScyllaDB # pylint: disable=too-many-lines -import traceback -from itertools import zip_longest, chain, count as itertools_count +import contextlib import json -from pathlib import Path import random -import time import re -from functools import wraps, cache -from typing import List -import contextlib +import time +import traceback +from functools import cache, wraps +from itertools import chain, zip_longest +from itertools import count as itertools_count +from pathlib import Path import cassandra import tenacity @@ -33,22 +33,26 @@ from sdcm.cluster import BaseNode from sdcm.fill_db_data import FillDatabaseData from sdcm.sct_events import Severity +from sdcm.sct_events.database import ( + DatabaseLogEvent, + IndexSpecialColumnErrorEvent, +) +from sdcm.sct_events.filters import EventsSeverityChangerFilter +from sdcm.sct_events.group_common_events import ( + decorate_with_context, + ignore_abort_requested_errors, + ignore_upgrade_schema_errors, + ignore_ycsb_connection_refused, +) +from sdcm.sct_events.system import InfoEvent from sdcm.stress_thread import CassandraStressThread +from sdcm.utils import loader_utils from sdcm.utils.user_profile import get_profile_content from sdcm.utils.version_utils import ( - get_node_supported_sstable_versions, ComparableScyllaVersion, + get_node_supported_sstable_versions, is_enterprise, ) -from sdcm.sct_events.system import InfoEvent -from sdcm.sct_events.database import ( - IndexSpecialColumnErrorEvent, - DatabaseLogEvent, -) -from sdcm.sct_events.filters import EventsSeverityChangerFilter -from sdcm.sct_events.group_common_events import ignore_upgrade_schema_errors, ignore_ycsb_connection_refused, \ - ignore_abort_requested_errors, decorate_with_context -from sdcm.utils import loader_utils from test_lib.sla import create_sla_auth NUMBER_OF_ROWS_FOR_TRUNCATE_TEST = 10 @@ -271,7 +275,7 @@ def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_v new_is_enterprise = 'scylla-enterprise' in result.stdout scylla_pkg = 'scylla-enterprise' if new_is_enterprise else 'scylla' - ver_suffix = r'\*{}'.format(new_version) if new_version else '' + ver_suffix = rf'\*{new_version}' if new_version else '' scylla_pkg_ver = f"{scylla_pkg}{ver_suffix}" if orig_is_enterprise != new_is_enterprise: self.upgrade_rollback_mode = 'reinstall' @@ -282,15 +286,15 @@ def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_v if node.distro.is_rhel_like: InfoEvent(message='upgrade_node - starting to remove and install scylla on RHEL').publish() node.remoter.run(r'sudo yum remove scylla\* -y') - node.remoter.run('sudo yum install {} -y'.format(scylla_pkg_ver)) + node.remoter.run(f'sudo yum install {scylla_pkg_ver} -y') InfoEvent(message='upgrade_node - ended to remove and install scylla on RHEL').publish() else: InfoEvent(message='upgrade_node - starting to remove and install scylla on debian').publish() node.remoter.run(r'sudo apt-get remove scylla\* -y') # fixme: add publick key node.remoter.run( - r'sudo apt-get install {} -y ' - r'-o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" '.format(scylla_pkg_ver)) + rf'sudo apt-get install {scylla_pkg_ver} -y ' + r'-o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" ') InfoEvent(message='upgrade_node - ended to remove and install scylla on debian').publish() InfoEvent(message='upgrade_node - starting to "recover_conf"').publish() recover_conf(node) @@ -301,14 +305,14 @@ def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_v else: # noqa: PLR5501 if node.distro.is_rhel_like: InfoEvent(message='upgrade_node - starting to "yum update"').publish() - node.remoter.run(r'sudo yum update {}\* -y'.format(scylla_pkg_ver)) + node.remoter.run(rf'sudo yum update {scylla_pkg_ver}\* -y') InfoEvent(message='upgrade_node - ended to "yum update"').publish() else: InfoEvent(message='upgrade_node - starting to "apt-get update"').publish() node.remoter.run('sudo apt-get update') node.remoter.run( - r'sudo apt-get dist-upgrade {} -y ' - r'-o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" '.format(scylla_pkg)) + rf'sudo apt-get dist-upgrade {scylla_pkg} -y ' + r'-o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" ') InfoEvent(message='upgrade_node - ended to "apt-get update"').publish() InfoEvent(message='upgrade_node - starting to "check_reload_systemd_config"').publish() check_reload_systemd_config(node) @@ -426,7 +430,7 @@ def _rollback_node(self, node, upgrade_sstables=True): # noqa: PLR0912, PLR0915 node.run_scylla_sysconfig_setup() node.start_scylla_server(verify_up_timeout=500) - InfoEvent(message='original scylla-server version is %s, latest: %s' % (orig_ver, new_ver)).publish() + InfoEvent(message=f'original scylla-server version is {orig_ver}, latest: {new_ver}').publish() assert orig_ver != new_ver, "scylla-server version isn't changed" if upgrade_sstables: @@ -489,7 +493,7 @@ def wait_for_node_to_finish(): sstable_versions = {sstable_version_regex.search(f).group( 1) for f in all_sstable_files if sstable_version_regex.search(f)} - assert len(sstable_versions) == 1, "expected all table format to be the same found {}".format(sstable_versions) + assert len(sstable_versions) == 1, f"expected all table format to be the same found {sstable_versions}" assert list(sstable_versions)[0] == self.expected_sstable_format_version, ( "expected to format version to be '{}', found '{}'".format( self.expected_sstable_format_version, list(sstable_versions)[0])) @@ -575,7 +579,7 @@ def fill_and_verify_db_data(self, note, pre_fill=False, rewrite_data=True): # in base and this view is not backing a secondary index) # @staticmethod def search_for_idx_token_error_after_upgrade(self, node, step): - self.log.debug('Search for idx_token error. Step {}'.format(step)) + self.log.debug(f'Search for idx_token error. Step {step}') idx_token_error = list(node.follow_system_log( patterns=["Column idx_token doesn't exist"], start_from_beginning=True)) if idx_token_error: @@ -585,7 +589,7 @@ def search_for_idx_token_error_after_upgrade(self, node, step): ).publish() @staticmethod - def shuffle_nodes_and_alternate_dcs(nodes: List[BaseNode]) -> List[BaseNode]: + def shuffle_nodes_and_alternate_dcs(nodes: list[BaseNode]) -> list[BaseNode]: """Shuffles provided nodes based on dc (alternates dc's). based on https://stackoverflow.com/a/21482016/3891194""" dc_nodes = {} @@ -771,7 +775,7 @@ def test_rolling_upgrade(self): # pylint: disable=too-many-locals,too-many-stat if all(upgradesstables): InfoEvent(message='Upgrading sstables if new version is available').publish() tables_upgraded = self.db_cluster.run_func_parallel(func=self.wait_for_sstable_upgrade) - assert all(tables_upgraded), "Failed to upgrade the sstable format {}".format(tables_upgraded) + assert all(tables_upgraded), f"Failed to upgrade the sstable format {tables_upgraded}" # Verify sstabledump / scylla sstable dump-data InfoEvent(message='Starting sstabledump to verify correctness of sstables').publish() @@ -849,17 +853,17 @@ def test_rolling_upgrade(self): # pylint: disable=too-many-locals,too-many-stat workload_prioritization_error_num = self.count_log_errors(search_pattern='workload prioritization.*read_failure_exception', step=step, search_for_idx_token_error=False) - InfoEvent(message='schema_load_error_num: %s; workload_prioritization_error_num: %s' % - (schema_load_error_num, workload_prioritization_error_num)).publish() + InfoEvent(message=f'{schema_load_error_num=}; ' + f'{workload_prioritization_error_num=}' + ).publish() # Issue #https://github.com/scylladb/scylla-enterprise/issues/1391 # By Eliran's comment: For 'Failed to load schema version' error which is expected and non offensive is # to count the 'workload prioritization' warning and subtract that amount from the amount of overall errors. load_error_num = schema_load_error_num - workload_prioritization_error_num assert load_error_num <= error_factor * 8 * \ - len(self.db_cluster.nodes), 'Only allowing shards_num * %d schema load errors per host during the ' \ - 'entire test, actual: %d' % ( - error_factor, schema_load_error_num) + len(self.db_cluster.nodes), f'Only allowing shards_num * {error_factor} schema load errors per host during the ' \ + 'entire test, actual: {schema_load_error_num}' InfoEvent(message='Step10 - Verify that gemini did not failed during upgrade').publish() if self.version_cdc_support(): @@ -870,10 +874,10 @@ def test_rolling_upgrade(self): # pylint: disable=too-many-locals,too-many-stat def _start_and_wait_for_node_upgrade(self, node: BaseNode, step: int) -> None: InfoEvent( message=f"Step {step} - Upgrade {node.name} from dc {node.dc_idx}").publish() - InfoEvent(message='Upgrade Node %s begins' % node.name).publish() + InfoEvent(message=f'Upgrade Node {node.name} begins').publish() with ignore_ycsb_connection_refused(): self.upgrade_node(node, upgrade_sstables=self.params.get('upgrade_sstables')) - InfoEvent(message='Upgrade Node %s ended' % node.name).publish() + InfoEvent(message=f'Upgrade Node {node.name} ended').publish() node.check_node_health() def _start_and_wait_for_node_rollback(self, node: BaseNode, step: int) -> None: @@ -1060,8 +1064,8 @@ def test_kubernetes_scylla_upgrade(self): step='AFTER UPGRADE', search_for_idx_token_error=False ) - InfoEvent(message='schema_load_error_num: %s; workload_prioritization_error_num: %s' % - (schema_load_error_num, workload_prioritization_error_num)).publish() + InfoEvent( + message=f'schema_load_error_num: {schema_load_error_num}; workload_prioritization_error_num: {workload_prioritization_error_num}').publish() # Issue #https://github.com/scylladb/scylla-enterprise/issues/1391 # By Eliran's comment: For 'Failed to load schema version' error which is expected and non offensive is @@ -1446,6 +1450,6 @@ def _custom_profile_rolling_upgrade(self, cs_user_profiles, new_scylla_repo=None if all(upgradesstables): InfoEvent(message='Upgrading sstables if new version is available').publish() tables_upgraded = self.db_cluster.run_func_parallel(func=self.wait_for_sstable_upgrade) - assert all(tables_upgraded), "Failed to upgrade the sstable format {}".format(tables_upgraded) + assert all(tables_upgraded), f"Failed to upgrade the sstable format {tables_upgraded}" InfoEvent(message='all nodes were upgraded, and last workaround is verified.').publish() diff --git a/utils/build_system/create_test_release_jobs.py b/utils/build_system/create_test_release_jobs.py index f8ad9fbda4e..3d081f17929 100644 --- a/utils/build_system/create_test_release_jobs.py +++ b/utils/build_system/create_test_release_jobs.py @@ -18,7 +18,6 @@ from sdcm.wait import wait_for - DIR_TEMPLATE = Path(__file__).parent.joinpath("folder-template.xml").read_text(encoding="utf-8") JOB_TEMPLATE = Path(__file__).parent.joinpath("template.xml").read_text(encoding="utf-8") LOGGER = logging.getLogger(__name__) diff --git a/utils/decode_kallsyms.py b/utils/decode_kallsyms.py index c3eef31bc78..73292518eab 100755 --- a/utils/decode_kallsyms.py +++ b/utils/decode_kallsyms.py @@ -23,7 +23,7 @@ def cut_addresses_from_log_line_kernel_callstack(line): def get_lines_from_log(file_path): file_content = {} - with open(file_path, 'r') as file: # pylint: disable=unspecified-encoding + with open(file_path) as file: # pylint: disable=unspecified-encoding for line in file: if ('kernel callstack' in line and '0x' in line) or 'Reactor stalled for' in line: file_content[line] = cut_addresses_from_log_line_kernel_callstack(line) @@ -31,7 +31,7 @@ def get_lines_from_log(file_path): def get_kallsyms(path_to_kallsyms): - with open(path_to_kallsyms, 'r') as file: # pylint: disable=unspecified-encoding + with open(path_to_kallsyms) as file: # pylint: disable=unspecified-encoding file_content = file.readlines() return file_content diff --git a/utils/fix_es_mapping.py b/utils/fix_es_mapping.py index c15686baac6..19f65aa67a1 100755 --- a/utils/fix_es_mapping.py +++ b/utils/fix_es_mapping.py @@ -1,10 +1,9 @@ #!/usr/bin/env python -from __future__ import print_function -import sys import os.path +import sys -import requests import click +import requests from sdcm.keystore import KeyStore @@ -33,7 +32,7 @@ def fix_es_mapping(index_name): print(res.text) res.raise_for_status() - click.secho("fixed {index_name}".format(index_name=index_name), fg='green') + click.secho(f"fixed {index_name}", fg='green') if __name__ == '__main__': diff --git a/utils/get_supported_scylla_base_versions.py b/utils/get_supported_scylla_base_versions.py index f8c3f5d9c53..c295b9d9913 100755 --- a/utils/get_supported_scylla_base_versions.py +++ b/utils/get_supported_scylla_base_versions.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 import logging -import sys -import re import os +import re +import sys -from sdcm.utils.version_utils import is_enterprise, get_all_versions -from sdcm.utils.version_utils import ComparableScyllaVersion, get_s3_scylla_repos_mapping +from sdcm.utils.version_utils import ( + ComparableScyllaVersion, + get_all_versions, + get_s3_scylla_repos_mapping, + is_enterprise, +) sys.path.append(os.path.join(os.path.dirname(__file__), "..")) diff --git a/utils/migrate_nemesis_data.py b/utils/migrate_nemesis_data.py index 21c5518cb3d..9a83f3de552 100755 --- a/utils/migrate_nemesis_data.py +++ b/utils/migrate_nemesis_data.py @@ -1,13 +1,14 @@ #!/usr/bin/env python -import sys -import os.path import datetime import logging import logging.config +import os.path +import sys import click from elasticsearch import Elasticsearch from elasticsearch.helpers import scan + from sdcm.keystore import KeyStore sys.path.append(os.path.join(os.path.dirname(__file__), "..")) diff --git a/utils/mocks/aws_mock.py b/utils/mocks/aws_mock.py index c71110c15d8..02bec20074d 100644 --- a/utils/mocks/aws_mock.py +++ b/utils/mocks/aws_mock.py @@ -12,16 +12,15 @@ # Copyright (c) 2021 ScyllaDB import logging +from collections import namedtuple +from itertools import chain from pathlib import Path from textwrap import dedent -from itertools import chain -from collections import namedtuple from sdcm.utils.common import list_resources_docker from sdcm.utils.docker_utils import ContainerManager, DockerException from sdcm.utils.get_username import get_username - AWS_MOCK_IP_FILE = Path("aws_mock_ip") AWS_MOCK_NODE_TYPE = "aws-mock" AWS_MOCK_IMAGE = "scylladb/aws_mock:latest" diff --git a/utils/split_sct_log.py b/utils/split_sct_log.py index 03a3934c2d8..201b429cc28 100755 --- a/utils/split_sct_log.py +++ b/utils/split_sct_log.py @@ -32,7 +32,7 @@ def get_time(line_content): input_dir = Path.cwd() Path(input_dir / "parts").mkdir(exist_ok=True) -with open(input_dir / "sct.log", "r", encoding="utf-8") as sct_log: +with open(input_dir / "sct.log", encoding="utf-8") as sct_log: idx = 0 # pylint: disable=invalid-name line = sct_log.readline() time = get_time(line) diff --git a/ycsb_performance_regression_test.py b/ycsb_performance_regression_test.py index a1627c4855e..e9443cdbc93 100644 --- a/ycsb_performance_regression_test.py +++ b/ycsb_performance_regression_test.py @@ -20,7 +20,6 @@ from performance_regression_test import PerformanceRegressionTest from sdcm.sct_events.system import InfoEvent - LOGGER = logging.getLogger(__name__) @@ -105,7 +104,7 @@ def test_latency(self): for workload in self.ycsb_workloads: self.wait_no_compactions_running() self.run_fstrim_on_all_db_nodes() - InfoEvent(message="Starting YCSB %s (%s)" % (workload.name, workload.detailed_name)).publish() + InfoEvent(message=f"Starting YCSB {workload.name} ({workload.detailed_name})").publish() self.run_workload(stress_cmd=self._create_stress_cmd(workload), sub_type=workload.sub_type)