feature(pre-commit): replace pylint with ruff scylladb#5799

Ruff is rust base linter for python that work incredibly fast since pylint speed is causing use issue on SCT CI we should start using ruff, for now it uses the pylint set of rules it has implemeted and we can slow extand it's configuration to use more of the available rules set it has Ref: https://github.com/charliermarsh/ruff Ref: https://www.youtube.com/watch?v=jeoL4qsSLbE
fruch · Dec 17, 2023 · bbebcf3 · bbebcf3
1 parent 9016c20
commit bbebcf3
Show file tree

Hide file tree

Showing 143 changed files with 721 additions and 788 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -39,12 +39,12 @@ repos:
         types: [python]
         exclude: '\.sh$'
 
-    -   id: pylint
-        name: pylint
-        entry: pylint -j 2 -d consider-using-f-string
+    -   id: ruff
+        name: ruff
+        entry: ruff --force-exclude --fix
         language: system
         exclude: ^docker/alternator-dns/.*$
-        types: [python]
+        'types_or': [python, pyi]
 
 - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
   rev: v5.0.0

diff --git a/artifacts_test.py b/artifacts_test.py
@@ -299,7 +299,7 @@ def run_pre_create_schema(self, replication_factor=1):
                                   compaction=compaction_strategy, sstable_size=sstable_size)
 
     # pylint: disable=too-many-statements,too-many-branches
-    def test_scylla_service(self):
+    def test_scylla_service(self):  # noqa: PLR0915
 
         self.run_pre_create_schema()
 

diff --git a/cdc_replication_test.py b/cdc_replication_test.py
@@ -227,7 +227,7 @@ def test_replication_longevity(self) -> None:
 
     # pylint: disable=too-many-statements,too-many-branches,too-many-locals
 
-    def test_replication(self, is_gemini_test: bool, mode: Mode) -> None:
+    def test_replication(self, is_gemini_test: bool, mode: Mode) -> None:  # noqa: PLR0915
         assert is_gemini_test or (mode == Mode.DELTA), "cassandra-stress doesn't work with preimage/postimage modes"
 
         self.consistency_ok = False

diff --git a/docker/alternator-dns/dns_server.py b/docker/alternator-dns/dns_server.py
@@ -20,7 +20,7 @@
 
 
 def livenodes_update():
-    global alternator_port
+    global alternator_port  # noqa: PLW0602
     global livenodes
     while True:
         # Contact one of the already known nodes by random, to fetch a new

diff --git a/docker/env/version b/docker/env/version
@@ -1 +1 @@
-1.54-update-k8s-components-1
+1.55-introduce-ruff
diff --git a/functional_tests/scylla_operator/conftest.py b/functional_tests/scylla_operator/conftest.py
@@ -70,7 +70,7 @@ def publish_test_result():
 
 @pytest.fixture(autouse=True, scope='package', name="tester")
 def fixture_tester() -> ScyllaOperatorFunctionalClusterTester:
-    global TESTER  # pylint: disable=global-statement
+    global TESTER  # pylint: disable=global-statement  # noqa: PLW0603
     os.chdir(sct_abs_path())
     tester_inst = ScyllaOperatorFunctionalClusterTester()
     TESTER = tester_inst  # putting tester global, so we can report skipped test (one with mark.skip)

diff --git a/functional_tests/scylla_operator/libs/helpers.py b/functional_tests/scylla_operator/libs/helpers.py
@@ -100,7 +100,7 @@ def scylla_services_names(db_cluster: ScyllaPodCluster) -> list:
             if name not in ('NAME', f"{scylla_cluster_name}-client")]
 
 
-def wait_for_resource_absence(db_cluster: ScyllaPodCluster,  # pylint: disable=too-many-arguments
+def wait_for_resource_absence(db_cluster: ScyllaPodCluster,  # pylint: disable=too-many-arguments  # noqa: PLR0913
                               resource_type: str, resource_name: str,
                               namespace: str = SCYLLA_NAMESPACE,
                               step: int = 2, timeout: int = 60) -> None:

diff --git a/functional_tests/scylla_operator/test_functional.py b/functional_tests/scylla_operator/test_functional.py
@@ -94,7 +94,7 @@ def test_single_operator_image_tag_is_everywhere(db_cluster):
 
 
 @pytest.mark.required_operator("v1.11.0")
-def test_deploy_quasi_multidc_db_cluster(db_cluster: ScyllaPodCluster):  # pylint: disable=too-many-locals,too-many-statements,too-many-branches
+def test_deploy_quasi_multidc_db_cluster(db_cluster: ScyllaPodCluster):  # pylint: disable=too-many-locals,too-many-statements,too-many-branches  # noqa: PLR0915
     """
     Deploy 2 'ScyllaCluster' K8S objects in 2 different namespaces in the single K8S cluster
     and combine them into a single DB cluster.

diff --git a/longevity_test.py b/longevity_test.py
@@ -113,7 +113,7 @@ def _run_validate_large_collections_warning_in_logs(self, node):
         if not res:
             InfoEvent("Did not find expected log message warning: {}".format(msg), severity=Severity.ERROR)
 
-    def test_custom_time(self):
+    def test_custom_time(self):  # noqa: PLR0912, PLR0915
         """
         Run cassandra-stress with params defined in data_dir/scylla.yaml
         """
@@ -327,7 +327,7 @@ def chunks(_list, chunk_size):
                 self._pre_create_templated_user_schema(batch_start=extra_tables_idx,
                                                        batch_end=extra_tables_idx+num_of_newly_created_tables)
                 for i in range(num_of_newly_created_tables):
-                    batch += self.create_templated_user_stress_params(extra_tables_idx + i, cs_profile=cs_profile)
+                    batch.append(self.create_templated_user_stress_params(extra_tables_idx + i, cs_profile=cs_profile))
 
             nodes_ips = self.all_node_ips_for_stress_command
             for params in batch:

diff --git a/mgmt_cli_test.py b/mgmt_cli_test.py
@@ -172,7 +172,7 @@ def restore_backup_from_backup_task(self, mgr_cluster, backup_task, keyspace_and
                             keyspace_and_table_list=keyspace_and_table_list)
 
     # pylint: disable=too-many-arguments
-    def verify_backup_success(self, mgr_cluster, backup_task, keyspace_name='keyspace1', tables_names=None,
+    def verify_backup_success(self, mgr_cluster, backup_task, keyspace_name='keyspace1', tables_names=None,  # noqa: PLR0913
                               truncate=True, restore_data_with_task=False, timeout=None):
         if tables_names is None:
             tables_names = ['standard1']
@@ -188,7 +188,7 @@ def verify_backup_success(self, mgr_cluster, backup_task, keyspace_name='keyspac
             self.restore_backup_from_backup_task(mgr_cluster=mgr_cluster, backup_task=backup_task,
                                                  keyspace_and_table_list=per_keyspace_tables_dict)
 
-    def restore_backup_with_task(self, mgr_cluster, snapshot_tag, timeout, restore_schema=False, restore_data=False,
+    def restore_backup_with_task(self, mgr_cluster, snapshot_tag, timeout, restore_schema=False, restore_data=False,  # noqa: PLR0913
                                  location_list=None):
         location_list = location_list if location_list else self.locations
         restore_task = mgr_cluster.create_restore_task(restore_schema=restore_schema, restore_data=restore_data,
@@ -866,6 +866,7 @@ def test_repair_multiple_keyspace_types(self):  # pylint: disable=invalid-name
             keyspace_repair_percentage = per_keyspace_progress.get(keyspace_name, None)
             assert keyspace_repair_percentage is not None, \
                 "The keyspace {} was not included in the repair!".format(keyspace_name)
+
             assert keyspace_repair_percentage == 100, \
                 "The repair of the keyspace {} stopped at {}%".format(
                     keyspace_name, keyspace_repair_percentage)

diff --git a/mgmt_upgrade_test.py b/mgmt_upgrade_test.py
@@ -52,7 +52,7 @@ def _create_and_add_cluster(self):
                                                auth_token=self.monitors.mgmt_auth_token)
         return mgr_cluster, current_manager_version
 
-    def test_upgrade(self):  # pylint: disable=too-many-locals,too-many-statements
+    def test_upgrade(self):  # pylint: disable=too-many-locals,too-many-statements  # noqa: PLR0915
         manager_node = self.monitors.nodes[0]
 
         target_upgrade_server_version = self.params.get('target_scylla_mgmt_server_address')
@@ -277,9 +277,8 @@ def validate_previous_task_details(task, previous_task_details):
             # and as a result it could be a BIT imprecise
             if abs(delta.total_seconds()) > 60:
                 mismatched_details_name_list.append(detail_name)
-        else:
-            if current_value != previous_task_details[detail_name]:
-                mismatched_details_name_list.append(detail_name)
+        elif current_value != previous_task_details[detail_name]:
+            mismatched_details_name_list.append(detail_name)
     complete_error_description = _create_mismatched_details_error_message(previous_task_details,
                                                                           current_task_details,
                                                                           mismatched_details_name_list)

diff --git a/performance_regression_alternator_test.py b/performance_regression_alternator_test.py
@@ -27,7 +27,7 @@ def __init__(self, *args):
         self.stack.enter_context(ignore_alternator_client_errors())
         self.stack.enter_context(ignore_operation_errors())
 
-    def _workload(self, stress_cmd, stress_num, test_name=None, sub_type=None, keyspace_num=1, prefix='', debug_message='',  # pylint: disable=too-many-arguments,arguments-differ
+    def _workload(self, stress_cmd, stress_num, test_name=None, sub_type=None, keyspace_num=1, prefix='', debug_message='',  # pylint: disable=too-many-arguments,arguments-differ  # noqa: PLR0913
                   save_stats=True, is_alternator=True):
         if not is_alternator:
             stress_cmd = stress_cmd.replace('dynamodb', 'cassandra-cql')

diff --git a/performance_regression_cdc_test.py b/performance_regression_cdc_test.py
@@ -152,7 +152,7 @@ def cdc_workflow(self, use_cdclog_reader=False):  # pylint: disable=unused-varia
 
         self.check_regression_with_baseline(subtest_baseline="cdc_disabled")
 
-    def _workload_cdc(self, stress_cmd, stress_num, test_name, sub_type=None,  # pylint: disable=too-many-arguments
+    def _workload_cdc(self, stress_cmd, stress_num, test_name, sub_type=None,  # pylint: disable=too-many-arguments  # noqa: PLR0913
                       save_stats=True, read_cdclog_cmd=None, update_cdclog_stats=False, enable_batching=True):
         cdc_stress_queue = None
 

diff --git a/performance_regression_gradual_grow_throughput.py b/performance_regression_gradual_grow_throughput.py
@@ -129,7 +129,7 @@ def preload_data(self, compaction_strategy=None):
         self.log.info("Dataset has been populated")
 
     # pylint: disable=too-many-arguments,too-many-locals
-    def run_gradual_increase_load(self, stress_cmd_templ,
+    def run_gradual_increase_load(self, stress_cmd_templ,  # noqa: PLR0913
                                   start_ops, max_ops, throttle_step,
                                   stress_num, num_loaders, compaction_strategy, test_name):
         self.warmup_cache(compaction_strategy)

diff --git a/performance_regression_row_level_repair_test.py b/performance_regression_row_level_repair_test.py
@@ -80,7 +80,7 @@ def preload_data(self, consistency_level=None):
 
                 for stress_cmd in prepare_write_cmd:
                     if consistency_level:
-                        stress_cmd = self._update_cl_in_stress_cmd(
+                        stress_cmd = self._update_cl_in_stress_cmd(  # noqa: PLW2901
                             str_stress_cmd=stress_cmd, consistency_level=consistency_level)
                     params.update({'stress_cmd': stress_cmd})
 

diff --git a/performance_regression_test.py b/performance_regression_test.py
@@ -179,7 +179,7 @@ def display_results(self, results, test_name=''):
             self.log.debug('Failed to display results: {0}'.format(results))
             self.log.debug('Exception: {0}'.format(ex))
 
-    def _workload(self, stress_cmd, stress_num, test_name, sub_type=None, keyspace_num=1, prefix='', debug_message='',  # pylint: disable=too-many-arguments
+    def _workload(self, stress_cmd, stress_num, test_name, sub_type=None, keyspace_num=1, prefix='', debug_message='',  # pylint: disable=too-many-arguments  # noqa: PLR0913
                   save_stats=True):
         if debug_message:
             self.log.debug(debug_message)

diff --git a/performance_search_max_throughput_test.py b/performance_search_max_throughput_test.py
@@ -47,7 +47,7 @@ def test_search_best_mixed_throughput(self):  # pylint: disable=too-many-locals
 
         self.run_search_best_performance(**stress_params)
 
-    def run_search_best_performance(self, stress_cmd_tmpl: str,  # pylint: disable=too-many-arguments,too-many-locals,too-many-statements
+    def run_search_best_performance(self, stress_cmd_tmpl: str,  # pylint: disable=too-many-arguments,too-many-locals,too-many-statements  # noqa: PLR0913, PLR0915
                                     stress_num: int,
                                     stress_num_step: int,
                                     stress_step_duration: str,

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,6 @@
+[tool.ruff]
+select = ["PL"]
+
+ignore = ["E501", "PLR2004"]
+
+target-version = "py311"
diff --git a/requirements.in b/requirements.in
@@ -31,7 +31,7 @@ python-jenkins==1.7.0
 ssh2-python==1.0.0
 argus-alm==0.11.7
 parameterized==0.8.1
-pylint==2.11.1  # Needed for pre-commit hooks
+ruff==0.1.8  # Needed for pre-commit hooks
 autopep8==1.5.7  # Needed for pre-commit hooks
 kubernetes==24.2.0
 packaging==21.3