test(tablets): Upgrade to tablets-enabled cluster

The test case starts with scylla 5.4 and during the upgrade process we activate
tablets (and raft).

configurations/rolling-upgrade-with-tablets.yaml

@@ -0,0 +1,8 @@
+stress_before_upgrade: cassandra-stress write no-warmup cl=ALL n=10100200 -schema 'replication(strategy=NetworkTopologyStrategy,replication_factor=3) compaction(strategy=LeveledCompactionStrategy)' -mode cql3 native -rate threads=1000 -pop seq=1..10100200 -log interval=5
+stress_during_entire_upgrade: cassandra-stress write no-warmup cl=QUORUM n=20100200 -schema 'replication(strategy=NetworkTopologyStrategy,replication_factor=3) compaction(strategy=LeveledCompactionStrategy)' -mode cql3 native -rate threads=60 -pop seq=10100201..30200400 -log interval=5
+stress_after_cluster_upgrade: cassandra-stress read no-warmup cl=QUORUM n=30200400 -schema 'replication(strategy=NetworkTopologyStrategy,replication_factor=3)' -mode cql3 native -rate threads=1000 -pop seq=1..30200400 -log interval=5
+
+
+n_db_nodes: 4
+
+enable_tablets_on_upgrade: true

defaults/test_default.yaml

@@ -140,6 +140,7 @@ scylla_version: ''
 test_upgrade_from_installed_3_1_0: false
 target_upgrade_version: ''
 disable_raft: true
+enable_tablets_on_upgrade: false
 
 stress_cdclog_reader_cmd: "cdc-stressor -stream-query-round-duration 30s"
 

docs/configuration_options.md

@@ -275,6 +275,7 @@
 | **<a href="#user-content-new_version" name="new_version">new_version</a>**  | Assign new upgrade version, use it to upgrade to specific minor release. eg: 3.0.1 | N/A | SCT_NEW_VERSION
 | **<a href="#user-content-target_upgrade_version" name="target_upgrade_version">target_upgrade_version</a>**  | Assign target upgrade version, use for decide if the truncate entries test should be run. This test should be performed in case the target upgrade version >= 3.1 | N/A | SCT_TARGET_UPGRADE_VERSION
 | **<a href="#user-content-disable_raft" name="disable_raft">disable_raft</a>**  | As for now, raft will be enable by default in all [upgrade] tests, so this flag will allow usto still run [upgrade] test without raft enabled (or disabling raft), so we will have bettercoverage | True | SCT_DISABLE_RAFT
+| **<a href="#user-content-enable_tablets_on_upgrade" name="enable_tablets_on_upgrade">enable_tablets_on_upgrade</a>**  | Used to enable tablets mid rolling upgrade | False | SCT_ENABLE_TABLETS_ON_UPGRADE
 | **<a href="#user-content-upgrade_node_packages" name="upgrade_node_packages">upgrade_node_packages</a>**  |  | N/A | SCT_UPGRADE_NODE_PACKAGES
 | **<a href="#user-content-test_sst3" name="test_sst3">test_sst3</a>**  |  | N/A | SCT_TEST_SST3
 | **<a href="#user-content-test_upgrade_from_installed_3_1_0" name="test_upgrade_from_installed_3_1_0">test_upgrade_from_installed_3_1_0</a>**  | Enable an option for installed 3.1.0 for work around a scylla issue if it's true | N/A | SCT_TEST_UPGRADE_FROM_INSTALLED_3_1_0

jenkins-pipelines/master-triggers/tablets_weekly_trigger.xml

@@ -27,7 +27,7 @@ email_recipients=qa@scylladb.com,tablets@scylladb.com</properties>
               <textParamValueOnNewLine>false</textParamValueOnNewLine>
             </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
           </configs>
-          <projects>tablets/longevity-100gb-4h-test,tablets/longevity-harry-2h-test,tablets/longevity-large-partition-8h-test,tablets/longevity-multidc-multirack-test</projects>
+          <projects>tablets/longevity-100gb-4h-test,tablets/longevity-harry-2h-test,tablets/longevity-large-partition-8h-test,tablets/longevity-multidc-multirack-test,tablets/rolling-upgrade-activate-tablets-test</projects>
           <condition>SUCCESS</condition>
           <triggerWithNoParameters>false</triggerWithNoParameters>
           <triggerFromChildProjects>false</triggerFromChildProjects>

jenkins-pipelines/oss/tablets/rolling-upgrade-activate-tablets.jenkinsfile

@@ -0,0 +1,14 @@
+#!groovy
+
+// trick from https://github.com/jenkinsci/workflow-cps-global-lib-plugin/pull/43
+def lib = library identifier: 'sct@snapshot', retriever: legacySCM(scm)
+
+rollingUpgradePipeline(
+    backend: 'gce',
+    base_versions: '5.4',
+    linux_distro: 'ubuntu-jammy',
+    use_preinstalled_scylla: true,
+    test_name: 'upgrade_test.UpgradeTest.test_generic_cluster_upgrade',
+    test_config: '''["test-cases/upgrades/generic-rolling-upgrade.yaml", "configurations/rolling-upgrade-with-tablets.yaml"]''',
+    internode_compression: 'all',
+)

sdcm/sct_config.py

@@ -1167,6 +1167,10 @@ class SCTConfiguration(dict):
                   "to still run [upgrade] test without raft enabled (or disabling raft), so we will have better"
                   "coverage"),
 
+        dict(name="enable_tablets_on_upgrade", env="SCT_ENABLE_TABLETS_ON_UPGRADE", type=boolean,
+             help="By default, the tablets feature is disabled. With this parameter, created for the upgrade test,"
+                  "the tablets feature will only be enabled after the upgrade"),
+
         dict(name="upgrade_node_packages", env="SCT_UPGRADE_NODE_PACKAGES", type=str,
              help=""),
 

upgrade_test.py

@@ -206,6 +206,9 @@ def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_v
         if not self.params.get('disable_raft'):
             scylla_yaml_updates.update({"consistent_cluster_management": True})
 
+        if self.params.get("enable_tablets_on_upgrade"):
+            scylla_yaml_updates.update({"experimental_features": ["tablets", "consistent-topology-changes"]})
+
         if self.params.get('test_sst3'):
             scylla_yaml_updates.update({"enable_sstables_mc_format": True})
 
@@ -369,6 +372,15 @@ def _rollback_node(self, node, upgrade_sstables=True):
         node.run_nodetool("snapshot")
         node.stop_scylla_server(verify_down=False)
 
+        if self.params.get("enable_tablets_on_upgrade"):
+            with node.remote_scylla_yaml() as scylla_yml:
+                current_experimental_features = scylla_yml.experimental_features
+                current_experimental_features.remove("tablets")
+                current_experimental_features.remove("consistent-topology-changes")
+                if len(current_experimental_features) == 0:
+                    current_experimental_features = None
+                scylla_yml.experimental_features = current_experimental_features
+
         if node.distro.is_rhel_like:
             node.remoter.run('sudo cp ~/scylla.repo-backup /etc/yum.repos.d/scylla.repo')
             node.remoter.run('sudo chown root.root /etc/yum.repos.d/scylla.repo')