Fix failing upgrade test - test_simple_bootstrap_mixed_versions

Adds force_3_0_protocol_version flag to upgrades 3.0.14 upwards which fixes schema migrations during mixed version bootstrap

Patch by Vinay Chella; reviewed by Ariel Weisberg for CASSANDRA-15016

bootstrap_test.py

@@ -15,7 +15,7 @@
 
 import pytest
 
-from dtest import Tester, create_ks, create_cf
+from dtest import Tester, create_ks, create_cf, data_size
 from tools.assertions import (assert_almost_equal, assert_bootstrap_state, assert_not_running,
                               assert_one, assert_stderr_clean)
 from tools.data import query_c1c2
@@ -82,16 +82,16 @@ def default_bootstrap(cluster, token):
         create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
 
         # record the size before inserting any of our own data
-        empty_size = node1.data_size()
-        logger.debug("node1 empty size : %s" % float(empty_size))
+        empty_size = data_size(node1, 'ks','cf')
+        logger.debug("node1 empty size for ks.cf: %s" % float(empty_size))
 
         insert_statement = session.prepare("INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')")
         execute_concurrent_with_args(session, insert_statement, [['k%d' % k] for k in range(keys)])
 
         node1.flush()
         node1.compact()
-        initial_size = node1.data_size()
-        logger.debug("node1 size before bootstrapping node2: %s" % float(initial_size))
+        initial_size = data_size(node1,'ks','cf')
+        logger.debug("node1 size for ks.cf before bootstrapping node2: %s" % float(initial_size))
 
         # Reads inserted data all during the bootstrap process. We shouldn't
         # get any error
@@ -103,14 +103,14 @@ def default_bootstrap(cluster, token):
         node2.compact()
 
         node1.cleanup()
-        logger.debug("node1 size after cleanup: %s" % float(node1.data_size()))
+        logger.debug("node1 size for ks.cf after cleanup: %s" % float(data_size(node1,'ks','cf')))
         node1.compact()
-        logger.debug("node1 size after compacting: %s" % float(node1.data_size()))
+        logger.debug("node1 size for ks.cf after compacting: %s" % float(data_size(node1,'ks','cf')))
 
-        logger.debug("node2 size after compacting: %s" % float(node2.data_size()))
+        logger.debug("node2 size for ks.cf after compacting: %s" % float(data_size(node2,'ks','cf')))
 
-        size1 = float(node1.data_size())
-        size2 = float(node2.data_size())
+        size1 = float(data_size(node1,'ks','cf'))
+        size2 = float(data_size(node2,'ks','cf'))
         assert_almost_equal(size1, size2, error=0.3)
         assert_almost_equal(float(initial_size - empty_size), 2 * (size1 - float(empty_size)))
 
@@ -140,6 +140,20 @@ def bootstrap_on_write_survey_and_join(cluster, token):
 
         self._base_bootstrap_test(bootstrap_on_write_survey_and_join)
 
+    def _test_bootstrap_with_compatibility_flag_on(self, bootstrap_from_version):
+        def bootstrap_with_compatibility_flag_on(cluster, token):
+            node2 = new_node(cluster)
+            node2.set_configuration_options(values={'initial_token': token})
+            # cassandra.force_3_0_protocol_version parameter is needed to allow schema
+            # changes during the bootstrapping for upgrades from 3.0.14+ to anything upwards for 3.0.x or 3.x clusters.
+            # @jira_ticket CASSANDRA-13004 for detailed context on `cassandra.force_3_0_protocol_version` flag
+            node2.start(jvm_args=["-Dcassandra.force_3_0_protocol_version=true"],
+                        wait_for_binary_proto=True)
+            return node2
+
+        self._base_bootstrap_test(bootstrap_with_compatibility_flag_on,
+                                      bootstrap_from_version=bootstrap_from_version)
+
     @since('3.10')
     @pytest.mark.no_vnodes
     def test_simple_bootstrap_small_keepalive_period(self):

dtest.py

@@ -384,6 +384,35 @@ def private_auth(node_ip):
     return private_auth
 
 
+def data_size(node, ks, cf):
+    """
+    Return the size in bytes for given table in a node.
+    This gets the size from nodetool cfstats output.
+    This might brake if the format of nodetool cfstats change
+    as it is looking for specific text "Space used (total)" in output.
+    @param node: Node in which table size to be checked for
+    @param ks: Keyspace name for the table
+    @param cf: table name
+    @return: data size in bytes
+    """
+    cfstats = node.nodetool("cfstats {}.{}".format(ks,cf))[0]
+    regex = re.compile(r'[\t]')
+    stats_lines = [regex.sub("", s) for s in cfstats.split('\n')
+                  if regex.sub("", s).startswith('Space used (total)')]
+    if not len(stats_lines) == 1:
+        msg = ('Expected output from `nodetool cfstats` to contain exactly 1 '
+               'line starting with "Space used (total)". Found:\n') + cfstats
+        raise RuntimeError(msg)
+    space_used_line = stats_lines[0].split()
+
+    if len(space_used_line) == 4:
+        return float(space_used_line[3])
+    else:
+        msg = ('Expected format for `Space used (total)` in nodetool cfstats is `Space used (total): <number>`.'
+               'Found:\n') + stats_lines[0]
+        raise RuntimeError(msg)
+
+
 def get_port_from_node(node):
     """
     Return the port that this node is listening on.

upgrade_tests/bootstrap_upgrade_test.py

@@ -17,4 +17,7 @@ class TestBootstrapUpgrade(TestBootstrap):
     @pytest.mark.no_vnodes
     @since('3.10', max_version='3.99')
     def test_simple_bootstrap_mixed_versions(self):
-        self._base_bootstrap_test(bootstrap_from_version="3.5")
+        # Compatibility flag ensures that bootstrapping gets schema information during
+        # upgrades from 3.0.14+ to anything upwards for 3.0.x or 3.x clusters.
+        # @jira_ticket CASSANDRA-13004 for detailed context on `force_3_0_protocol_version` flag
+        self._test_bootstrap_with_compatibility_flag_on(bootstrap_from_version="3.5")