From 9c6aa8a76de24df75de73218c917571346cc509c Mon Sep 17 00:00:00 2001 From: Vinay Chella Date: Wed, 6 Feb 2019 22:57:53 -0800 Subject: [PATCH] Fix failing upgrade test - test_simple_bootstrap_mixed_versions Adds force_3_0_protocol_version flag to upgrades 3.0.14 upwards which fixes schema migrations during mixed version bootstrap Patch by Vinay Chella; reviewed by Ariel Weisberg for CASSANDRA-15016 --- bootstrap_test.py | 34 +++++++++++++++++-------- dtest.py | 29 +++++++++++++++++++++ upgrade_tests/bootstrap_upgrade_test.py | 5 +++- 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/bootstrap_test.py b/bootstrap_test.py index e33749e7ee..940bef5dd3 100644 --- a/bootstrap_test.py +++ b/bootstrap_test.py @@ -15,7 +15,7 @@ import pytest -from dtest import Tester, create_ks, create_cf +from dtest import Tester, create_ks, create_cf, data_size from tools.assertions import (assert_almost_equal, assert_bootstrap_state, assert_not_running, assert_one, assert_stderr_clean) from tools.data import query_c1c2 @@ -82,16 +82,16 @@ def default_bootstrap(cluster, token): create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) # record the size before inserting any of our own data - empty_size = node1.data_size() - logger.debug("node1 empty size : %s" % float(empty_size)) + empty_size = data_size(node1, 'ks','cf') + logger.debug("node1 empty size for ks.cf: %s" % float(empty_size)) insert_statement = session.prepare("INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')") execute_concurrent_with_args(session, insert_statement, [['k%d' % k] for k in range(keys)]) node1.flush() node1.compact() - initial_size = node1.data_size() - logger.debug("node1 size before bootstrapping node2: %s" % float(initial_size)) + initial_size = data_size(node1,'ks','cf') + logger.debug("node1 size for ks.cf before bootstrapping node2: %s" % float(initial_size)) # Reads inserted data all during the bootstrap process. We shouldn't # get any error @@ -103,14 +103,14 @@ def default_bootstrap(cluster, token): node2.compact() node1.cleanup() - logger.debug("node1 size after cleanup: %s" % float(node1.data_size())) + logger.debug("node1 size for ks.cf after cleanup: %s" % float(data_size(node1,'ks','cf'))) node1.compact() - logger.debug("node1 size after compacting: %s" % float(node1.data_size())) + logger.debug("node1 size for ks.cf after compacting: %s" % float(data_size(node1,'ks','cf'))) - logger.debug("node2 size after compacting: %s" % float(node2.data_size())) + logger.debug("node2 size for ks.cf after compacting: %s" % float(data_size(node2,'ks','cf'))) - size1 = float(node1.data_size()) - size2 = float(node2.data_size()) + size1 = float(data_size(node1,'ks','cf')) + size2 = float(data_size(node2,'ks','cf')) assert_almost_equal(size1, size2, error=0.3) assert_almost_equal(float(initial_size - empty_size), 2 * (size1 - float(empty_size))) @@ -140,6 +140,20 @@ def bootstrap_on_write_survey_and_join(cluster, token): self._base_bootstrap_test(bootstrap_on_write_survey_and_join) + def _test_bootstrap_with_compatibility_flag_on(self, bootstrap_from_version): + def bootstrap_with_compatibility_flag_on(cluster, token): + node2 = new_node(cluster) + node2.set_configuration_options(values={'initial_token': token}) + # cassandra.force_3_0_protocol_version parameter is needed to allow schema + # changes during the bootstrapping for upgrades from 3.0.14+ to anything upwards for 3.0.x or 3.x clusters. + # @jira_ticket CASSANDRA-13004 for detailed context on `cassandra.force_3_0_protocol_version` flag + node2.start(jvm_args=["-Dcassandra.force_3_0_protocol_version=true"], + wait_for_binary_proto=True) + return node2 + + self._base_bootstrap_test(bootstrap_with_compatibility_flag_on, + bootstrap_from_version=bootstrap_from_version) + @since('3.10') @pytest.mark.no_vnodes def test_simple_bootstrap_small_keepalive_period(self): diff --git a/dtest.py b/dtest.py index ec79771abf..d6aff2d60b 100644 --- a/dtest.py +++ b/dtest.py @@ -384,6 +384,35 @@ def private_auth(node_ip): return private_auth +def data_size(node, ks, cf): + """ + Return the size in bytes for given table in a node. + This gets the size from nodetool cfstats output. + This might brake if the format of nodetool cfstats change + as it is looking for specific text "Space used (total)" in output. + @param node: Node in which table size to be checked for + @param ks: Keyspace name for the table + @param cf: table name + @return: data size in bytes + """ + cfstats = node.nodetool("cfstats {}.{}".format(ks,cf))[0] + regex = re.compile(r'[\t]') + stats_lines = [regex.sub("", s) for s in cfstats.split('\n') + if regex.sub("", s).startswith('Space used (total)')] + if not len(stats_lines) == 1: + msg = ('Expected output from `nodetool cfstats` to contain exactly 1 ' + 'line starting with "Space used (total)". Found:\n') + cfstats + raise RuntimeError(msg) + space_used_line = stats_lines[0].split() + + if len(space_used_line) == 4: + return float(space_used_line[3]) + else: + msg = ('Expected format for `Space used (total)` in nodetool cfstats is `Space used (total): `.' + 'Found:\n') + stats_lines[0] + raise RuntimeError(msg) + + def get_port_from_node(node): """ Return the port that this node is listening on. diff --git a/upgrade_tests/bootstrap_upgrade_test.py b/upgrade_tests/bootstrap_upgrade_test.py index efe8ae41f1..20352e59d7 100644 --- a/upgrade_tests/bootstrap_upgrade_test.py +++ b/upgrade_tests/bootstrap_upgrade_test.py @@ -17,4 +17,7 @@ class TestBootstrapUpgrade(TestBootstrap): @pytest.mark.no_vnodes @since('3.10', max_version='3.99') def test_simple_bootstrap_mixed_versions(self): - self._base_bootstrap_test(bootstrap_from_version="3.5") + # Compatibility flag ensures that bootstrapping gets schema information during + # upgrades from 3.0.14+ to anything upwards for 3.0.x or 3.x clusters. + # @jira_ticket CASSANDRA-13004 for detailed context on `force_3_0_protocol_version` flag + self._test_bootstrap_with_compatibility_flag_on(bootstrap_from_version="3.5")