Skip to content
Permalink
Browse files
Fix Cassandra version families, and more…
 - Remove OVERRIDE_MANIFEST, as it is unused.
 - Upgrade manifest, remove upgrades from unsupported indev versions, and remove upgrades to current if indev is still supported.
 - Correct the enforced protocol version to use v3 for 2.2/3.0 upgrades. (ref: CASSANDRA-15193)
 - Fix broken thrift_upgrade_test.py
 - Fix upgrade_udtfix_test.py (not applicable before 3.11.6, ref: CASSANDRA-15035)
 - Remove hardcoded references to gitbox.a.o
 - Remove unnecessary implicit upgrade scenarios. `--upgrade-target-version-only` only tests upgrades from the target version and to the target version, i.e. those upgrade paths where the target (indev) version was the origin or destination version. Previously the filter was applied only for the version family but not on the variant, pulling in completely unrelated upgrade scenarios.

 patch by Jacek Lewandowski; reviewed by Tomek Łasica, Mick Semb Wever for CASSANDRA-16433
  • Loading branch information
jacek-lewandowski authored and michaelsembwever committed Feb 20, 2021
1 parent 8762267 commit e7f7a593427f83ff7a256d91ee48bbb8380bc788
Show file tree
Hide file tree
Showing 10 changed files with 155 additions and 104 deletions.
@@ -276,11 +276,11 @@ def get_eager_protocol_version(cassandra_version):
Returns the highest protocol version accepted
by the given C* version
"""
if cassandra_version >= '2.2':
if LooseVersion('3.0') <= cassandra_version:
protocol_version = 4
elif cassandra_version >= '2.1':
elif LooseVersion('2.1') <= cassandra_version:
protocol_version = 3
elif cassandra_version >= '2.0':
elif LooseVersion('2.0') <= cassandra_version:
protocol_version = 2
else:
protocol_version = 1
@@ -52,9 +52,10 @@ it is not yet released code.

The second useful concept is that of a __version family__. This simply describes which
'version line' a particular version belongs to. For example, Cassandra 3.0.7 belongs
to the '3.0.x' version family. Organizing specific versions into families allows
to the `CASSANDRA_3_0` version family. Organizing specific versions into families allows
us to generalize about support, so we can say, for example that: "2.2.x versions should be
able to upgrade to 3.0.x versions, or 2.2.x versions can skip 3.0.x versions and upgrade directly to a 3.x version".
able to upgrade to 3.0.x versions, or 2.2.x versions can skip 3.0.x versions and upgrade
directly to a 3.x version".

### Code generation

@@ -29,7 +29,7 @@
from tools.data import rows_to_list
from tools.misc import add_skip
from .upgrade_base import UpgradeTester
from .upgrade_manifest import build_upgrade_pairs
from .upgrade_manifest import build_upgrade_pairs, CASSANDRA_4_0

since = pytest.mark.since
logger = logging.getLogger(__name__)
@@ -39,7 +39,7 @@
class TestCQL(UpgradeTester):

def is_40_or_greater(self):
return self.UPGRADE_PATH.upgrade_meta.family in ('trunk', '4.0')
return LooseVersion(self.UPGRADE_PATH.upgrade_meta.family) >= CASSANDRA_4_0

def test_static_cf(self):
""" Test static CF syntax """
@@ -12,6 +12,7 @@
from tools.assertions import (assert_all, assert_length_equal, assert_none,
assert_one)
from tools.misc import new_node
from upgrade_tests.upgrade_manifest import indev_3_0_x, indev_2_1_x

since = pytest.mark.since
logger = logging.getLogger(__name__)
@@ -39,9 +40,9 @@ def _setup_cluster(self, create_keyspace=True, cluster_options=None):

# Forcing cluster version on purpose
if self.dtest_config.cassandra_version_from_build >= MAJOR_VERSION_4:
cluster.set_install_dir(version="git:cassandra-3.0")
cluster.set_install_dir(version=indev_3_0_x.version)
else:
cluster.set_install_dir(version="git:cassandra-2.1")
cluster.set_install_dir(version=indev_2_1_x.version)
self.fixture_dtest_setup.reinitialize_cluster_for_different_version()
cluster.populate(1).start()

@@ -15,7 +15,7 @@
assert_lists_of_dicts_equal)
from tools.misc import wait_for_agreement, add_skip
from .upgrade_base import UpgradeTester
from .upgrade_manifest import build_upgrade_pairs
from .upgrade_manifest import build_upgrade_pairs, CASSANDRA_4_0

since = pytest.mark.since
logger = logging.getLogger(__name__)
@@ -270,7 +270,7 @@ def test_dense_supercolumn_3_0_created(self):
if node.get_cassandra_version() < '4':
client = get_thrift_client(host, port)
_validate_dense_thrift(client, cf='dense_super_1')
_validate_dense_cql(cursor, cf='dense_super_1', is_version_4_or_greater=node.get_cassandra_version() >= '4')
_validate_dense_cql(cursor, cf='dense_super_1', is_version_4_or_greater=node.get_cassandra_version() >= CASSANDRA_4_0)

def test_dense_supercolumn(self):
cluster = self.prepare()
@@ -317,7 +317,7 @@ def test_dense_supercolumn(self):
_validate_dense_thrift(client, cf='dense_super_1')

cursor = self.patient_cql_connection(node, row_factory=dict_factory)
_validate_dense_cql(cursor, cf='dense_super_1', is_version_4_or_greater=node.get_cassandra_version() >= '4')
_validate_dense_cql(cursor, cf='dense_super_1', is_version_4_or_greater=node.get_cassandra_version() >= CASSANDRA_4_0)

def test_sparse_supercolumn(self):
cluster = self.prepare()
@@ -355,7 +355,7 @@ def test_sparse_supercolumn(self):
_validate_sparse_thrift(client, cf='sparse_super_2')

self.set_node_to_current_version(node)
is_version_4_or_greater = node.get_cassandra_version() >= '4'
is_version_4_or_greater = node.get_cassandra_version() >= CASSANDRA_4_0
#4.0 doesn't support compact storage
if is_version_4_or_greater:
cursor.execute("ALTER TABLE ks.sparse_super_2 DROP COMPACT STORAGE;")
@@ -374,7 +374,7 @@ def test_sparse_supercolumn(self):


@pytest.mark.upgrade_test
@since('4')
@since('4.0')
class TestUpgradeTo40(Tester):
"""
Thrift is dead in 4.0. However, we still want to ensure users that used thrift
@@ -593,6 +593,7 @@ def test_dense_supercolumn(self):

for is_upgraded, cursor in self.do_upgrade(cursor, row_factory=dict_factory, use_thrift=True):
logger.debug("Querying {} node".format("upgraded" if is_upgraded else "old"))
is_version_4_or_greater = node.get_cassandra_version() >= CASSANDRA_4_0
if not is_version_4_or_greater:
client = get_thrift_client(host, port)
_validate_dense_thrift(client)
@@ -630,6 +631,7 @@ def test_dense_supercolumn_with_renames(self):

for is_upgraded, cursor in self.do_upgrade(cursor, row_factory=dict_factory, use_thrift=True):
logger.debug("Querying {} node".format("upgraded" if is_upgraded else "old"))
is_version_4_or_greater = node.get_cassandra_version() >= CASSANDRA_4_0
if not is_version_4_or_greater:
client = get_thrift_client(host, port)
_validate_dense_thrift(client, cf='dense_super_2')
@@ -670,6 +672,7 @@ def test_sparse_supercolumn_with_renames(self):

for is_upgraded, cursor in self.do_upgrade(cursor, row_factory=dict_factory, use_thrift=True):
logger.debug("Querying {} node".format("upgraded" if is_upgraded else "old"))
is_version_4_or_greater = node.get_cassandra_version() >= CASSANDRA_4_0
if not is_version_4_or_greater:
client = get_thrift_client(host, port)
_validate_sparse_thrift(client)
@@ -707,6 +710,7 @@ def test_sparse_supercolumn(self):

for is_upgraded, cursor in self.do_upgrade(cursor, row_factory=dict_factory, use_thrift=True):
logger.debug("Querying {} node".format("upgraded" if is_upgraded else "old"))
is_version_4_or_greater = node.get_cassandra_version() >= CASSANDRA_4_0
if not is_version_4_or_greater:
client = get_thrift_client(host, port)
_validate_sparse_thrift(client, cf='sparse_super_2')
@@ -1,3 +1,5 @@
from distutils.version import LooseVersion

import os
import sys
import time
@@ -8,6 +10,8 @@

from ccmlib.common import get_version_from_build, is_win

from .upgrade_manifest import CASSANDRA_4_0

from dtest import Tester, create_ks

logger = logging.getLogger(__name__)
@@ -261,5 +265,5 @@ def upgrade_version_family(self):
return self.UPGRADE_PATH.upgrade_meta.family

def upgrade_is_version_4_or_greater(self):
upgrade_version = self.upgrade_version_family()
return upgrade_version == 'trunk' or upgrade_version >= '4.0'
upgrade_version = LooseVersion(self.upgrade_version_family())
return upgrade_version >= CASSANDRA_4_0
@@ -19,6 +19,14 @@
VERSION_FAMILY = None
CONFIG = None

# TODO add a new item whenever Cassandra is branched and update TRUNK to the version present in trunk
CASSANDRA_2_0 = '2.0'
CASSANDRA_2_1 = '2.1'
CASSANDRA_2_2 = '2.2'
CASSANDRA_3_0 = '3.0'
CASSANDRA_3_11 = '3.11'
CASSANDRA_4_0 = '4.0'
TRUNK = CASSANDRA_4_0

def is_same_family_current_to_indev(origin, destination):
"""
@@ -76,21 +84,22 @@ def set_version_family():
else:
current_version = get_version_from_build(cassandra_dir)

# TODO add a new item whenever Cassandra is branched
if current_version.vstring.startswith('2.0'):
version_family = '2.0.x'
version_family = CASSANDRA_2_0
elif current_version.vstring.startswith('2.1'):
version_family = '2.1.x'
version_family = CASSANDRA_2_1
elif current_version.vstring.startswith('2.2'):
version_family = '2.2.x'
version_family = CASSANDRA_2_2
elif current_version.vstring.startswith('3.0'):
version_family = '3.0.x'
elif '3.1' <= current_version < '4.0':
version_family = '3.x'
elif '4.0' <= current_version < '4.1':
version_family = 'trunk'
version_family = CASSANDRA_3_0
elif current_version.vstring.startswith('3.11'):
version_family = CASSANDRA_3_11
elif current_version.vstring.startswith('4.0'):
version_family = CASSANDRA_4_0
else:
# when this occurs, it's time to update this manifest a bit!
raise RuntimeError("4.1+ not yet supported on upgrade tests!")
raise RuntimeError("Testing upgrades from/to version %s is not supported. Please use a custom manifest (see upgrade_manifest.py)" % current_version.vstring)

global VERSION_FAMILY
VERSION_FAMILY = version_family
@@ -116,6 +125,14 @@ def matches_current_env_version_family(self):
"""
return self.family == VERSION_FAMILY

@property
def matches_current_env_version_family_and_is_indev(self):
"""
Returns boolean indicating whether this meta matches the current version family of the environment
and whether this meta is in indev variant
"""
return self.family == VERSION_FAMILY and self.variant == "indev"

def clone_with_local_env_version(self):
"""
Returns a new object cloned from this one, with the version replaced with the local env version.
@@ -126,19 +143,20 @@ def clone_with_local_env_version(self):
return self._replace(version="clone:{}".format(cassandra_dir))


indev_2_1_x = VersionMeta(name='indev_2_1_x', family='2.1', variant='indev', version='github:apache/cassandra-2.1', min_proto_v=1, max_proto_v=3, java_versions=(7, 8))
current_2_1_x = VersionMeta(name='current_2_1_x', family='2.1', variant='current', version='2.1.20', min_proto_v=1, max_proto_v=3, java_versions=(7, 8))
# TODO define new versions whenever Cassandra is branched
indev_2_1_x = VersionMeta(name='indev_2_1_x', family=CASSANDRA_2_1, variant='indev', version='github:apache/cassandra-2.1', min_proto_v=1, max_proto_v=3, java_versions=(7, 8))
current_2_1_x = VersionMeta(name='current_2_1_x', family=CASSANDRA_2_1, variant='current', version='2.1.22', min_proto_v=1, max_proto_v=3, java_versions=(7, 8))

indev_2_2_x = VersionMeta(name='indev_2_2_x', family='2.2', variant='indev', version='github:apache/cassandra-2.2', min_proto_v=1, max_proto_v=4, java_versions=(7, 8))
current_2_2_x = VersionMeta(name='current_2_2_x', family='2.2', variant='current', version='2.2.13', min_proto_v=1, max_proto_v=4, java_versions=(7, 8))
indev_2_2_x = VersionMeta(name='indev_2_2_x', family=CASSANDRA_2_2, variant='indev', version='github:apache/cassandra-2.2', min_proto_v=1, max_proto_v=3, java_versions=(7, 8))
current_2_2_x = VersionMeta(name='current_2_2_x', family=CASSANDRA_2_2, variant='current', version='2.2.19', min_proto_v=1, max_proto_v=3, java_versions=(7, 8))

indev_3_0_x = VersionMeta(name='indev_3_0_x', family='3.0', variant='indev', version='github:apache/cassandra-3.0', min_proto_v=3, max_proto_v=4, java_versions=(8,))
current_3_0_x = VersionMeta(name='current_3_0_x', family='3.0', variant='current', version='3.0.23', min_proto_v=3, max_proto_v=4, java_versions=(8,))
indev_3_0_x = VersionMeta(name='indev_3_0_x', family=CASSANDRA_3_0, variant='indev', version='github:apache/cassandra-3.0', min_proto_v=3, max_proto_v=4, java_versions=(8,))
current_3_0_x = VersionMeta(name='current_3_0_x', family=CASSANDRA_3_0, variant='current', version='3.0.24', min_proto_v=3, max_proto_v=4, java_versions=(8,))

indev_3_11_x = VersionMeta(name='indev_3_11_x', family='3.11', variant='indev', version='github:apache/cassandra-3.11', min_proto_v=3, max_proto_v=4, java_versions=(8,))
current_3_11_x = VersionMeta(name='current_3_11_x', family='3.11', variant='current', version='3.11.9', min_proto_v=3, max_proto_v=4, java_versions=(8,))
indev_3_11_x = VersionMeta(name='indev_3_11_x', family=CASSANDRA_3_11, variant='indev', version='github:apache/cassandra-3.11', min_proto_v=3, max_proto_v=4, java_versions=(8,))
current_3_11_x = VersionMeta(name='current_3_11_x', family=CASSANDRA_3_11, variant='current', version='3.11.9', min_proto_v=3, max_proto_v=4, java_versions=(8,))

indev_trunk = VersionMeta(name='indev_trunk', family='trunk', variant='indev', version='github:apache/trunk', min_proto_v=4, max_proto_v=5, java_versions=(8,))
indev_trunk = VersionMeta(name='indev_trunk', family=TRUNK, variant='indev', version='github:apache/trunk', min_proto_v=4, max_proto_v=5, java_versions=(8,))


# MANIFEST maps a VersionMeta representing a line/variant to a list of other VersionMeta's representing supported upgrades
@@ -148,38 +166,18 @@ def clone_with_local_env_version(self):
# 2) Features exclusive to version B may not work until all nodes are running version B.
# 3) Nodes upgraded to version B can read data stored by the predecessor version A, and from a data standpoint will function the same as if they always ran version B.
# 4) If a new sstable format is present in version B, writes will occur in that format after upgrade. Running sstableupgrade on version B will proactively convert version A sstables to version B.
# TODO define new upgrade scenarios whenever Cassandra is branched
MANIFEST = {
indev_2_1_x: [indev_2_2_x, current_2_2_x, indev_3_0_x, current_3_0_x, indev_3_11_x, current_3_11_x],
current_2_1_x: [indev_2_1_x, indev_2_2_x, current_2_2_x, indev_3_0_x, current_3_0_x, indev_3_11_x, current_3_11_x],

indev_2_2_x: [indev_3_0_x, current_3_0_x, indev_3_11_x, current_3_11_x],
current_2_2_x: [indev_2_2_x, indev_3_0_x, current_3_0_x, indev_3_11_x, current_3_11_x],

indev_3_0_x: [indev_3_11_x, current_3_11_x, indev_trunk],
current_3_0_x: [indev_3_0_x, indev_3_11_x, current_3_11_x, indev_trunk],

current_2_1_x: [indev_2_2_x, indev_3_0_x, indev_3_11_x],
current_2_2_x: [indev_2_2_x, indev_3_0_x, indev_3_11_x],
current_3_0_x: [indev_3_0_x, indev_3_11_x, indev_trunk],
current_3_11_x: [indev_3_11_x, indev_trunk],
indev_3_11_x: [indev_trunk]
}

# Local env and custom path testing instructions. Use these steps to REPLACE the normal upgrade test cases with your own.
# 1) Add a VersionMeta for each version you wish to test (see examples below). Update the name, family, version, and protocol restrictions as needed. Use a unique name for each VersionMeta.
# 2) Update OVERRIDE_MANIFEST (see example below).
# 3) If you want to test using local code, set the version attribute using local slugs in the format 'local:/path/to/cassandra/:branch_name'
# 4) Run the tests!
# To run all, use 'nosetests -v upgrade_tests/'. To run specific tests, use 'nosetests -vs --collect-only' to preview the test names, then run nosetests using the desired test name.
# Note that nosetests outputs test names in a format that needs to be tweaked a bit before they will run from the command line.
custom_1 = VersionMeta(name='custom_branch_1', family='2.1.x', variant='indev', version='local:some_branch', min_proto_v=3, max_proto_v=4, java_versions=(7, 8))
custom_2 = VersionMeta(name='custom_branch_2', family='2.2.x', variant='indev', version='git:trunk', min_proto_v=3, max_proto_v=4, java_versions=(7, 8))
custom_3 = VersionMeta(name='custom_branch_3', family='3.0.x', variant='indev', version='git:cassandra-3.5', min_proto_v=3, max_proto_v=4, java_versions=(7, 8))
custom_4 = VersionMeta(name='custom_branch_4', family='3.x', variant='indev', version='git:cassandra-3.6', min_proto_v=3, max_proto_v=4, java_versions=(7, 8))
OVERRIDE_MANIFEST = {
# EXAMPLE:
# custom_1: [custom_2, custom_3], # creates a test of custom_1 -> custom_2, and another test from custom_1 -> custom_3
# custom_3: [custom_4] # creates a test of custom_3 -> custom_4
indev_2_2_x: [indev_3_0_x, indev_3_11_x],
indev_3_0_x: [indev_3_11_x, indev_trunk],
indev_3_11_x: [indev_trunk]
}


def _have_common_proto(origin_meta, destination_meta):
"""
Takes two VersionMeta objects, in order of test from start version to next version.
@@ -194,7 +192,7 @@ def build_upgrade_pairs():
Returns a list of UpgradePath's.
"""
valid_upgrade_pairs = []
manifest = OVERRIDE_MANIFEST or MANIFEST
manifest = MANIFEST

configured_strategy = CONFIG.getoption("--upgrade-version-selection").upper()
version_select_strategy = VersionSelectionStrategies[configured_strategy].value[0]
@@ -215,13 +213,13 @@ def build_upgrade_pairs():

# if either origin or destination match version, then do the test
# the assumption is that a change in 3.0 could break upgrades to trunk, so include those tests as well
if filter_for_current_family and not origin_meta.matches_current_env_version_family and not destination_meta.matches_current_env_version_family:
if filter_for_current_family and not origin_meta.matches_current_env_version_family_and_is_indev and not destination_meta.matches_current_env_version_family:
logger.debug("skipping class creation, origin version {} and destination version {} do not match target version {}, and --upgrade-target-version-only was set".format(origin_meta.name, destination_meta.name, VERSION_FAMILY))
continue

path_name = 'Upgrade_' + origin_meta.name + '_To_' + destination_meta.name

if not (RUN_STATIC_UPGRADE_MATRIX or OVERRIDE_MANIFEST):
if not RUN_STATIC_UPGRADE_MATRIX:
if destination_meta.matches_current_env_version_family:
# looks like this test should actually run in the current env, so let's set the final version to match the env exactly
oldmeta = destination_meta

0 comments on commit e7f7a59

Please sign in to comment.