From 680deffe7a08b7bdec4c66126eaf89f9a3f4853a Mon Sep 17 00:00:00 2001 From: "Matthias J. Sax" Date: Fri, 20 Oct 2023 16:20:00 -0700 Subject: [PATCH] KAFKA-15378: fix streams upgrade system test (#14539) Fixing bad test setup. We tried to fix an upgrade bug for FK-joins in 3.1 release, but it later turned out that the PR was not sufficient to fix it. We finally fixed in 3.4 release. This PR updates the system test matrix to only test working versions with FK-joins, limited to available test versions. Reviewers: Guozhang Wang , Hao Li , Mickael Maison --- .../streams/streams_broker_down_resilience_test.py | 2 +- .../tests/streams/streams_upgrade_test.py | 14 ++++++++------ tests/kafkatest/version.py | 6 ++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py b/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py index eea7f56f01b3..92e01b87c75b 100644 --- a/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py +++ b/tests/kafkatest/tests/streams/streams_broker_down_resilience_test.py @@ -100,7 +100,7 @@ def test_streams_runs_with_broker_down_initially(self, metadata_quorum): processor_3 = StreamsBrokerDownResilienceService(self.test_context, self.kafka, configs) processor_3.start() - broker_unavailable_message = "Broker may not be available" + broker_unavailable_message = "Node may not be available" # verify streams instances unable to connect to broker, kept trying self.wait_for_verification(processor, broker_unavailable_message, processor.LOG_FILE, 10) diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py index 9039d9d7473a..955ce0295012 100644 --- a/tests/kafkatest/tests/streams/streams_upgrade_test.py +++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py @@ -40,11 +40,13 @@ metadata_1_versions = [str(LATEST_0_10_0)] metadata_2_versions = [str(LATEST_0_10_1), str(LATEST_0_10_2), str(LATEST_0_11_0), str(LATEST_1_0), str(LATEST_1_1), str(LATEST_2_4), str(LATEST_2_5), str(LATEST_2_6), str(LATEST_2_7), str(LATEST_2_8), - str(LATEST_3_0)] -# upgrading from version (2.4...3.0) is broken and only fixed later in 3.1 -# we cannot test two bounce rolling upgrade because we know it's broken -# instead we add version 2.4...3.0 to the `metadata_2_versions` upgrade list -fk_join_versions = [str(LATEST_3_1), str(LATEST_3_2), str(LATEST_3_3)] + str(LATEST_3_0), str(LATEST_3_1), str(LATEST_3_2), str(LATEST_3_3)] +# upgrading from version (2.4...3.3) is broken and only fixed later in 3.3.3 (unreleased) and 3.4.0 +# -> https://issues.apache.org/jira/browse/KAFKA-14646 +# thus, we cannot test two bounce rolling upgrade because we know it's broken +# instead we add version 2.4...3.3 to the `metadata_2_versions` upgrade list +#fk_join_versions = [str(LATEST_3_4)] + """ After each release one should first check that the released version has been uploaded to @@ -202,7 +204,7 @@ def test_upgrade_downgrade_brokers(self, from_version, to_version): @cluster(num_nodes=6) @matrix(from_version=metadata_1_versions, to_version=[str(DEV_VERSION)]) @matrix(from_version=metadata_2_versions, to_version=[str(DEV_VERSION)]) - @matrix(from_version=fk_join_versions, to_version=[str(DEV_VERSION)]) + #@matrix(from_version=fk_join_versions, to_version=[str(DEV_VERSION)]) def test_rolling_upgrade_with_2_bounces(self, from_version, to_version): """ This test verifies that the cluster successfully upgrades despite changes in the metadata and FK diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py index fdac15fe1720..0eeeab26d63e 100644 --- a/tests/kafkatest/version.py +++ b/tests/kafkatest/version.py @@ -107,8 +107,10 @@ def supports_topic_ids_when_using_zk(self): return self >= V_2_8_0 def supports_fk_joins(self): - # while we support FK joins since 2.4, rolling upgrade is broken in older versions and only fixed in 3.1 - return hasattr(self, "version") and self >= V_3_1_2 + # while we support FK joins since 2.4, rolling upgrade is broken in older versions + # it's only fixed in 3.3.3 (unreleased) and 3.4.0 + # -> https://issues.apache.org/jira/browse/KAFKA-14646 + return hasattr(self, "version") and self >= V_3_4_0 def get_version(node=None): """Return the version attached to the given node.