From de39d6d59abdaf8e5346f936bdfcf4fc86caf262 Mon Sep 17 00:00:00 2001 From: Sylwia Szunejko Date: Fri, 9 May 2025 12:59:35 +0200 Subject: [PATCH 1/4] Introduce BackendType.Scylla and fix logic to work with it --- .github/workflows/main.yml | 1 + .../TestClusterManagement/CcmBridge.cs | 8 +++--- .../TestClusterManagement/CcmCluster.cs | 6 ++--- .../TestClusterManager.cs | 25 ++++++++++++++----- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b8f41afc6..091b7070e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,6 +39,7 @@ jobs: run: | pip3 install https://github.com/scylladb/scylla-ccm/archive/5392dd68748ee5e71f7fbad346667038437d2123.zip CCM_PATH=$(which ccm) + echo "CCM_DISTRIBUTION=scylla" >> $GITHUB_ENV sudo mkdir -p /usr/local/bin sudo ln -sf ${CCM_PATH} /usr/local/bin/ccm diff --git a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmBridge.cs b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmBridge.cs index 0030470da..cb4eb614a 100644 --- a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmBridge.cs +++ b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmBridge.cs @@ -31,13 +31,12 @@ public class CcmBridge : IDisposable public DirectoryInfo CcmDir { get; private set; } public string Name { get; private set; } public string Version { get; private set; } - public string ScyllaVersion { get; private set; } public string IdPrefix { get; private set; } public string IpPrefix => $"127.0.{IdPrefix}."; public ICcmProcessExecuter CcmProcessExecuter { get; set; } private readonly string _dseInstallPath; - public CcmBridge(string name, string idPrefix, string dsePath, string version, string scyllaVersion, ICcmProcessExecuter executor) + public CcmBridge(string name, string idPrefix, string dsePath, string version, ICcmProcessExecuter executor) { Name = name; IdPrefix = idPrefix; @@ -45,7 +44,6 @@ public CcmBridge(string name, string idPrefix, string dsePath, string version, s CcmProcessExecuter = executor; _dseInstallPath = dsePath; Version = version; - ScyllaVersion = scyllaVersion; } public void Dispose() @@ -69,9 +67,9 @@ public void Create(bool useSsl) sslParams = "--ssl " + sslPath; } - if (!string.IsNullOrEmpty(ScyllaVersion)) + if (TestClusterManager.IsScylla) { - ExecuteCcm($"create {Name} --scylla -v release:{ScyllaVersion} {sslParams}"); + ExecuteCcm($"create {Name} --scylla -v release:{Version} {sslParams}"); } else if (string.IsNullOrEmpty(_dseInstallPath)) { diff --git a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmCluster.cs b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmCluster.cs index 558047ea7..8a71be204 100644 --- a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmCluster.cs +++ b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmCluster.cs @@ -25,7 +25,6 @@ public class CcmCluster : ITestCluster { public string Name { get; set; } public string Version { get; set; } - public string ScyllaVersion { get; set; } public Builder Builder { get; set; } public Cluster Cluster { get; set; } public ISession Session { get; set; } @@ -38,7 +37,7 @@ public class CcmCluster : ITestCluster private CcmBridge _ccm; private int _nodeLength; - public CcmCluster(string name, string idPrefix, string dsePath, ICcmProcessExecuter executor, string defaultKeyspace, string version, string scyllaVersion = null) + public CcmCluster(string name, string idPrefix, string dsePath, ICcmProcessExecuter executor, string defaultKeyspace, string version) { _executor = executor; Name = name; @@ -48,14 +47,13 @@ public CcmCluster(string name, string idPrefix, string dsePath, ICcmProcessExecu InitialContactPoint = ClusterIpPrefix + "1"; DsePath = dsePath; Version = version; - ScyllaVersion = scyllaVersion; } public void Create(int nodeLength, TestClusterOptions options = null) { _nodeLength = nodeLength; options = options ?? TestClusterOptions.Default; - _ccm = new CcmBridge(Name, IdPrefix, DsePath, Version, ScyllaVersion, _executor); + _ccm = new CcmBridge(Name, IdPrefix, DsePath, Version, _executor); _ccm.Create(options.UseSsl); _ccm.Populate(nodeLength, options.Dc2NodeLength, options.UseVNodes); _ccm.UpdateConfig(options.CassandraYaml); diff --git a/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs b/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs index ec4804f5c..14685c278 100644 --- a/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs +++ b/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs @@ -107,11 +107,12 @@ public enum BackendType { Hcd, Dse, - Cassandra + Cassandra, + Scylla } /// - /// "hcd", "dse", or "cassandra" (default), based on CCM_DISTRIBUTION + /// "hcd", "dse", or "cassandra" (default), or "scylla", based on CCM_DISTRIBUTION /// if there's env var DSE_VERSION, ignore CCM_DISTRIBUTION /// public static BackendType CurrentBackendType @@ -131,6 +132,8 @@ public static BackendType CurrentBackendType return BackendType.Dse; case "cassandra": return BackendType.Cassandra; + case "scylla": + return BackendType.Scylla; default: throw new TestInfrastructureException("Unknown CCM_DISTRIBUTION value: " + distribution); } @@ -169,18 +172,29 @@ public static string CassandraVersionString { return Environment.GetEnvironmentVariable("DSE_VERSION"); } + if (Environment.GetEnvironmentVariable("SCYLLA_VERSION") != null) + { + return "3.10.0"; + } return Environment.GetEnvironmentVariable("CASSANDRA_VERSION") ?? "3.11.2"; } } public static string ScyllaVersionString { - get { return Environment.GetEnvironmentVariable("SCYLLA_VERSION"); } + get + { + if (Environment.GetEnvironmentVariable("SCYLLA_VERSION") == null) + { + throw new TestInfrastructureException("SCYLLA_VERSION is not set"); + } + return Environment.GetEnvironmentVariable("SCYLLA_VERSION"); + } } public static bool IsScylla { - get { return !string.IsNullOrEmpty(ScyllaVersionString); } + get { return CurrentBackendType == BackendType.Scylla; } } public static bool IsDse @@ -299,8 +313,7 @@ private static ITestCluster CreateNewNoRetry(int nodeLength, TestClusterOptions DsePath, Executor, DefaultKeyspaceName, - IsDse ? DseVersionString : CassandraVersionString, - ScyllaVersionString); + IsDse ? DseVersionString : (IsScylla ? ScyllaVersionString : CassandraVersionString)); testCluster.Create(nodeLength, options); if (startCluster) { From 3453354f92d268dd033701ab845d726dffeecc61 Mon Sep 17 00:00:00 2001 From: Sylwia Szunejko Date: Fri, 9 May 2025 13:04:34 +0200 Subject: [PATCH 2/4] Fix version matching to match scylla if Cassandra version < 4.0.0 --- .../TestBase/TestCassandraVersion.cs | 26 ++++++++++++++----- .../TestClusterManager.cs | 13 ++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/Cassandra.IntegrationTests/TestBase/TestCassandraVersion.cs b/src/Cassandra.IntegrationTests/TestBase/TestCassandraVersion.cs index 71db1d443..9ee2027a0 100644 --- a/src/Cassandra.IntegrationTests/TestBase/TestCassandraVersion.cs +++ b/src/Cassandra.IntegrationTests/TestBase/TestCassandraVersion.cs @@ -103,12 +103,6 @@ public static bool VersionMatch(Version expectedVersion, bool requiresDse, bool return false; } - if (TestClusterManager.IsScylla && requiresOss) - { - message = "Test designed to run with OSS Cassandra (executing Scylla)"; - return true; - } - if (TestClusterManager.IsDse && requiresOss) { message = string.Format("Test designed to run with OSS {0} v{1} (executing DSE {2})", @@ -125,6 +119,26 @@ public static bool VersionMatch(Version expectedVersion, bool requiresDse, bool return false; } + if (TestClusterManager.IsScylla) + { + var scyllaExecutingVersion = TestClusterManager.ScyllaVersion; + if (expectedVersion.Major >= 4) + { + message = + $"Test designed to run with Cassandra {TestCassandraVersion.GetComparisonText(comparison)} v{expectedVersion} (executing Scylla v{scyllaExecutingVersion})"; + return false; + } + if (!TestCassandraVersion.VersionMatch(expectedVersion, new Version(3, 10), comparison)) + { + message = + $"Test designed to run with Cassandra {TestCassandraVersion.GetComparisonText(comparison)} v{expectedVersion} (executing Scylla v{scyllaExecutingVersion})"; + return false; + } + + message = null; + return true; + } + var executingVersion = requiresDse ? TestClusterManager.DseVersion : TestClusterManager.CassandraVersion; if (!TestCassandraVersion.VersionMatch(expectedVersion, executingVersion, comparison)) { diff --git a/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs b/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs index 14685c278..2e1005e2d 100644 --- a/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs +++ b/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs @@ -87,6 +87,19 @@ public static Version CassandraVersion } } + public static Version ScyllaVersion + { + get + { + var scyllaVersion = ScyllaVersionString; + if (scyllaVersion == null) + { + throw new TestInfrastructureException("SCYLLA_VERSION is not set"); + } + return new Version(TestClusterManager.ScyllaVersionString.Split('-')[0]); + } + } + /// /// Gets the IP prefix for the DSE instances /// From 5d5ae33e9d2448d1135b6f75957f02590c18c7f2 Mon Sep 17 00:00:00 2001 From: Sylwia Szunejko Date: Fri, 9 May 2025 15:41:35 +0200 Subject: [PATCH 3/4] Add TestScyllaVersion --- .../TestBase/TestScyllaVersion.cs | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 src/Cassandra.IntegrationTests/TestBase/TestScyllaVersion.cs diff --git a/src/Cassandra.IntegrationTests/TestBase/TestScyllaVersion.cs b/src/Cassandra.IntegrationTests/TestBase/TestScyllaVersion.cs new file mode 100644 index 000000000..feeac1545 --- /dev/null +++ b/src/Cassandra.IntegrationTests/TestBase/TestScyllaVersion.cs @@ -0,0 +1,126 @@ +using System; +using Cassandra.IntegrationTests.TestClusterManagement; +using NUnit.Framework; +using NUnit.Framework.Interfaces; + +namespace Cassandra.IntegrationTests.TestBase +{ + public class TestScyllaVersion : NUnitAttribute, IApplyToTest + { + public int Major { get; set; } + + public int Minor { get; set; } + + public int Build { get; set; } + + public Comparison Comparison { get; set; } + + public TestScyllaVersion(int major, int minor, Comparison comparison = Comparison.GreaterThanOrEqualsTo) + : this(major, minor, 0, comparison) + { + + } + + public TestScyllaVersion(int major, int minor, int build, Comparison comparison = Comparison.GreaterThanOrEqualsTo) + { + Major = major; + Minor = minor; + Build = build; + Comparison = comparison; + } + + public void ApplyToTest(NUnit.Framework.Internal.Test test) + { + if (!Applies(out string msg)) + { + test.RunState = RunState.Ignored; + var message = msg; + test.Properties.Set("_SKIPREASON", message); + } + } + + public bool Applies(out string msg) + { + var expectedVersion = new Version(Major, Minor, Build); + return TestScyllaVersion.VersionMatch(expectedVersion, Comparison, out msg); + } + + public static bool VersionMatch(Version expectedVersion, Comparison comparison, out string message) + { + if (!TestClusterManager.IsScylla) + { + message = $"Test designed to run with Scylla {TestScyllaVersion.GetComparisonText(comparison)} v{expectedVersion}"; + return false; + } + + var executingVersion = TestClusterManager.ScyllaVersion; + if (!TestScyllaVersion.VersionMatch(expectedVersion, executingVersion, comparison)) + { + message = + $"Test designed to run with Scylla {TestScyllaVersion.GetComparisonText(comparison)} v{expectedVersion} (executing {executingVersion})"; + return false; + } + + message = null; + return true; + } + + public static bool VersionMatch(Version expectedVersion, Version executingVersion, Comparison comparison) + { + expectedVersion = AdaptVersion(expectedVersion); + executingVersion = AdaptVersion(executingVersion); + + var comparisonResult = (Comparison)executingVersion.CompareTo(expectedVersion); + + if (comparisonResult >= Comparison.Equal && comparison == Comparison.GreaterThanOrEqualsTo) + { + return true; + } + return comparisonResult == comparison; + } + + /// + /// Replace -1 (undefined) with 0 on the version string. + /// + private static Version AdaptVersion(Version v) + { + var minor = v.Minor; + if (minor < 0) + { + minor = 0; + } + + var build = v.Build; + if (build < 0) + { + build = 0; + } + + var revision = v.Revision; + if (revision < 0) + { + revision = 0; + } + + return new Version(v.Major, minor, build, revision); + } + + private static string GetComparisonText(Comparison comparison) + { + string result; + switch (comparison) + { + case Comparison.GreaterThanOrEqualsTo: + result = "greater than or equals to"; + break; + case Comparison.LessThan: + result = "lower than"; + break; + default: + result = "equals to"; + break; + } + return result; + } + } +} From fc568446f72b3de3bff94b9fc16febb53321c542 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Thu, 15 May 2025 00:36:00 -0400 Subject: [PATCH 4/4] Use ccm to get scylla version --- .github/workflows/main.yml | 6 +-- .../TestBase/TestUtils.cs | 19 +++---- .../TestClusterManagement/CcmBridge.cs | 12 ++++- .../TestClusterManagement/CcmCluster.cs | 5 ++ .../CcmProcessExecuter.cs | 8 +-- .../LocalCcmProcessExecuter.cs | 2 +- .../TestClusterManager.cs | 50 +++++++++++++++++-- 7 files changed, 78 insertions(+), 24 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 091b7070e..78aa0228d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,11 +55,11 @@ jobs: run: | cd get-version if [[ "${{ matrix.scylla-version }}" == "ENTERPRISE-RELEASE" ]]; then - echo "SCYLLA_VERSION=$(./get-version --source dockerhub-imagetag --repo scylladb/scylla-enterprise -filters "2024.1.LAST" | tr -d '\"')" >> $GITHUB_ENV + echo "SCYLLA_VERSION=release:$(./get-version --source dockerhub-imagetag --repo scylladb/scylla-enterprise -filters "2024.1.LAST" | tr -d '\"')" >> $GITHUB_ENV elif [[ "${{ matrix.scylla-version }}" == "OSS-RELEASE" ]]; then - echo "SCYLLA_VERSION=$(./get-version --source dockerhub-imagetag --repo scylladb/scylla -filters "6.1.LAST" | tr -d '\"')" >> $GITHUB_ENV + echo "SCYLLA_VERSION=release:$(./get-version --source dockerhub-imagetag --repo scylladb/scylla -filters "6.1.LAST" | tr -d '\"')" >> $GITHUB_ENV elif echo "${{ matrix.scylla-version }}" | grep -P '^[0-9\.]+'; then # If you want to run specific version do just that - echo "SCYLLA_VERSION=${{ matrix.scylla-version }}" >> $GITHUB_ENV + echo "SCYLLA_VERSION=release:${{ matrix.scylla-version }}" >> $GITHUB_ENV else echo "Unknown scylla version name `${{ matrix.scylla-version }}`" exit 1 diff --git a/src/Cassandra.IntegrationTests/TestBase/TestUtils.cs b/src/Cassandra.IntegrationTests/TestBase/TestUtils.cs index 2593a5392..6c7dcc7d5 100644 --- a/src/Cassandra.IntegrationTests/TestBase/TestUtils.cs +++ b/src/Cassandra.IntegrationTests/TestBase/TestUtils.cs @@ -367,7 +367,7 @@ public static ProcessOutput ExecuteLocalCcm(string ccmArgs, string ccmConfigDir, ccmConfigDir = TestUtils.EscapePath(ccmConfigDir); var args = ccmArgs + " --config-dir=" + ccmConfigDir; Trace.TraceInformation("Executing ccm: " + ccmArgs); - var processName = "/usr/local/bin/ccm"; + var processName = "ccm"; if (TestUtils.IsWin) { processName = "cmd.exe"; @@ -683,7 +683,13 @@ public class ProcessOutput public StringBuilder OutputText { get; set; } - private string Output { get; set; } + private string Output => + StdOut + Environment.NewLine + + "STDERR:" + Environment.NewLine + + StdErr; + + public string StdOut { get; set; } + public string StdErr { get; set; } public ProcessOutput() { @@ -694,13 +700,8 @@ public ProcessOutput() public override string ToString() { return - "Exit Code: " + this.ExitCode + Environment.NewLine + - "Output Text: " + (this.Output ?? this.OutputText.ToString()) + Environment.NewLine; - } - - public void SetOutput(string output) - { - Output = output; + "Exit Code: " + ExitCode + Environment.NewLine + + "Output Text: " + (Output ?? OutputText.ToString()) + Environment.NewLine; } } diff --git a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmBridge.cs b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmBridge.cs index cb4eb614a..cd15b6880 100644 --- a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmBridge.cs +++ b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmBridge.cs @@ -69,7 +69,7 @@ public void Create(bool useSsl) if (TestClusterManager.IsScylla) { - ExecuteCcm($"create {Name} --scylla -v release:{Version} {sslParams}"); + ExecuteCcm($"create {Name} --scylla -v {Version} {sslParams}"); } else if (string.IsNullOrEmpty(_dseInstallPath)) { @@ -242,6 +242,16 @@ public void Stop() ExecuteCcm("stop"); } + public string GetVersion(int nodeId) + { + ProcessOutput output = ExecuteCcm(string.Format("node{0} versionfrombuild", nodeId)); + if (output.ExitCode != 0) + { + throw new TestInfrastructureException("Process exited in error " + output.ToString()); + } + return output.StdOut.Trim(); + } + public void StopForce() { ExecuteCcm("stop --not-gently"); diff --git a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmCluster.cs b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmCluster.cs index 8a71be204..28a27bb15 100644 --- a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmCluster.cs +++ b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmCluster.cs @@ -116,6 +116,11 @@ public void ShutDown() _ccm.Stop(); } + public string GetVersion(int nodeId) + { + return _ccm.GetVersion(nodeId); + } + public void Remove() { Trace.TraceInformation($"Removing Cluster with Name: '{Name}', InitialContactPoint: {InitialContactPoint}, and CcmDir: {_ccm.CcmDir}"); diff --git a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmProcessExecuter.cs b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmProcessExecuter.cs index 2373518ca..d8dfe23b9 100644 --- a/src/Cassandra.IntegrationTests/TestClusterManagement/CcmProcessExecuter.cs +++ b/src/Cassandra.IntegrationTests/TestClusterManagement/CcmProcessExecuter.cs @@ -110,12 +110,8 @@ public static ProcessOutput ExecuteProcess(string processName, string args, int processEndTokenSource.Cancel(); timeoutTokenSource.Cancel(); } - var stdOut = tStandardOutput.GetAwaiter().GetResult(); - var stdErr = tStandardError.GetAwaiter().GetResult(); - - output.SetOutput(stdOut + Environment.NewLine + - "STDERR:" + Environment.NewLine + - stdErr); + output.StdErr = tStandardError.GetAwaiter().GetResult(); + output.StdOut = tStandardOutput.GetAwaiter().GetResult(); } return output; } diff --git a/src/Cassandra.IntegrationTests/TestClusterManagement/LocalCcmProcessExecuter.cs b/src/Cassandra.IntegrationTests/TestClusterManagement/LocalCcmProcessExecuter.cs index 9ee75f79f..cf6117ea2 100644 --- a/src/Cassandra.IntegrationTests/TestClusterManagement/LocalCcmProcessExecuter.cs +++ b/src/Cassandra.IntegrationTests/TestClusterManagement/LocalCcmProcessExecuter.cs @@ -20,7 +20,7 @@ namespace Cassandra.IntegrationTests.TestClusterManagement { public class LocalCcmProcessExecuter : CcmProcessExecuter { - public const string CcmCommandPath = "/usr/local/bin/ccm"; + public const string CcmCommandPath = "ccm"; public static readonly LocalCcmProcessExecuter Instance = new LocalCcmProcessExecuter(); private LocalCcmProcessExecuter() diff --git a/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs b/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs index 2e1005e2d..c451ab264 100644 --- a/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs +++ b/src/Cassandra.IntegrationTests/TestClusterManagement/TestClusterManager.cs @@ -17,6 +17,7 @@ using System; using System.Diagnostics; using System.Linq; +using System.Threading; using Cassandra.IntegrationTests.TestBase; namespace Cassandra.IntegrationTests.TestClusterManagement @@ -87,16 +88,57 @@ public static Version CassandraVersion } } + private static Version _scyllaVersion; + private static Mutex _scyllaVersionLock = new Mutex(); + public static Version ScyllaVersion { get { - var scyllaVersion = ScyllaVersionString; - if (scyllaVersion == null) + if (_scyllaVersion != null) { - throw new TestInfrastructureException("SCYLLA_VERSION is not set"); + return _scyllaVersion; + } + _scyllaVersionLock.WaitOne(); + try + { + if (_scyllaVersion != null) + { + return _scyllaVersion; + } + var scyllaVersion = ScyllaVersionString; + if (scyllaVersion == null) + { + throw new TestInfrastructureException("SCYLLA_VERSION is not set"); + } + + if (scyllaVersion.StartsWith("release:")) + { + scyllaVersion = scyllaVersion.Substring(8); + if (scyllaVersion.Count(c => c == '.') == 2) + { + _scyllaVersion = new Version(scyllaVersion); + return _scyllaVersion; + } + } + + TryRemove(); + var testCluster = new CcmCluster( + "get-scylla-version-" + TestUtils.GetTestClusterNameBasedOnRandomString(), + GetUniqueIdPrefix(), + DsePath, + Executor, + "", + ScyllaVersionString); + testCluster.Create(1, null); + string versionString = testCluster.GetVersion(1); + _scyllaVersion = new Version(versionString); + return _scyllaVersion; + } + finally + { + _scyllaVersionLock.ReleaseMutex(); } - return new Version(TestClusterManager.ScyllaVersionString.Split('-')[0]); } }