ci: add a backward compatibility checks to the end-to-end runner

.github/workflows/backward-compatibility.yml

@@ -158,9 +158,12 @@ jobs:
               --signed-entity-types ${{ inputs.signed-entity-types }}
             EXIT_CODE=$?
             if [ $EXIT_CODE -eq 0 ]; then
-              echo "SUCCESS=true" >> $GITHUB_ENV
+              echo "RESULT=success" >> $GITHUB_ENV
+            elif [ $EXIT_CODE -eq 3 ]; then
+              echo "RESULT=incompatible" >> $GITHUB_ENV
+              exit 0
             else
-              echo "SUCCESS=false" >> $GITHUB_ENV
+              echo "RESULT=failure" >> $GITHUB_ENV
             fi
             exit $EXIT_CODE
 
@@ -195,8 +198,8 @@ jobs:
                 --arg AGGREGATOR "$AGGREGATOR_TAG" \
                 --arg SIGNER "$SIGNER_TAG" \
                 --arg CLIENT "$CLIENT_TAG" \
-                --argjson SUCCESS "${{ env.SUCCESS }}" \
-                '{tag: $TAG, node: $NODE, mithril_signer: $SIGNER, mithril_aggregator: $AGGREGATOR, mithril_client: $CLIENT, cardano_node_version: $CARDANO_NODE, success: $SUCCESS}' \
+                --arg RESULT "${{ env.RESULT }}" \
+                '{tag: $TAG, node: $NODE, mithril_signer: $SIGNER, mithril_aggregator: $AGGREGATOR, mithril_client: $CLIENT, cardano_node_version: $CARDANO_NODE, result: $RESULT}' \
                 > ./${{ env.RESULT_FILE_NAME }}.json
 
       - name: Upload test result JSON
@@ -241,6 +244,7 @@ jobs:
         shell: bash
         run: |
           CHECK_MARK=":heavy_check_mark:"
+          WARN_MARK=":warning:"
           CROSS_MARK=":no_entry:"
 
           echo "## Distributions backward compatibility" >> $GITHUB_STEP_SUMMARY
@@ -250,22 +254,22 @@ jobs:
           echo "" >> $GITHUB_STEP_SUMMARY
 
           echo "**Signed entity types**: ${{ inputs.signed-entity-types }}" >> $GITHUB_STEP_SUMMARY
-          echo "**Cardano nodes**: ${{ inputs.cardano_node_version }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Cardano nodes**: ${{ inputs.cardano-node-version }}" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
 
           echo "| Compatibility | mithril-signer | mithril-aggregator | mithril-client |" >> $GITHUB_STEP_SUMMARY
           echo "| --- | :---: | :---: | :---: |" >> $GITHUB_STEP_SUMMARY
 
           # Transform summary.json into Markdown table rows
-          jq -r --arg CHECK_MARK "$CHECK_MARK" --arg CROSS_MARK "$CROSS_MARK" \
-            'group_by(.tag) |
-            sort_by(.[0].tag | sub("-[a-z]+$"; "") | tonumber) | reverse |
+          jq -r --arg CHECK_MARK "$CHECK_MARK" --arg WARN_MARK "$WARN_MARK" --arg CROSS_MARK "$CROSS_MARK" \
+            'def parseresult(result): if result == "success" then $CHECK_MARK elif result == "incompatible" then $WARN_MARK else $CROSS_MARK end;
+            group_by(.tag) | sort_by(.[0].tag | sub("-[a-z]+$"; "") | tonumber) | reverse |
             .[] |
             {
               tag: .[0].tag, 
-              signer: (map(select(.node == "mithril-signer") | if .success then $CHECK_MARK else $CROSS_MARK end) | join("")),
-              aggregator: (map(select(.node == "mithril-aggregator") | if .success then $CHECK_MARK else $CROSS_MARK end) | join("")),
-              client: (map(select(.node == "mithril-client") | if .success then $CHECK_MARK else $CROSS_MARK end) | join(""))
+              signer: (map(select(.node == "mithril-signer") | parseresult(.result)) | join("")),
+              aggregator: (map(select(.node == "mithril-aggregator") | parseresult(.result)) | join("")),
+              client: (map(select(.node == "mithril-client") | parseresult(.result)) | join(""))
             } |
             "| `\(.tag)` | \(.signer) | \(.aggregator) | \(.client) |"' "./test-results/summary.json" >> $GITHUB_STEP_SUMMARY
 

.github/workflows/nightly-dispatcher.yml

@@ -22,6 +22,12 @@ jobs:
 
   notify-on-failure:
     uses: ./.github/workflows/test-notify-on-failure.yml
-    needs: [docker-builds, aggregator-stress-test, test-client]
+    needs:
+      [
+        backward-compatibility,
+        docker-builds,
+        aggregator-stress-test,
+        test-client,
+      ]
     if: failure()
     secrets: inherit

flake.nix

@@ -91,7 +91,7 @@
               cargoArtifacts = buildDeps cargoToml baseCargoArtifacts;
             }
             // {
-              cargoTestCommand = "cargo test";
+              cargoTestCommand = "RUST_BACKTRACE=1 cargo test";
             }
             // args);
 

mithril-test-lab/mithril-end-to-end/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-end-to-end"
-version = "0.4.108"
+version = "0.4.109"
 authors = { workspace = true }
 edition = { workspace = true }
 documentation = { workspace = true }
@@ -29,6 +29,7 @@ mithril-cardano-node-internal-database = { path = "../../internal/cardano-node/m
 mithril-common = { path = "../../mithril-common" }
 mithril-doc = { path = "../../internal/mithril-doc" }
 reqwest = { workspace = true, features = ["default"] }
+semver = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 slog = { workspace = true, features = [

mithril-test-lab/mithril-end-to-end/backward-compatibility.md

@@ -0,0 +1,68 @@
+# Backward compatibility
+
+The end-to-end tests scenarios are backward compatible with previous nodes versions.
+
+This is achieved by differentiating the scenarios based on the version of the nodes they are testing.
+
+There are multiple ways to do this:
+
+- disabling part of the scenarios
+- adding or removing arguments to the binaries (i.e. `--backend` flag for the client-cli)
+
+> [!TIP]
+> Node versions and distribution versions below are the ones that first included the changes, has node versions evolve
+> faster than distribution versions the actual node version included in the distribution may be higher.
+>
+> i.e. `--origin-tag` flag for the client-cli was introduced in version `0.11.13`, the first distribution that included
+> that change is `2517.0`, which include client-cli `0.12.0`.
+
+---
+
+## Supported changes
+
+List of breaking changes that are supported by the end-to-end tests.
+
+format is: `- since 'X.Y.Z' (distribution version) [to 'X.Y.Z' (distribution version) (optional)]: supported change`
+
+### Mithril client
+
+- since `0.12.34` (2543.0): test of new `--epoch` filter to `cardano-db list` (disabled on lower versions)
+- since `0.12.11` (2524.0): removal of `cardano-db-v2` replaced with `cardano-db [command] --backend [v1,v2]`
+- since `0.11.14` (2517.0): addition of `--include-ancillary` flag to `cardano-db download`
+- since `0.11.13` (2517.0): addition of global `--origin-tag` parameter
+
+### Mithril aggregator
+
+- since `0.7.94` (next to 2543.1): only the leader aggregator must be restarted when updating protocol parameters
+
+### Mithril signer
+
+---
+
+## Not supported changes
+
+List of breaking changes that are NOT supported by the end-to-end tests, running a node with those versions against
+`unstable` will result in an error and an exit code of `3`.
+
+format is:
+
+- for an incompatibility of a node below a specific version:
+  `- 'min supported version is 'X.Y.Z' (distribution version): explanatory message`
+- for an incompatibility between two nodes:
+  `- below 'X.Y.Z' (distribution version) with {other node} `X.Y.Z` (other distribution version) and up: explanatory message`
+
+### Mithril client
+
+- below `0.11.14` (2517.0) with aggregator `0.7.31` (2517.0) and up: split ancillary files are not correctly supported
+  for older clients, causing the verification to fail because the incomplete immutable files trio is missing.
+
+### Mithril aggregator
+
+- below `0.7.91` (next to 2543.1) with signer `0.2.277` (next to 2543.1) and up: new `/protocol-configuration/{epoch}`aggregator
+  route to update network parameters (required by signer`0.2.277` and up)
+- below `0.7.55` (2524.0) with cardano-node version `10.4.1` and up: support of UTxO-HD was added only on aggregator `0.7.55` and up
+
+### Mithril signer
+
+- min supported version is `0.2.221` (2450.0): addition of the `CardanoDatabase` signed entity type, previous signers
+  are not able to handle unknown signed entities

mithril-test-lab/mithril-end-to-end/src/assertions/check.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 use anyhow::{Context, anyhow};
 use reqwest::StatusCode;
 use serde::de::DeserializeOwned;
-use slog_scope::info;
+use slog_scope::{info, warn};
 
 use mithril_common::{
     StdResult,
@@ -558,13 +558,21 @@ pub async fn assert_client_can_verify_cardano_database(
     client
         .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::List))
         .await?;
-    client
-        .run(ClientCommand::CardanoDbV2(
-            CardanoDbV2Command::ListPerEpoch {
-                epoch_specifier: EpochSpecifier::LatestMinusOffset(5),
-            },
-        ))
-        .await?;
+
+    if client.version().is_above_or_equal("0.12.34") {
+        client
+            .run(ClientCommand::CardanoDbV2(
+                CardanoDbV2Command::ListPerEpoch {
+                    epoch_specifier: EpochSpecifier::LatestMinusOffset(5),
+                },
+            ))
+            .await?;
+    } else {
+        warn!(
+            "Client version is below 0.12.34, skipping `cardano-db snapshot list --epoch latest-5` check"
+        );
+    }
+
     client
         .run(ClientCommand::CardanoDbV2(CardanoDbV2Command::Show {
             hash: hash.to_string(),

mithril-test-lab/mithril-end-to-end/src/devnet/runner.rs

@@ -68,7 +68,7 @@ pub struct DevnetBootstrapArgs {
     pub number_of_full_nodes: u8,
     pub cardano_slot_length: f64,
     pub cardano_epoch_length: f64,
-    pub cardano_node_version: String,
+    pub cardano_node_version: semver::Version,
     pub cardano_hard_fork_latest_era_at_epoch: u16,
     pub skip_cardano_bin_download: bool,
 }
@@ -118,7 +118,10 @@ impl Devnet {
             "EPOCH_LENGTH",
             bootstrap_args.cardano_epoch_length.to_string(),
         );
-        bootstrap_command.env("CARDANO_NODE_VERSION", &bootstrap_args.cardano_node_version);
+        bootstrap_command.env(
+            "CARDANO_NODE_VERSION",
+            bootstrap_args.cardano_node_version.to_string(),
+        );
         bootstrap_command.env(
             "CARDANO_HARD_FORK_LATEST_ERA_AT_EPOCH",
             bootstrap_args.cardano_hard_fork_latest_era_at_epoch.to_string(),

mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs

@@ -136,7 +136,8 @@ impl Spec {
             "epoch after which the protocol parameters will change".to_string(),
         )
         .await?;
-        if aggregator.is_first() {
+
+        if aggregator.is_first() || aggregator.version().is_below("0.7.94") {
             assertions::update_protocol_parameters(aggregator).await?;
         }
 

mithril-test-lab/mithril-end-to-end/src/lib.rs

@@ -10,6 +10,7 @@ pub use devnet::*;
 pub use end_to_end_spec::Spec;
 pub use mithril::*;
 pub use run_only::RunOnly;
+pub use utils::{CompatibilityChecker, CompatibilityCheckerError, NodeVersion};
 
 use clap::ValueEnum;
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]

mithril-test-lab/mithril-end-to-end/src/main.rs

@@ -3,6 +3,7 @@ use clap::{CommandFactory, Parser, Subcommand};
 use slog::{Drain, Level, Logger};
 use slog_scope::{error, info};
 use std::{
+    collections::BTreeMap,
     fmt, fs,
     path::{Path, PathBuf},
     process::{ExitCode, Termination},
@@ -19,8 +20,9 @@ use tokio::{
 use mithril_common::StdResult;
 use mithril_doc::GenerateDocCommands;
 use mithril_end_to_end::{
-    Devnet, DevnetBootstrapArgs, DmqNodeFlavor, MithrilInfrastructure, MithrilInfrastructureConfig,
-    RetryableDevnetError, RunOnly, Spec,
+    Aggregator, Client, CompatibilityChecker, CompatibilityCheckerError, Devnet,
+    DevnetBootstrapArgs, DmqNodeFlavor, MithrilInfrastructure, MithrilInfrastructureConfig,
+    NodeVersion, RelaySigner, RetryableDevnetError, RunOnly, Signer, Spec,
 };
 
 /// Tests args
@@ -67,9 +69,9 @@ pub struct Args {
     #[clap(long, default_value_t = 30.0)]
     cardano_epoch_length: f64,
 
-    /// Cardano node version
+    /// Cardano node version, must be a valid semver version
     #[clap(long, default_value = "10.5.1")]
-    cardano_node_version: String,
+    cardano_node_version: semver::Version,
 
     /// Epoch at which hard fork to the latest Cardano era will be made (starts with the latest era by default)
     #[clap(long, default_value_t = 0)]
@@ -253,14 +255,16 @@ enum AppResult {
     UnretryableError(anyhow::Error),
     RetryableError(anyhow::Error),
     Cancelled(anyhow::Error),
+    IncompatibleNode(anyhow::Error),
 }
 
 impl AppResult {
     fn exit_code(&self) -> ExitCode {
         match self {
             AppResult::Success() => ExitCode::SUCCESS,
-            AppResult::UnretryableError(_) | AppResult::Cancelled(_) => ExitCode::FAILURE,
-            AppResult::RetryableError(_) => ExitCode::from(2),
+            AppResult::UnretryableError(..) | AppResult::Cancelled(..) => ExitCode::FAILURE,
+            AppResult::RetryableError(..) => ExitCode::from(2),
+            AppResult::IncompatibleNode(..) => ExitCode::from(3),
         }
     }
 }
@@ -272,6 +276,7 @@ impl fmt::Display for AppResult {
             AppResult::UnretryableError(error) => write!(f, "Error(Unretryable): {error:?}"),
             AppResult::RetryableError(error) => write!(f, "Error(Retryable): {error:?}"),
             AppResult::Cancelled(error) => write!(f, "Cancelled: {error:?}"),
+            AppResult::IncompatibleNode(error) => write!(f, "{error:?}"),
         }
     }
 }
@@ -298,6 +303,8 @@ impl From<StdResult<()>> for AppResult {
                     AppResult::RetryableError(error)
                 } else if error.is::<SignalError>() {
                     AppResult::Cancelled(error)
+                } else if error.is::<CompatibilityCheckerError>() {
+                    AppResult::IncompatibleNode(error)
                 } else {
                     AppResult::UnretryableError(error)
                 }
@@ -352,6 +359,26 @@ impl App {
 
         let use_p2p_passive_relays = args.use_p2p_passive_relays;
 
+        CompatibilityChecker::default().check(BTreeMap::from([
+            (
+                Aggregator::BIN_NAME,
+                NodeVersion::fetch_semver(Aggregator::BIN_NAME, &args.bin_directory)?,
+            ),
+            (
+                Signer::BIN_NAME,
+                NodeVersion::fetch_semver(Signer::BIN_NAME, &args.bin_directory)?,
+            ),
+            (
+                Client::BIN_NAME,
+                NodeVersion::fetch_semver(Client::BIN_NAME, &args.bin_directory)?,
+            ),
+            (
+                RelaySigner::BIN_NAME,
+                NodeVersion::fetch_semver(RelaySigner::BIN_NAME, &args.bin_directory)?,
+            ),
+            ("cardano-node", args.cardano_node_version.to_owned()),
+        ]))?;
+
         let devnet = Devnet::bootstrap(&DevnetBootstrapArgs {
             devnet_scripts_dir: args.devnet_scripts_directory,
             artifacts_target_dir: work_dir.join("devnet"),