orchestratord test: Introduce workflow for downtime measurement

ci/nightly/pipeline.template.yml

@@ -2366,19 +2366,34 @@ steps:
     key: orchestratord-test
     steps:
       - id: orchestratord-defaults
-        label: "Orchestratord test (defaults from documentation)"
+        label: "Orchestratord + documentation defaults"
+        artifact_paths: ["mz_debug_*.zip"]
         depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
               composition: orchestratord
-              run: defaults
+              run: documentation-defaults
+              ci-builder: stable
+        agents:
+          queue: hetzner-aarch64-16cpu-32gb
+
+      - id: orchestratord-rolling-upgrade-downtime
+        label: "Orchestratord + rolling upgrade downtime"
+        artifact_paths: ["mz_debug_*.zip"]
+        depends_on: devel-docker-tags
+        timeout_in_minutes: 120
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: orchestratord
+              run: upgrade-downtime
               ci-builder: stable
         agents:
           queue: hetzner-aarch64-16cpu-32gb
 
       - id: orchestratord-default-properties
-        label: "Orchestratord test (defaults for properties)"
+        label: "Orchestratord + defaults for properties"
+        artifact_paths: ["mz_debug_*.zip"]
         depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
@@ -2390,7 +2405,8 @@ steps:
           queue: hetzner-aarch64-16cpu-32gb
 
       - id: orchestratord-individual
-        label: "Orchestratord test (individual properties)"
+        label: "Orchestratord + individual properties"
+        artifact_paths: ["mz_debug_*.zip"]
         depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
@@ -2402,7 +2418,8 @@ steps:
           queue: hetzner-aarch64-16cpu-32gb
 
       - id: orchestratord-combine
-        label: "Orchestratord test (combine properties)"
+        label: "Orchestratord + combine properties"
+        artifact_paths: ["mz_debug_*.zip"]
         depends_on: build-aarch64
         timeout_in_minutes: 120
         plugins:
@@ -2414,65 +2431,53 @@ steps:
           queue: hetzner-aarch64-16cpu-32gb
 
       - id: orchestratord-upgrade-individual
-        label: "Orchestratord test (upgrade, individual props)"
+        label: "Orchestratord + upgrade, individual props"
+        artifact_paths: ["mz_debug_*.zip"]
         depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
               composition: orchestratord
               args: [--action=upgrade, --properties=individual, --runtime=3600, --recreate-cluster]
               ci-builder: stable
-        env:
-          # Old versions are not on GHCR yet
-          MZ_GHCR: 0
         agents:
-          queue: hetzner-aarch64-8cpu-16gb
-        skip: "https://github.com/MaterializeInc/materialize/pull/34214"
+          queue: hetzner-x86-64-dedi-16cpu-64gb
 
       - id: orchestratord-upgrade-combine
-        label: "Orchestratord test (upgrade, combine props)"
+        label: "Orchestratord + upgrade, combine props"
+        artifact_paths: ["mz_debug_*.zip"]
         depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
               composition: orchestratord
               args: [--action=upgrade, --properties=combine, --runtime=3600, --recreate-cluster]
               ci-builder: stable
-        env:
-          # Old versions are not on GHCR yet
-          MZ_GHCR: 0
         agents:
-          queue: hetzner-aarch64-8cpu-16gb
-        skip: "https://github.com/MaterializeInc/materialize/pull/34214"
+          queue: hetzner-x86-64-dedi-16cpu-64gb
 
       - id: orchestratord-upgrade-chain-individual
-        label: "Orchestratord test (upgrade chain, individual props)"
+        label: "Orchestratord + upgrade chain, individual props"
+        artifact_paths: ["mz_debug_*.zip"]
         depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
               composition: orchestratord
               args: [--action=upgrade-chain, --properties=individual, --runtime=3600, --recreate-cluster]
               ci-builder: stable
-        env:
-          # Old versions are not on GHCR yet
-          MZ_GHCR: 0
         agents:
-          queue: hetzner-aarch64-8cpu-16gb
-        skip: "https://github.com/MaterializeInc/materialize/pull/34214"
+          queue: hetzner-x86-64-dedi-16cpu-64gb
 
       - id: orchestratord-upgrade-chain-combine
-        label: "Orchestratord test (upgrade chain, combine props)"
+        label: "Orchestratord + upgrade chain, combine props"
+        artifact_paths: ["mz_debug_*.zip"]
         depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
               composition: orchestratord
               args: [--action=upgrade-chain, --properties=combine, --runtime=3600, --recreate-cluster]
               ci-builder: stable
-        env:
-          # Old versions are not on GHCR yet
-          MZ_GHCR: 0
         agents:
-          queue: hetzner-aarch64-16cpu-32gb
-        skip: "https://github.com/MaterializeInc/materialize/pull/34214"
+          queue: hetzner-x86-64-dedi-16cpu-64gb

ci/plugins/mzcompose/hooks/command

@@ -344,7 +344,7 @@ cleanup() {
     && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] \
     && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] \
     && [[ ! "$BUILDKITE_LABEL" =~ Terraform\ .* ]] \
-    && [[ ! "$BUILDKITE_LABEL" =~ Orchestratord\ test\ .* ]] \
+    && [[ ! "$BUILDKITE_LABEL" =~ Orchestratord\ .* ]] \
     && [[ ! "$BUILDKITE_LABEL" =~ Cluster\ spec\ sheet.* ]]; then
       echo "+++ services.log is empty, failing"
       exit 1

test/orchestratord/cluster.yaml

@@ -164,6 +164,7 @@ nodes:
     image: kindest/node:v1.32.5
     labels:
       materialize.cloud/scratch-fs: "true"
+      materialize.cloud/disk: "true"
       materialize.cloud/availability-zone: "2"
       topology.kubernetes.io/zone: "2"
       workload: "materialize-instance"