diff --git a/.github/workflows/ci-integration-tests-gcp.patch-external.yml b/.github/workflows/ci-tests.patch-external.yml similarity index 62% rename from .github/workflows/ci-integration-tests-gcp.patch-external.yml rename to .github/workflows/ci-tests.patch-external.yml index 30a82a83a4e..48a6ab667b9 100644 --- a/.github/workflows/ci-integration-tests-gcp.patch-external.yml +++ b/.github/workflows/ci-tests.patch-external.yml @@ -1,5 +1,5 @@ -# Workflow patches for skipping Google Cloud unit test CI on PRs from external repositories. -name: Integration Tests on GCP +# Workflow patches for skipping CI tests on PRs from external repositories +name: Run tests # Run on PRs from external repositories, let them pass, and then Mergify will check them. # GitHub doesn't support filtering workflows by source branch names, so we have to do it for each @@ -7,109 +7,133 @@ name: Integration Tests on GCP on: pull_request: -# IMPORTANT -# -# The job names in `ci-integration-tests-gcp.yml`, `ci-integration-tests-gcp.patch.yml` and -# `ci-integration-tests-gcp.patch-external.yml` must be kept in sync. +#! IMPORTANT +#! +#! The job names in `sub-ci-unit-tests-docker.yml`, `sub-ci-integration-tests-gcp.yml`, +#! `ci-tests.patch.yml` and `ci-tests.patch-external.yml` must be kept in sync. jobs: - # We don't patch the testnet job, because testnet isn't required to merge (it's too unstable) - get-available-disks: - name: Check if cached state disks exist for Mainnet / Check if cached state disks exist + build: + name: Build CI Docker / Build images # Only run on PRs from external repositories. if: ${{ startsWith(github.event_name, 'pull') && github.event.pull_request.head.repo.fork }} runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' - build: - name: Build CI Docker / Build images - # This dependency allows all these jobs to depend on a single condition, making it easier to - # change. - needs: get-available-disks + #### + ## The following jobs are related to sub-ci-unit-tests-docker.yml + ### + test-all: + name: Test all + runs-on: ubuntu-latest + steps: + - run: 'echo "Skipping job on fork"' + + test-fake-activation-heights: + name: Test with fake activation heights + runs-on: ubuntu-latest + steps: + - run: 'echo "Skipping job on fork"' + + test-empty-sync: + name: Test checkpoint sync from empty state + runs-on: ubuntu-latest + steps: + - run: 'echo "Skipping job on fork"' + + test-lightwalletd-integration: + name: Test integration with lightwalletd + runs-on: ubuntu-latest + steps: + - run: 'echo "Skipping job on fork"' + + test-configuration-file: + name: Test CI default Docker config file / Test default-conf in Docker + runs-on: ubuntu-latest + steps: + - run: 'echo "Skipping job on fork"' + + test-configuration-file-testnet: + name: Test CI testnet Docker config file / Test default-conf in Docker + needs: build + runs-on: ubuntu-latest + steps: + - run: 'echo "Skipping job on fork"' + + test-zebra-conf-path: + name: Test CI custom Docker config file / Test custom-conf in Docker + runs-on: ubuntu-latest + steps: + - run: 'echo "Skipping job on fork"' + + + #### + ## The following jobs are related to sub-ci-integration-tests-gcp.yml + ### + + # We don't patch the testnet job, because testnet isn't required to merge (it's too unstable) + get-available-disks: + name: Check if cached state disks exist for Mainnet / Check if cached state disks exist runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' test-stateful-sync: name: Zebra checkpoint update / Run sync-past-checkpoint test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' test-update-sync: name: Zebra tip update / Run update-to-tip test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' checkpoints-mainnet: name: Generate checkpoints mainnet / Run checkpoints-mainnet test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' lightwalletd-rpc-test: name: Zebra tip JSON-RPC / Run fully-synced-rpc test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' lightwalletd-transactions-test: name: lightwalletd tip send / Run lwd-send-transactions test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' get-block-template-test: name: get block template / Run get-block-template test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' submit-block-test: name: submit block / Run submit-block test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' - scan-start-where-left-test: - name: Scan starts where left / Run scan-start-where-left test - needs: get-available-disks - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - scan-task-commands: - name: scan task commands / Run scan-task-commands test - needs: get-available-disks - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - lightwalletd-full-sync: name: lightwalletd tip / Run lwd-full-sync test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' lightwalletd-update-sync: name: lightwalletd tip update / Run lwd-update-sync test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' lightwalletd-grpc-test: name: lightwalletd GRPC tests / Run lwd-grpc-wallet test - needs: get-available-disks runs-on: ubuntu-latest steps: - run: 'echo "Skipping job on fork"' diff --git a/.github/workflows/ci-integration-tests-gcp.patch.yml b/.github/workflows/ci-tests.patch.yml similarity index 52% rename from .github/workflows/ci-integration-tests-gcp.patch.yml rename to .github/workflows/ci-tests.patch.yml index ac5d1dbf9cd..5320b149bbb 100644 --- a/.github/workflows/ci-integration-tests-gcp.patch.yml +++ b/.github/workflows/ci-tests.patch.yml @@ -1,45 +1,98 @@ -# Workflow patches for skipping Google Cloud integration test CI when Rust code or dependencies +# Workflow patches for skipping CI tests when Rust code or dependencies # aren't modified in a PR. -name: Integration Tests on GCP +name: Run tests # Run on PRs with unmodified code and dependency files. on: pull_request: paths-ignore: # code and tests - - '**/*.rs' + - "**/*.rs" # hard-coded checkpoints and proptest regressions - - '**/*.txt' + - "**/*.txt" # test data snapshots - - '**/*.snap' + - "**/*.snap" # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' + - "**/Cargo.toml" + - "**/Cargo.lock" # configuration files - - '.cargo/config.toml' - - '**/clippy.toml' + - ".cargo/config.toml" + - "**/clippy.toml" # workflow definitions - - 'docker/**' - - '.dockerignore' - - '.github/workflows/ci-unit-tests-docker.yml' - - '.github/workflows/sub-deploy-integration-tests-gcp.yml' - - '.github/workflows/sub-find-cached-disks.yml' - - '.github/workflows/sub-build-docker-image.yml' - -# IMPORTANT -# -# The job names in `ci-integration-tests-gcp.yml`, `ci-integration-tests-gcp.patch.yml` and -# `ci-integration-tests-gcp.patch-external.yml` must be kept in sync. + - "docker/**" + - ".dockerignore" + - ".github/workflows/ci-tests.yml" + - ".github/workflows/sub-ci-unit-tests-docker.yml" + - ".github/workflows/sub-ci-integration-tests-gcp.yml" + - ".github/workflows/sub-deploy-integration-tests-gcp.yml" + - ".github/workflows/sub-find-cached-disks.yml" + - ".github/workflows/sub-build-docker-image.yml" + +#! IMPORTANT +#! +#! The job names in `sub-ci-unit-tests-docker.yml`, `sub-ci-integration-tests-gcp.yml`, +#! `ci-tests.patch.yml` and `ci-tests.patch-external.yml` must be kept in sync. jobs: - # We don't patch the testnet job, because testnet isn't required to merge (it's too unstable) - get-available-disks: - name: Check if cached state disks exist for Mainnet / Check if cached state disks exist + build: + name: Build CI Docker / Build images runs-on: ubuntu-latest steps: - run: 'echo "No build required"' - build: - name: Build CI Docker / Build images + #### + ## The following jobs are related to sub-ci-unit-tests-docker.yml + ### + test-all: + name: Test all + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-fake-activation-heights: + name: Test with fake activation heights + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-empty-sync: + name: Test checkpoint sync from empty state + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-lightwalletd-integration: + name: Test integration with lightwalletd + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-configuration-file: + name: Test CI default Docker config file / Test default-conf in Docker + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-configuration-file-testnet: + name: Test CI testnet Docker config file / Test default-conf in Docker + needs: build + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + test-zebra-conf-path: + name: Test CI custom Docker config file / Test custom-conf in Docker + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + + + #### + ## The following jobs are related to sub-ci-integration-tests-gcp.yml + ### + + # We don't patch the testnet job, because testnet isn't required to merge (it's too unstable) + get-available-disks: + name: Check if cached state disks exist for Mainnet / Check if cached state disks exist runs-on: ubuntu-latest steps: - run: 'echo "No build required"' diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml new file mode 100644 index 00000000000..2c8ce336706 --- /dev/null +++ b/.github/workflows/ci-tests.yml @@ -0,0 +1,150 @@ +# This workflow builds a Zebra Docker image and runs integration and unit tests +# on the Zebra codebase. It is designed to add the different test workflows +name: Run tests + +# Ensures that only one workflow task will run at a time. Previous builds, if +# already in process, will get cancelled. Only the latest commit will be allowed +# to run, cancelling any workflows in between +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + schedule: + # Run this job every Friday at mid-day UTC + # This is limited to the Zebra and lightwalletd Full Sync jobs + # TODO: we should move this behavior to a separate workflow + - cron: "0 12 * * 5" + + workflow_dispatch: + inputs: + network: + default: "Mainnet" + description: "Network to deploy: Mainnet or Testnet" + required: true + regenerate-disks: + type: boolean + default: false + description: "Just run a Zebra checkpoint sync and update checkpoint disks" + required: true + run-full-sync: + type: boolean + default: false + description: "Just run a Zebra full sync on `network`, and update tip disks" + required: true + run-lwd-sync: + type: boolean + default: false + description: "Just run a lightwalletd full sync and update tip disks" + required: true + force_save_to_disk: + required: false + type: boolean + default: false + description: "Force tests to always create a cached state disk, if they already create disks" + no_cache: + description: "Disable the Docker cache for this build" + required: false + type: boolean + default: false + + pull_request: + # Run only on PRs that modify Rust code or dependencies. + paths: + # code and tests + - "**/*.rs" + # hard-coded checkpoints and proptest regressions + - "**/*.txt" + # test data snapshots + - "**/*.snap" + # dependencies + - "**/Cargo.toml" + - "**/Cargo.lock" + # configuration files + - ".cargo/config.toml" + - "**/clippy.toml" + # workflow definitions + - "docker/**" + - ".dockerignore" + - ".github/workflows/ci-tests.yml" + - ".github/workflows/sub-ci-unit-tests-docker.yml" + - ".github/workflows/sub-ci-integration-tests-gcp.yml" + - ".github/workflows/sub-deploy-integration-tests-gcp.yml" + - ".github/workflows/sub-find-cached-disks.yml" + - ".github/workflows/sub-build-docker-image.yml" + + push: + # Run only on main branch updates that modify Rust code or dependencies. + branches: + - main + paths: + # code and tests + - "**/*.rs" + # hard-coded checkpoints and proptest regressions + - "**/*.txt" + # test data snapshots + - "**/*.snap" + # dependencies + - "**/Cargo.toml" + - "**/Cargo.lock" + # configuration files + - ".cargo/config.toml" + - "**/clippy.toml" + # workflow definitions + - "docker/**" + - ".dockerignore" + - ".github/workflows/ci-tests.yml" + - ".github/workflows/sub-ci-unit-tests-docker.yml" + - ".github/workflows/sub-ci-integration-tests-gcp.yml" + - ".github/workflows/sub-deploy-integration-tests-gcp.yml" + - ".github/workflows/sub-find-cached-disks.yml" + - ".github/workflows/sub-build-docker-image.yml" + +env: + RUST_LOG: ${{ vars.RUST_LOG }} + RUST_BACKTRACE: ${{ vars.RUST_BACKTRACE }} + RUST_LIB_BACKTRACE: ${{ vars.RUST_LIB_BACKTRACE }} + COLORBT_SHOW_HIDDEN: ${{ vars.COLORBT_SHOW_HIDDEN }} + CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }} + +#! IMPORTANT +#! +#! The job names in `sub-ci-unit-tests-docker.yml`, `sub-ci-integration-tests-gcp.yml`, +#! `ci-tests.patch.yml` and `ci-tests.patch-external.yml` must be kept in sync. +jobs: + # Build the docker image used by the tests. + # + # The default network in the Zebra config in the image is mainnet, unless a manually triggered + # workflow or repository variable is configured differently. Testnet jobs change that config to + # testnet when running the image. + build: + name: Build images + # Skip PRs from external repositories, let them pass, and then Mergify will check them + if: ${{ !startsWith(github.event_name, 'pull') || !github.event.pull_request.head.repo.fork }} + uses: ./.github/workflows/sub-build-docker-image.yml + with: + dockerfile_path: ./docker/Dockerfile + dockerfile_target: tests + image_name: ${{ vars.CI_IMAGE_NAME }} + no_cache: ${{ inputs.no_cache || false }} + rust_backtrace: full + rust_lib_backtrace: full + rust_log: info + + # Runs Zebra unit tests + unit-tests: + name: Unit tests + # Skip Unit tests when the event is a scheduled run, as this is just needed for integration tests + if: ${{ github.event_name != 'schedule' }} + needs: build + uses: ./.github/workflows/sub-ci-unit-tests-docker.yml + with: + image_digest: ${{ needs.build.outputs.image_digest }} + secrets: inherit + + # Runs Zebra integration tests + integration-tests: + name: Integration tests + needs: build + uses: ./.github/workflows/sub-ci-integration-tests-gcp.yml + secrets: inherit diff --git a/.github/workflows/ci-unit-tests-docker.patch-external.yml b/.github/workflows/ci-unit-tests-docker.patch-external.yml deleted file mode 100644 index b68d1c4448f..00000000000 --- a/.github/workflows/ci-unit-tests-docker.patch-external.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Workflow patches for skipping Google Cloud unit test CI on PRs from external repositories. -name: Docker Unit Tests - -# Run on PRs from external repositories, let them pass, and then Mergify will check them. -# GitHub doesn't support filtering workflows by source branch names, so we have to do it for each -# job. -on: - pull_request: - -# IMPORTANT -# -# The job names in `ci-unit-tests-docker.yml`, `ci-unit-tests-docker.patch.yml` and -# `ci-unit-tests-docker.patch-external.yml` must be kept in sync. -jobs: - build: - name: Build CI Docker / Build images - # Only run on PRs from external repositories. - if: ${{ startsWith(github.event_name, 'pull') && github.event.pull_request.head.repo.fork }} - runs-on: ubuntu-latest - steps: - - run: 'echo "Skipping job on fork"' - - test-all: - name: Test all - # This dependency allows all these jobs to depend on a single condition, making it easier to - # change. - needs: build - runs-on: ubuntu-latest - steps: - - run: 'echo "Skipping job on fork"' - - test-fake-activation-heights: - name: Test with fake activation heights - needs: build - runs-on: ubuntu-latest - steps: - - run: 'echo "Skipping job on fork"' - - test-empty-sync: - name: Test checkpoint sync from empty state - needs: build - runs-on: ubuntu-latest - steps: - - run: 'echo "Skipping job on fork"' - - test-lightwalletd-integration: - name: Test integration with lightwalletd - needs: build - runs-on: ubuntu-latest - steps: - - run: 'echo "Skipping job on fork"' - - test-configuration-file: - name: Test CI default Docker config file / Test default-conf in Docker - needs: build - runs-on: ubuntu-latest - steps: - - run: 'echo "Skipping job on fork"' - - test-configuration-file-testnet: - name: Test CI testnet Docker config file / Test default-conf in Docker - needs: build - runs-on: ubuntu-latest - steps: - - run: 'echo "Skipping job on fork"' - - test-zebra-conf-path: - name: Test CI custom Docker config file / Test custom-conf in Docker - needs: build - runs-on: ubuntu-latest - steps: - - run: 'echo "Skipping job on fork"' diff --git a/.github/workflows/ci-unit-tests-docker.patch.yml b/.github/workflows/ci-unit-tests-docker.patch.yml deleted file mode 100644 index fa4969837ce..00000000000 --- a/.github/workflows/ci-unit-tests-docker.patch.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Workflow patches for skipping unit test CI when Rust code or dependencies aren't modified in a PR. -name: Docker Unit Tests - -# Run on PRs with unmodified code and dependency files. -on: - pull_request: - paths-ignore: - # code and tests - - '**/*.rs' - # hard-coded checkpoints and proptest regressions - - '**/*.txt' - # test data snapshots - - '**/*.snap' - # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' - # configuration files - - '.cargo/config.toml' - - '**/clippy.toml' - # workflow definitions - - 'docker/**' - - '.dockerignore' - - '.github/workflows/ci-unit-tests-docker.yml' - - '.github/workflows/sub-deploy-integration-tests-gcp.yml' - - '.github/workflows/sub-find-cached-disks.yml' - - '.github/workflows/sub-build-docker-image.yml' - -# IMPORTANT -# -# The job names in `ci-unit-tests-docker.yml`, `ci-unit-tests-docker.patch.yml` and -# `ci-unit-tests-docker.patch-external.yml` must be kept in sync. -jobs: - build: - name: Build CI Docker / Build images - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-all: - name: Test all - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-fake-activation-heights: - name: Test with fake activation heights - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-empty-sync: - name: Test checkpoint sync from empty state - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-lightwalletd-integration: - name: Test integration with lightwalletd - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-configuration-file: - name: Test CI default Docker config file / Test default-conf in Docker - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-configuration-file-testnet: - name: Test CI testnet Docker config file / Test default-conf in Docker - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' - - test-zebra-conf-path: - name: Test CI custom Docker config file / Test custom-conf in Docker - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required"' diff --git a/.github/workflows/ci-integration-tests-gcp.yml b/.github/workflows/sub-ci-integration-tests-gcp.yml similarity index 89% rename from .github/workflows/ci-integration-tests-gcp.yml rename to .github/workflows/sub-ci-integration-tests-gcp.yml index b26510280d8..7e8bd141d54 100644 --- a/.github/workflows/ci-integration-tests-gcp.yml +++ b/.github/workflows/sub-ci-integration-tests-gcp.yml @@ -5,103 +5,32 @@ # Each test has a description of the conditions under which it runs. name: Integration Tests on GCP -# Ensures that only one workflow task will run at a time. Previous builds, if -# already in process, will get cancelled. Only the latest commit will be allowed -# to run, cancelling any workflows in between -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - on: - schedule: - # Run this job every Friday at mid-day UTC - # This is limited to the Zebra and lightwalletd Full Sync jobs - # TODO: we should move this behavior to a separate workflow - - cron: "0 12 * * 5" - - workflow_dispatch: + workflow_call: inputs: network: default: "Mainnet" - description: "Network to deploy: Mainnet or Testnet" - required: true + type: string regenerate-disks: - type: boolean default: false - description: "Just run a Zebra checkpoint sync and update checkpoint disks" - required: true - run-full-sync: type: boolean + run-full-sync: default: false - description: "Just run a Zebra full sync on `network`, and update tip disks" - required: true - run-lwd-sync: type: boolean + run-lwd-sync: default: false - description: "Just run a lightwalletd full sync and update tip disks" - required: true - force_save_to_disk: - required: false type: boolean + force_save_to_disk: default: false - description: "Force tests to always create a cached state disk, if they already create disks" - no_cache: - description: "Disable the Docker cache for this build" - required: false type: boolean + no_cache: default: false + type: boolean - pull_request: - # Skip PRs where Rust code and dependencies aren't modified. - paths: - # code and tests - - "**/*.rs" - # hard-coded checkpoints and proptest regressions - - "**/*.txt" - # test data snapshots - - "**/*.snap" - # dependencies - - "**/Cargo.toml" - - "**/Cargo.lock" - # configuration files - - ".cargo/config.toml" - - "**/clippy.toml" - # workflow definitions - - "docker/**" - - ".github/workflows/ci-integration-tests-gcp.yml" - - ".github/workflows/sub-deploy-integration-tests-gcp.yml" - - ".github/workflows/sub-build-docker-image.yml" - - ".github/workflows/sub-find-cached-disks.yml" - - push: - # Skip main branch updates where Rust code and dependencies aren't modified. - branches: - - main - paths: - # code and tests - - "**/*.rs" - # hard-coded checkpoints and proptest regressions - - "**/*.txt" - # test data snapshots - - "**/*.snap" - # dependencies - - "**/Cargo.toml" - - "**/Cargo.lock" - # configuration files - - ".cargo/config.toml" - - "**/clippy.toml" - # workflow definitions - - "docker/**" - - ".dockerignore" - - ".github/workflows/ci-integration-tests-gcp.yml" - - ".github/workflows/sub-deploy-integration-tests-gcp.yml" - - ".github/workflows/sub-find-cached-disks.yml" - - ".github/workflows/sub-build-docker-image.yml" - -# IMPORTANT -# -# The job names in `ci-integration-tests-gcp.yml`, `ci-integration-tests-gcp.patch.yml` and -# `ci-integration-tests-gcp.patch-external.yml` must be kept in sync. +#! IMPORTANT +#! +#! The job names in `ci-integration-tests-gcp.yml`, `ci-integration-tests-gcp.patch.yml` and +#! `ci-integration-tests-gcp.patch-external.yml` must be kept in sync. jobs: # to also run a job on Mergify head branches, # add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`: @@ -132,24 +61,6 @@ jobs: with: network: "Testnet" - # Build the docker image used by the tests. - # - # The default network in the Zebra config in the image is mainnet, unless a manually triggered - # workflow or repository variable is configured differently. Testnet jobs change that config to - # testnet when running the image. - build: - name: Build CI Docker - if: ${{ !startsWith(github.event_name, 'pull') || !github.event.pull_request.head.repo.fork }} - uses: ./.github/workflows/sub-build-docker-image.yml - with: - dockerfile_path: ./docker/Dockerfile - dockerfile_target: tests - image_name: ${{ vars.CI_IMAGE_NAME }} - no_cache: ${{ inputs.no_cache || false }} - rust_backtrace: full - rust_lib_backtrace: full - rust_log: info - # zebrad cached checkpoint state tests # Regenerate mandatory checkpoint Zebra cached state disks. @@ -161,7 +72,7 @@ jobs: # Note: the output from get-available-disks should match with the caller workflow inputs regenerate-stateful-disks: name: Zebra checkpoint - needs: [build, get-available-disks] + needs: [get-available-disks] uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ !fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || github.event.inputs.regenerate-disks == 'true' }} with: @@ -218,7 +129,7 @@ jobs: # Note: the output from get-available-disks should match with the caller workflow inputs test-full-sync: name: Zebra tip - needs: [build, get-available-disks] + needs: [get-available-disks] uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet') }} with: @@ -329,7 +240,7 @@ jobs: # Note: the output from get-available-disks-testnet should match with the caller workflow inputs test-full-sync-testnet: name: Zebra tip on testnet - needs: [build, get-available-disks-testnet] + needs: [get-available-disks-testnet] uses: ./.github/workflows/sub-deploy-integration-tests-gcp.yml if: ${{ (github.event_name == 'schedule' && vars.SCHEDULE_TESTNET_FULL_SYNC == 'true') || !fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Testnet') }} with: diff --git a/.github/workflows/ci-unit-tests-docker.yml b/.github/workflows/sub-ci-unit-tests-docker.yml similarity index 67% rename from .github/workflows/ci-unit-tests-docker.yml rename to .github/workflows/sub-ci-unit-tests-docker.yml index 3b44aa4550c..3f80d24ebbd 100644 --- a/.github/workflows/ci-unit-tests-docker.yml +++ b/.github/workflows/sub-ci-unit-tests-docker.yml @@ -13,74 +13,18 @@ # 8. 'test-zebra-conf-path': Tests Zebra using a custom Docker configuration. name: Docker Unit Tests -# Ensures that only one workflow task will run at a time. Previous builds, if -# already in process, will get cancelled. Only the latest commit will be allowed -# to run, cancelling any workflows in between -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - on: - workflow_dispatch: + workflow_call: inputs: + image_digest: + type: string network: + type: string default: 'Mainnet' - description: 'Network to deploy: Mainnet or Testnet' - required: true no_cache: - description: 'Disable the Docker cache for this build' - required: false type: boolean default: false - pull_request: - # Skip PRs where Rust code and dependencies aren't modified. - paths: - # code and tests - - '**/*.rs' - # hard-coded checkpoints and proptest regressions - - '**/*.txt' - # test data snapshots - - '**/*.snap' - # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' - # configuration files - - '.cargo/config.toml' - - '**/clippy.toml' - # workflow definitions - - 'docker/**' - - '.github/workflows/ci-unit-tests-docker.yml' - - '.github/workflows/sub-deploy-integration-tests-gcp.yml' - - '.github/workflows/sub-build-docker-image.yml' - - '.github/workflows/sub-find-cached-disks.yml' - - '.github/workflows/sub-test-zebra-config.yml' - - push: - branches: - - main - # Skip main branch updates where Rust code and dependencies aren't modified. - paths: - # code and tests - - '**/*.rs' - # hard-coded checkpoints and proptest regressions - - '**/*.txt' - # test data snapshots - - '**/*.snap' - # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' - # configuration files - - '.cargo/config.toml' - - '**/clippy.toml' - # workflow definitions - - 'docker/**' - - '.dockerignore' - - '.github/workflows/ci-unit-tests-docker.yml' - - '.github/workflows/sub-deploy-integration-tests-gcp.yml' - - '.github/workflows/sub-find-cached-disks.yml' - - '.github/workflows/sub-build-docker-image.yml' - env: RUST_LOG: ${{ vars.RUST_LOG }} RUST_BACKTRACE: ${{ vars.RUST_BACKTRACE }} @@ -88,30 +32,11 @@ env: COLORBT_SHOW_HIDDEN: ${{ vars.COLORBT_SHOW_HIDDEN }} CARGO_INCREMENTAL: ${{ vars.CARGO_INCREMENTAL }} -# IMPORTANT -# -# The job names in `ci-unit-tests-docker.yml`, `ci-unit-tests-docker.patch.yml` and -# `ci-unit-tests-docker.patch-external.yml` must be kept in sync. +#! IMPORTANT +#! +#! The job names in `ci-unit-tests-docker.yml`, `ci-unit-tests-docker.patch.yml` and +#! `ci-unit-tests-docker.patch-external.yml` must be kept in sync. jobs: - # Build the docker image used by the tests. - # - # The default network in the Zebra config in the image is mainnet, unless a manually triggered - # workflow or repository variable is configured differently. Testnet jobs change that config to - # testnet when running the image. - build: - name: Build CI Docker - # Skip PRs from external repositories, let them pass, and then Mergify will check them - if: ${{ !startsWith(github.event_name, 'pull') || !github.event.pull_request.head.repo.fork }} - uses: ./.github/workflows/sub-build-docker-image.yml - with: - dockerfile_path: ./docker/Dockerfile - dockerfile_target: tests - image_name: ${{ vars.CI_IMAGE_NAME }} - no_cache: ${{ inputs.no_cache || false }} - rust_backtrace: full - rust_lib_backtrace: full - rust_log: info - # Run all the zebra tests, including tests that are ignored by default. # # - We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests. @@ -119,7 +44,6 @@ jobs: name: Test all timeout-minutes: 180 runs-on: ubuntu-latest-xl - needs: build steps: - uses: r7kamura/rust-problem-matchers@v1.5.0 @@ -136,8 +60,8 @@ jobs: env: NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} - docker run --tty -e NETWORK -e RUN_ALL_TESTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run --tty -e NETWORK -e RUN_ALL_TESTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} # Run unit, basic acceptance tests, and ignored tests with experimental features. # @@ -145,8 +69,8 @@ jobs: env: NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} - docker run --tty -e NETWORK -e RUN_ALL_EXPERIMENTAL_TESTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run --tty -e NETWORK -e RUN_ALL_EXPERIMENTAL_TESTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} # Run state tests with fake activation heights. # @@ -161,7 +85,6 @@ jobs: name: Test with fake activation heights timeout-minutes: 60 runs-on: ubuntu-latest - needs: build steps: - uses: r7kamura/rust-problem-matchers@v1.5.0 @@ -174,15 +97,14 @@ jobs: env: NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} - docker run --tty -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run --tty -e NETWORK -e TEST_FAKE_ACTIVATION_HEIGHTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state. test-empty-sync: name: Test checkpoint sync from empty state timeout-minutes: 60 runs-on: ubuntu-latest - needs: build steps: - uses: r7kamura/rust-problem-matchers@v1.5.0 @@ -195,15 +117,14 @@ jobs: env: NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} - docker run --tty -e NETWORK -e TEST_ZEBRA_EMPTY_SYNC=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run --tty -e NETWORK -e TEST_ZEBRA_EMPTY_SYNC=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} # Test launching lightwalletd with an empty lightwalletd and Zebra state. test-lightwalletd-integration: name: Test integration with lightwalletd timeout-minutes: 60 runs-on: ubuntu-latest - needs: build steps: - uses: r7kamura/rust-problem-matchers@v1.5.0 @@ -216,17 +137,16 @@ jobs: env: NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }} run: | - docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} - docker run --tty -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} + docker run --tty -e NETWORK -e ZEBRA_TEST_LIGHTWALLETD=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} # Test that Zebra works using the default config with the latest Zebra version. test-configuration-file: name: Test CI default Docker config file - needs: build uses: ./.github/workflows/sub-test-zebra-config.yml with: test_id: 'default-conf' - docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} grep_patterns: '-e "net.*=.*Main.*estimated progress to chain tip.*BeforeOverwinter"' test_variables: '-e NETWORK' network: 'Mainnet' @@ -234,12 +154,11 @@ jobs: # Test reconfiguring the the docker image for tesnet. test-configuration-file-testnet: name: Test CI testnet Docker config file - needs: build # Make sure Zebra can sync the genesis block on testnet uses: ./.github/workflows/sub-test-zebra-config.yml with: test_id: 'testnet-conf' - docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} grep_patterns: '-e "net.*=.*Test.*estimated progress to chain tip.*Genesis" -e "net.*=.*Test.*estimated progress to chain tip.*BeforeOverwinter"' # TODO: improve the entrypoint to avoid using `ENTRYPOINT_FEATURES=""` test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="/etc/zebrad/zebrad.toml" -e ENTRYPOINT_FEATURES=""' @@ -248,11 +167,10 @@ jobs: # Test that Zebra works using $ZEBRA_CONF_PATH config test-zebra-conf-path: name: Test CI custom Docker config file - needs: build uses: ./.github/workflows/sub-test-zebra-config.yml with: test_id: 'custom-conf' - docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ needs.build.outputs.image_digest }} + docker_image: ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }} grep_patterns: '-e "loaded zebrad config.*config_path.*=.*v1.0.0-rc.2.toml"' test_variables: '-e NETWORK -e ZEBRA_CONF_PATH="zebrad/tests/common/configs/v1.0.0-rc.2.toml"' network: ${{ inputs.network || vars.ZCASH_NETWORK }}