From 362f9e60314b8ea015cd95e18d571822b50b2ed7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 26 Oct 2022 15:26:52 +0200 Subject: [PATCH] Update CI workflow - Do not merge Add new terraform deployment --- .github/workflows/ci.yml | 483 ++++++++++++--------------------------- 1 file changed, 151 insertions(+), 332 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 764ceacfbb..57d441864b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,351 +1,133 @@ name: CI -on: - push: - branches: # only run on branch push, tag push will be ignored - - '**' - paths-ignore: # ignore docs only changes since they use a dedicated workflows: docs.yml - - 'docs/**' - - 'mithril-explorer/**' +# do not run workflow twice on PRs +on: [push] jobs: - build-ubuntu-X64: - runs-on: ubuntu-22.04 - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Build Mithril workspace & publish artifacts - uses: ./.github/workflows/actions/build-upload-mithril-artifact - with: - cache-version: ${{ secrets.CACHE_VERSION }} - - - name: Publish End-to-end runner (${{ runner.os }}-${{ runner.arch }}) - uses: actions/upload-artifact@v3 - with: - name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }} - path: target/release/mithril-end-to-end - if-no-files-found: error - - build: - strategy: - fail-fast: false - matrix: - os: [ macos-12, windows-latest ] - - include: - # Only build client on windows & mac - - os: macos-12 - build-args: -p mithril-client - - os: windows-latest - build-args: -p mithril-client - runs-on: ${{ matrix.os }} + terraform: - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Build Mithril workspace & publish artifacts - uses: ./.github/workflows/actions/build-upload-mithril-artifact - with: - cache-version: ${{ secrets.CACHE_VERSION }} - build-args: ${{ matrix.build-args }} - - test: strategy: fail-fast: false matrix: - os: [ ubuntu-22.04, macos-12, windows-latest ] + environment: [ release-preprod, dev-preview, dev-preprod ] include: - - os: ubuntu-22.04 - build-args: --workspace - test-args: --workspace - # Only test client on windows & mac (since its the only binaries supported for those os for now) - - os: macos-12 - build-args: -p mithril-client - test-args: -p mithril-client - - os: windows-latest - build-args: -p mithril-client - test-args: -p mithril-client - - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - key: ${{ runner.os }}-cache-v${{ secrets.CACHE_VERSION }} - - - name: Install cargo tools - shell: bash - run: | - cargo install cargo2junit 2>/dev/null || true # Suppress the "binary `xyz` already exists in destination" error - - - name: Run tests - shell: bash - run: | - set -o pipefail && \ - cargo test --features portable --no-fail-fast ${{ matrix.test-args }} \ - -- -Z unstable-options --format json --report-time \ - | tee >(cargo2junit > test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }}.xml) - - - name: Upload Tests Results - uses: actions/upload-artifact@v3 - if: success() || failure() - with: - name: test-results${{ matrix.artifact-suffix }}-${{ runner.os }}-${{ runner.arch }} - path: | - ./**/test-results-*.xml - - check: + - environment: release-preprod + environment_prefix: release + cardano_network: preprod + mithril_api_domain: api.mithril.network + mithril_protocol_parameters: | + { + k = 5 + m = 100 + phi_f = 0.65 + } + mithril_signers: | + { + "1" = { + pool_id = "pool1zr907nmfsq5kalxdjju349nwg6f03lyfmcjfqcz52jf45gcgh03", + }, + } + terraform_backend_bucket: hydra-terraform-admin + google_region: europe-west1 + google_zone: europe-west1-b + google_machine_type: e2-highmem-2 + - environment: dev-preview + environment_prefix: dev + cardano_network: preview + mithril_api_domain: api.jpraynaud.me + mithril_protocol_parameters: | + { + k = 5 + m = 100 + phi_f = 0.65 + } + mithril_signers: | + { + "1" = { + pool_id = "pool15qde6mnkc0jgycm69ua0grwxmmu0tke54h5uhml0j8ndw3kcu9x", + }, + "2" = { + pool_id = "pool1k4rdvvuhy7h927psyew9syupwddy77t4j8lc74hpgzpvv7yklwg", + } + } + terraform_backend_bucket: mithril-terraform-state + google_region: europe-west1 + google_zone: europe-west1-b + google_machine_type: e2-medium + - environment: dev-preprod + environment_prefix: dev + cardano_network: preprod + mithril_api_domain: api.jpraynaud.me + mithril_protocol_parameters: | + { + k = 5 + m = 100 + phi_f = 0.75 + } + mithril_signers: | + { + "1" = { + pool_id = "pool1zr907nmfsq5kalxdjju349nwg6f03lyfmcjfqcz52jf45gcgh03", + }, + "2" = { + pool_id = "pool1l8ywwf6nfrfmrg6edj2qjh6rxpuepnzlsq9mhvjk9xr9swkht93", + } + } + terraform_backend_bucket: mithril-terraform-state + google_region: europe-west1 + google_zone: europe-west1-b + google_machine_type: e2-medium + runs-on: ubuntu-22.04 - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - components: clippy, rustfmt - override: true - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - key: ${{ runner.os }}-cache-v${{ secrets.CACHE_VERSION }} - - - name: Install cargo tools - if: steps.cargo-cache.outputs.cache-hit == false - shell: bash - run: | - cargo install cargo-sort 2>/dev/null || true # Suppress the "binary `xyz` already exists in destination" error - - - name: Cargo check - uses: actions-rs/cargo@v1 - with: - command: check - args: --release --all-targets - - - name: Clippy Check - uses: actions-rs/clippy-check@v1 - with: - name: clippy - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features --all-targets --no-deps -- -D warnings - - - name: Cargo fmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --check - - - name: Cargo sort - shell: bash - run: cargo sort -w -c - - run-test-lab: - runs-on: ubuntu-22.04 - needs: [ build-ubuntu-X64 ] - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Download binaries - uses: actions/download-artifact@v3 - with: - name: mithril-distribution-${{ runner.os }}-${{ runner.arch }} - path: ./bin - - - name: Download rust test runner - uses: actions/download-artifact@v3 - with: - name: mithril-end-to-end-${{ runner.os }}-${{ runner.arch }} - path: ./ - - - run: | - chmod +x ./bin/mithril-aggregator - chmod +x ./bin/mithril-client - chmod +x ./bin/mithril-signer - chmod +x ./mithril-end-to-end - mkdir artifacts - - - name: Test - run: ./mithril-end-to-end --bin-directory ./bin --work-directory=./artifacts --devnet-scripts-directory=./mithril-test-lab/mithril-devnet - - - name: Upload E2E Tests Artifacts - if: ${{ failure() }} - uses: actions/upload-artifact@v3 - with: - name: mithril-e2e-tests-artifacts-run_${{ github.run_number }}-attempt_${{ github.run_attempt }} - path: | - ./artifacts/* - # including node.sock makes the upload fails so exclude them: - !./artifacts/**/node.sock - # exclude cardano tools, saving ~50mb of data: - !./artifacts/devnet/cardano-cli - !./artifacts/devnet/cardano-node - if-no-files-found: error - - publish-tests-results: - if: success() || failure() - runs-on: ubuntu-22.04 - needs: - - test - steps: - - name: Download Tests Results (${{ runner.os }}-${{ runner.arch }}) - if: success() || failure() - uses: actions/download-artifact@v3 - with: - name: test-results-${{ runner.os }}-${{ runner.arch }} - - - name: Download Tests Results (macOS-X64) - if: success() || failure() - uses: actions/download-artifact@v3 - with: - name: test-results-macOS-X64 - - - name: Download Tests Results (Windows-X64) - if: success() || failure() - uses: actions/download-artifact@v3 - with: - name: test-results-Windows-X64 - - - name: Publish Unit Test Results - if: success() || failure() - uses: EnricoMi/publish-unit-test-result-action@v1 - with: - files: ./**/test-results-*.xml - - docker-mithril: - runs-on: ubuntu-22.04 - needs: - - build - - check - - test - - run-test-lab - strategy: - fail-fast: false - matrix: - project: [ mithril-aggregator, mithril-client, mithril-signer ] - - permissions: - contents: read - packages: write - - env: - PUSH_PACKAGES: ${{ github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref) }} - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository_owner }}/${{ matrix.project }} - DOCKER_FILE: ./${{ matrix.project }}/Dockerfile.ci - CONTEXT: . - GITHUB_REF: ${{ github.ref}} - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - unstable - type=raw,value={{branch}}-{{sha}} - - - name: Download built artifacts - uses: actions/download-artifact@v3 - with: - name: mithril-distribution-${{ runner.os }}-${{ runner.arch }} - path: ${{ matrix.project }} - - - name: Build and push Docker image - uses: docker/build-push-action@v3 - with: - context: ${{ env.CONTEXT }} - file: ${{ env.DOCKER_FILE }} - push: ${{ env.PUSH_PACKAGES }} - tags: ${{ steps.meta.outputs.tags }} - - unstable-release: - if: github.ref == 'refs/heads/main' || startsWith('refs/heads/hotfix', github.ref) - runs-on: ubuntu-22.04 - needs: - - build - - test - - run-test-lab - - check - steps: - - name: Download built artifacts (${{ runner.os }}-${{ runner.arch }}) - uses: actions/download-artifact@v3 - with: - name: mithril-distribution-${{ runner.os }}-${{ runner.arch }} - path: ./build - - - name: Download built artifacts (macOS-X64) - uses: actions/download-artifact@v3 - with: - name: mithril-distribution-macOS-X64 - path: ./build - - - name: Download built artifacts (Windows-X64) - uses: actions/download-artifact@v3 - with: - name: mithril-distribution-Windows-X64 - path: ./build - - - name: Update unstable release - uses: marvinpinto/action-automatic-releases@latest - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - automatic_release_tag: unstable - prerelease: true - title: Unstable Development Builds - files: build/* - - terraform: - runs-on: ubuntu-22.04 - needs: - - docker-mithril + #TODO: rollback needs block below + #needs: + # - docker-mithril + environment: ${{ matrix.environment }} env: - # Contains a JSON-formatted service account key - GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }} - # Contains a RSA private key - GCLOUD_PRIVATE_KEY: ${{ secrets.GCLOUD_PRIVATE_KEY }} - GENESIS_SECRET_KEY: ${{ secrets.TEST_ONLY_GENESIS_SECRET_KEY }} - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} + GENESIS_SECRET_KEY: ${{ secrets.GENESIS_SECRET_KEY }} + GENESIS_VERIFICATION_KEY_URL: ${{ secrets.GENESIS_VERIFICATION_KEY_URL }} + #TODO: remove env var value DOCKER_IMAGE_ID + DOCKER_IMAGE_ID: main-ce3d42d defaults: run: - working-directory: mithril-infra-legacy + working-directory: mithril-infra steps: + - name: Checkout sources uses: actions/checkout@v3 - - name: Get short SHA - id: slug - run: echo "sha8=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT + - name: Get Docker image id + #TODO: rollback to DOCKER_IMAGE_ID + run: echo "DOCKER_IMAGE_ID_NEW=${{ github.head_ref || github.ref_name }}-$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV + + - name: Prepare service account credentials + run: | + echo '${{ env.GOOGLE_APPLICATION_CREDENTIALS}}' > ./google-application-credentials.json + chmod u+x ./assets/tools/google-credentials-public-key.sh + ./assets/tools/google-credentials-public-key.sh ./google-application-credentials.json ./assets/ssh_keys curry + + - name: Prepare terraform variables + run: | + cat > ./env.variables.tfvars << EOF + environment_prefix = "${{ matrix.environment_prefix }}" + cardano_network = "${{ matrix.cardano_network }}" + google_region = "${{ matrix.google_region }}" + google_zone = "${{ matrix.google_zone }}" + google_machine_type = "${{ matrix.google_machine_type }}" + google_service_credentials_json_file = "./google-application-credentials.json" + mithril_api_domain = "${{ matrix.mithril_api_domain }}" + mithril_image_id = "${{ env.DOCKER_IMAGE_ID }}" + mithril_genesis_verification_key_url = "${{ env.GENESIS_VERIFICATION_KEY_URL }}" + mithril_genesis_secret_key = "${{ env.GENESIS_SECRET_KEY }}" + mithril_protocol_parameters = ${{ matrix.mithril_protocol_parameters }} + mithril_signers = ${{ matrix.mithril_signers }} + EOF + terraform fmt ./env.variables.tfvars + cat ./env.variables.tfvars - name: Setup Terraform uses: hashicorp/setup-terraform@v2 @@ -353,16 +135,53 @@ jobs: terraform_wrapper: false - name: Init Terraform - run: terraform init + run: | + GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform init -backend-config="bucket=${{ matrix.terraform_backend_bucket }}" -backend-config="prefix=terraform/mithril-${{ matrix.environment }}" - name: Check Terraform run: terraform fmt -check - name: Terraform Plan run: | - terraform plan -var "image_id=${{ env.BRANCH_NAME }}-${{ steps.slug.outputs.sha8 }}" -var 'private_key=${{ env.GCLOUD_PRIVATE_KEY }}' -var 'google_application_credentials_json=${{ env.GOOGLE_CREDENTIALS }}' -var 'genesis_secret_key=${{ env.GENESIS_SECRET_KEY }}' + GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform plan --var-file=./env.variables.tfvars + + - name: Update Pull Request + uses: actions/github-script@v6 + if: github.event_name == 'pull_request' + env: + PLAN: "terraform\n${{ steps.plan.outputs.stdout }}" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\` + #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\` + #### Terraform Plan 📖\`${{ steps.plan.outcome }}\` + #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\` + +
Show Plan + + \`\`\`\n + ${process.env.PLAN} + \`\`\` + +
+ + *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) - name: Terraform Apply - if: github.ref == 'refs/heads/main' + #TODO: rollback if condition below + #if: github.ref == 'refs/heads/main' && github.event_name == 'push' + run: | + GOOGLE_APPLICATION_CREDENTIALS=./google-application-credentials.json terraform apply -auto-approve --var-file=./env.variables.tfvars + + - name: Cleanup run: | - terraform apply -auto-approve -var "image_id=${{ env.BRANCH_NAME }}-${{ steps.slug.outputs.sha8 }}" -var 'private_key=${{ env.GCLOUD_PRIVATE_KEY }}' -var 'google_application_credentials_json=${{ env.GOOGLE_CREDENTIALS }}' -var 'genesis_secret_key=${{ env.GENESIS_SECRET_KEY }}' + rm -f ./env.variables.tfvars + rm -f ./google-application-credentials.json