diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index e7bbb8c78070..1f31ceb391b8 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -59,6 +59,9 @@ stages: # Presubmit/default - ${{ if eq(variables.pipelineDefault, true) }}: - template: stages.yml + parameters: + buildStageDeps: + - env # Scheduled run anywhere - ${{ if eq(variables.pipelineScheduled, true) }}: diff --git a/.azure-pipelines/stage/verify.yml b/.azure-pipelines/stage/verify.yml index 5f7ddabe3d02..d423b07feeb2 100644 --- a/.azure-pipelines/stage/verify.yml +++ b/.azure-pipelines/stage/verify.yml @@ -9,55 +9,59 @@ parameters: - name: authGCP type: string default: "" +- name: authGithubWorkflow + type: string + default: "" +- name: authGithubWorkflowAppId + type: string + default: "" +- name: authGithubWorkflowInstallId + type: string + default: "" + +- name: runDocker + displayName: "Run Docker" + type: string + default: true + +# TODO(phlax): improve docker publishing job and move this there jobs: - job: examples displayName: Examples (Docker/x64) - condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true')) + condition: | + and(not(canceled()), + eq(${{ parameters.runDocker }}, 'true')) + timeoutInMinutes: 120 pool: vmImage: "ubuntu-20.04" steps: - - bash: .azure-pipelines/cleanup.sh - displayName: "Removing tools from agent" - - bash: | - set -e - - if [[ "$BUILD_REASON" == "PullRequest" ]]; then - DOWNLOAD_PATH="$(git rev-parse HEAD | head -c7)" - else - DOWNLOAD_PATH="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-${BUILD_SOURCEBRANCHNAME}}" - fi - - tmpdir=$(mktemp -d) - cd "$tmpdir" - images=("" "contrib" "google-vrp") - for image in "${images[@]}"; do - if [[ -n "$image" ]]; then - variant="${image}-dev" - filename="envoy-${image}.tar" - else - variant=dev - filename="envoy.tar" - fi - echo "Download docker image (https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/docker/${filename}) ..." - curl -sLO "https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/docker/${filename}" - echo "Copy oci image: oci-archive:${filename} docker-daemon:envoyproxy/envoy:${variant}" - skopeo copy -q "oci-archive:${filename}" "docker-daemon:envoyproxy/envoy:${variant}" - rm "$filename" - done - docker images | grep envoy + - task: DownloadSecureFile@1 + name: WorkflowTriggerKey + displayName: 'Download workflow trigger key' + inputs: + secureFile: '${{ parameters.authGithubWorkflow }}' - bash: | set -e - export DEBIAN_FRONTEND=noninteractive - sudo apt-get -qq update -y - sudo apt-get -qq install -y --no-install-recommends expect + KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)" + echo "##vso[task.setvariable variable=value;isoutput=true]$KEY" + name: key - - bash: ./ci/do_ci.sh verify_examples - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - NO_BUILD_SETUP: 1 - # ENVOY_EXAMPLES_DEBUG: 1 + - template: ../bazel.yml + parameters: + ciTarget: trigger_verify + authGithub: "$(key.value)" + env: + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + ENVOY_BRANCH: "$(System.PullRequest.TargetBranch)" + ENVOY_COMMIT: "$(System.PullRequest.SourceCommitId)" + ENVOY_HEAD_REF: "$(Build.SourceBranch)" + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + ENVOY_BRANCH: "$(Build.SourceBranch)" + GITHUB_APP_ID: ${{ parameters.authGithubWorkflowAppId }} + GITHUB_INSTALL_ID: ${{ parameters.authGithubWorkflowInstallId }} + ENVOY_REPO: phlax/envoy - job: packages_x64 displayName: Debs (x64) @@ -83,6 +87,7 @@ jobs: BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }} displayName: "Verify packages" + - job: packages_arm64 displayName: Debs (arm64) condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true')) diff --git a/.azure-pipelines/stages.yml b/.azure-pipelines/stages.yml index 6a36bb8da161..e678f892e541 100644 --- a/.azure-pipelines/stages.yml +++ b/.azure-pipelines/stages.yml @@ -169,11 +169,17 @@ stages: - stage: verify displayName: Verify dependsOn: ["env", "publish"] + variables: + RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']] jobs: - template: stage/verify.yml parameters: authGCP: $(GcpServiceAccountKey) bucketGCP: $(GcsArtifactBucket) + authGithubWorkflow: $(GitHubPublicRepoWorkflowKey) + authGithubWorkflowAppId: $(GitHubPublicRepoWorkflowAppId) + authGithubWorkflowInstallId: $(GitHubPublicRepoWorkflowInstallId) + runDocker: variables['RUN_DOCKER'] - stage: macos displayName: macOS diff --git a/.github/workflows/check-deps.yml b/.github/workflows/check-deps.yml index 8754308add19..4be8b0a3cc4b 100644 --- a/.github/workflows/check-deps.yml +++ b/.github/workflows/check-deps.yml @@ -3,7 +3,6 @@ name: Check dependencies on: schedule: - cron: '0 8 * * *' - workflow_dispatch: permissions: read-all @@ -11,7 +10,12 @@ permissions: read-all jobs: build: runs-on: ubuntu-20.04 - if: github.repository == 'envoyproxy/envoy' + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || !contains(github.actor, '[bot]')) + }} permissions: contents: read # to fetch code (actions/checkout) issues: write # required to open/close dependency issues diff --git a/.github/workflows/envoy-verify.yml b/.github/workflows/envoy-verify.yml new file mode 100644 index 000000000000..5fc1636c768c --- /dev/null +++ b/.github/workflows/envoy-verify.yml @@ -0,0 +1,87 @@ +name: Verify/examples + +on: + # This runs untrusted code, do not expose secrets in the verify job + workflow_dispatch: + inputs: + ref: + description: "Git SHA ref to checkout" + sha: + description: "Git SHA of commit HEAD (ie last commit of PR)" + head_ref: + description: "Ref for grouping PRs" + +concurrency: + group: ${{ github.event.inputs.head_ref || github.run_id }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check: + if: | + ${{ + github.repository == 'phlax/envoy' + && (!contains(github.actor, '[bot]') + || github.actor == trigger-workflow-verify-examples[bot]) + }} + uses: ./.github/workflows/workflow-start.yml + permissions: + contents: read + statuses: write + with: + workflowName: ${{ github.workflow }} + + # Runs untrusted code + verify-examples: + runs-on: ubuntu-20.04 + needs: check + steps: + # Checkout the repo at provided commit + - name: 'Checkout Repository' + uses: actions/checkout@v3 + with: + ref: "${{ inputs.ref }}" + + - run: | + set -e + + BUCKET="envoy-pr" + if [[ "${{ github.actor }}" == "envoy-cibot" ]]; then + BUCKET="envoy-postsubmit" + fi + + DOWNLOAD_PATH="$(echo "${{ github.event.inputs.sha }}" | head -c 7)" + + tmpdir=$(mktemp -d) + cd "$tmpdir" + images=("" "contrib" "google-vrp") + for image in "${images[@]}"; do + if [[ -n "$image" ]]; then + variant="${image}-dev" + filename="envoy-${image}.tar" + else + variant=dev + filename="envoy.tar" + fi + fileurl="https://storage.googleapis.com/${BUCKET}/${DOWNLOAD_PATH}/docker/${filename}" + echo "Download docker image (${fileurl}) ..." + curl -sLO "$fileurl" + echo "Copy oci image: oci-archive:${filename} docker-daemon:envoyproxy/envoy:${variant}" + skopeo copy -q "oci-archive:${filename}" "docker-daemon:envoyproxy/envoy:${variant}" + rm "$filename" + done + docker images | grep envoy + + - run: | + set -e + export DEBIAN_FRONTEND=noninteractive + sudo apt-get -qq update -y + sudo apt-get -qq install -y --no-install-recommends expect + + - run: | + ./ci/do_ci.sh verify_examples + env: + NO_BUILD_SETUP: 1 + # ENVOY_EXAMPLES_DEBUG: 1 diff --git a/.github/workflows/mobile_release.yml b/.github/workflows/mobile_release.yml index 8beed1ab6268..df91222186e8 100644 --- a/.github/workflows/mobile_release.yml +++ b/.github/workflows/mobile_release.yml @@ -8,7 +8,12 @@ on: jobs: android_release_artifacts: - if: github.repository == 'envoyproxy/envoy' + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || !contains(github.actor, '[bot]')) + }} name: android_release_artifacts runs-on: ubuntu-20.04 timeout-minutes: 120 diff --git a/.github/workflows/pr_notifier.yml b/.github/workflows/pr_notifier.yml index dab80b2579f2..bd99723b5fba 100644 --- a/.github/workflows/pr_notifier.yml +++ b/.github/workflows/pr_notifier.yml @@ -14,7 +14,12 @@ jobs: pull-requests: read # for pr_notifier.py name: PR Notifier runs-on: ubuntu-20.04 - if: github.repository == 'envoyproxy/envoy' + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || !contains(github.actor, '[bot]')) + }} steps: - uses: actions/checkout@v3 - name: Set up Python 3.8 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c544b7374e6f..faec7e043697 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,7 +10,12 @@ jobs: pull-requests: write # for actions/stale to close stale PRs name: Prune Stale runs-on: ubuntu-20.04 - if: github.repository == 'envoyproxy/envoy' + if: | + ${{ + github.repository == 'envoyproxy/envoy' + && (github.event.schedule + || !contains(github.actor, '[bot]')) + }} steps: - name: Prune Stale diff --git a/.github/workflows/workflow-complete.yml b/.github/workflows/workflow-complete.yml new file mode 100644 index 000000000000..8b0d7f8c98d1 --- /dev/null +++ b/.github/workflows/workflow-complete.yml @@ -0,0 +1,61 @@ +name: Workflow complete +# This workflow is only required for externally triggered jobs that have manually +# set the check status for a commit/PR + +on: + # Do not run untrusted code here + workflow_run: + workflows: + - Verify/examples + types: + - completed + +permissions: + contents: read + +jobs: + complete: + runs-on: ubuntu-20.04 + permissions: + statuses: write + steps: + - name: 'Download artifact' + uses: actions/github-script@v6 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "state_sha" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/state_sha.zip`, Buffer.from(download.data)); + + - run: | + set -e + unzip state_sha.zip + STATE_SHA="$(cat state_sha)" + echo "state_sha=$STATE_SHA" >> "$GITHUB_OUTPUT" + STATE="${{ github.event.workflow_run.conclusion }}" + if [[ ${STATE} != "success" ]]; then + STATE=failure + fi + echo "state=${STATE}" >> "$GITHUB_OUTPUT" + id: job + - name: Complete status check + uses: envoyproxy/toolshed/gh-actions/status@a6e1c951217efae1ac6b2bf32c5a9729976442b8 + with: + authToken: ${{ secrets.GITHUB_TOKEN }} + context: ${{ github.event.workflow.name }} + state: ${{ steps.job.outputs.state }} + sha: ${{ steps.job.outputs.state_sha }} + target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }} diff --git a/.github/workflows/workflow-start.yml b/.github/workflows/workflow-start.yml new file mode 100644 index 000000000000..64c2999c1201 --- /dev/null +++ b/.github/workflows/workflow-start.yml @@ -0,0 +1,38 @@ +name: Workflow start +# This workflow is only required for externally triggered jobs that need to manually +# set the check status for a commit/PR + +on: + workflow_call: + inputs: + workflowName: + required: true + type: string + +permissions: + contents: read + +jobs: + start: + runs-on: ubuntu-20.04 + permissions: + statuses: write + steps: + - name: Start status check + uses: envoyproxy/toolshed/gh-actions/status@a6e1c951217efae1ac6b2bf32c5a9729976442b8 + with: + authToken: ${{ secrets.GITHUB_TOKEN }} + context: ${{ inputs.workflowName }} + state: 'pending' + sha: ${{ inputs.sha }} + target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: Save the SHA + env: + STATE_SHA: ${{ inputs.sha }} + run: | + mkdir -p ./sha + echo $STATE_SHA > ./sha/state_sha + - uses: actions/upload-artifact@v3 + with: + name: state_sha + path: sha/ diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index ec0b65060918..ce55f10cd821 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -164,6 +164,16 @@ envoy_entry_point( script = "envoy.project", ) +envoy_entry_point( + name = "trigger", + args = [ + "trigger", + PATH, + ], + pkg = "envoy.base.utils", + script = "envoy.project", +) + ''') _envoy_repo = repository_rule( diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 4d516321753c..19981296b2e5 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -679,6 +679,35 @@ case $CI_TARGET in bazel_envoy_binary_build sizeopt ;; + trigger_verify) + setup_clang_toolchain + WORKFLOW="envoy-verify.yml" + # This may be a merge commit in a PR + COMMIT="$(git rev-parse HEAD)" + # For a PR this should be the last actual commit + ENVOY_COMMIT="${ENVOY_COMMIT:-${COMMIT}}" + ENVOY_REPO="${ENVOY_REPO:-envoyproxy/envoy}" + # ENVOY_HEAD_REF must also be set in PRs to provide a unique key for job grouping and cancellation. + echo "Trigger workflow (${WORKFLOW})" + echo " Repo: ${ENVOY_REPO}" + echo " Branch: ${ENVOY_BRANCH}" + echo " Inputs:" + echo " ref: ${COMMIT}" + echo " sha: ${ENVOY_COMMIT}" + echo " head_ref: ${ENVOY_HEAD_REF}" + GITHUB_APP_KEY="$(echo "$GITHUB_TOKEN" | base64 -d -w0)" + export GITHUB_APP_KEY + INPUTS="{\"ref\":\"$COMMIT\",\"sha\":\"$ENVOY_COMMIT\",\"head_ref\":\"$ENVOY_HEAD_REF\"}" + bazel run "${BAZEL_BUILD_OPTIONS[@]}" \ + @envoy_repo//:trigger \ + -- --repo="$ENVOY_REPO" \ + --trigger-app-id="$GITHUB_APP_ID" \ + --trigger-installation-id="$GITHUB_INSTALL_ID" \ + --trigger-ref="$ENVOY_BRANCH" \ + --trigger-workflow="$WORKFLOW" \ + --trigger-inputs="$INPUTS" + ;; + tsan) setup_clang_toolchain echo "bazel TSAN debug build with tests" diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 30cbe0094613..7fd9bac73335 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -124,15 +124,21 @@ docker run --rm \ -e GOOGLE_BES_PROJECT_ID \ -e GCP_SERVICE_ACCOUNT_KEY \ -e NUM_CPUS \ + -e ENVOY_BRANCH \ -e ENVOY_RBE \ -e ENVOY_BUILD_IMAGE \ -e ENVOY_SRCDIR \ -e ENVOY_BUILD_TARGET \ -e ENVOY_BUILD_DEBUG_INFORMATION \ -e ENVOY_BUILD_FILTER_EXAMPLE \ + -e ENVOY_COMMIT \ + -e ENVOY_HEAD_REF \ + -e ENVOY_REPO \ -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ -e GCS_ARTIFACT_BUCKET \ -e GITHUB_TOKEN \ + -e GITHUB_APP_ID \ + -e GITHUB_INSTALL_ID \ -e NETLIFY_TRIGGER_URL \ -e BUILD_SOURCEBRANCHNAME \ -e BAZELISK_BASE_URL \ diff --git a/ci/upload_gcs_artifact.sh b/ci/upload_gcs_artifact.sh index 43c0032cc936..2c54feb13528 100755 --- a/ci/upload_gcs_artifact.sh +++ b/ci/upload_gcs_artifact.sh @@ -38,9 +38,9 @@ if [ ! -d "${SOURCE_DIRECTORY}" ]; then exit 1 fi -if [[ "$BUILD_REASON" == "PullRequest" ]] || [[ "$TARGET_SUFFIX" == "docs" ]]; then +if [[ "$BUILD_REASON" == "PullRequest" ]] || [[ "$TARGET_SUFFIX" == "docs" ]] || [[ "$TARGET_SUFFIX" == "docker" ]]; then # upload to the last commit sha (first 7 chars), either - # - docs build on main + # - docs/docker build on main, eg docs # -> https://storage.googleapis.com/envoy-postsubmit/$UPLOAD_PATH/docs/envoy-docs-rst.tar.gz # - PR build (commit sha from the developers branch) # -> https://storage.googleapis.com/envoy-pr/$UPLOAD_PATH/$TARGET_SUFFIX diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 361ce48bf0bd..03a1e836e27a 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -5,7 +5,7 @@ cffi>=1.15.0 colorama coloredlogs dependatool>=0.2.2 -envoy.base.utils>=0.4.10 +envoy.base.utils>=0.4.11 envoy.code.check>=0.5.1 envoy.dependency.check>=0.1.7 envoy.distribution.release>=0.0.9