Skip to content

Commit

Permalink
ci: Add github workflow for verify examples
Browse files Browse the repository at this point in the history
Signed-off-by: Ryan Northey <ryan@synca.io>
  • Loading branch information
phlax committed May 24, 2023
1 parent 19231ba commit c7bd38e
Show file tree
Hide file tree
Showing 15 changed files with 310 additions and 46 deletions.
3 changes: 3 additions & 0 deletions .azure-pipelines/pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ stages:
# Presubmit/default
- ${{ if eq(variables.pipelineDefault, true) }}:
- template: stages.yml
parameters:
buildStageDeps:
- env

# Scheduled run anywhere
- ${{ if eq(variables.pipelineScheduled, true) }}:
Expand Down
81 changes: 43 additions & 38 deletions .azure-pipelines/stage/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,55 +9,59 @@ parameters:
- name: authGCP
type: string
default: ""
- name: authGithubWorkflow
type: string
default: ""
- name: authGithubWorkflowAppId
type: string
default: ""
- name: authGithubWorkflowInstallId
type: string
default: ""

- name: runDocker
displayName: "Run Docker"
type: string
default: true


# TODO(phlax): improve docker publishing job and move this there
jobs:
- job: examples
displayName: Examples (Docker/x64)
condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'))
condition: |
and(not(canceled()),
eq(${{ parameters.runDocker }}, 'true'))
timeoutInMinutes: 120
pool:
vmImage: "ubuntu-20.04"
steps:
- bash: .azure-pipelines/cleanup.sh
displayName: "Removing tools from agent"
- bash: |
set -e
if [[ "$BUILD_REASON" == "PullRequest" ]]; then
DOWNLOAD_PATH="$(git rev-parse HEAD | head -c7)"
else
DOWNLOAD_PATH="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-${BUILD_SOURCEBRANCHNAME}}"
fi
tmpdir=$(mktemp -d)
cd "$tmpdir"
images=("" "contrib" "google-vrp")
for image in "${images[@]}"; do
if [[ -n "$image" ]]; then
variant="${image}-dev"
filename="envoy-${image}.tar"
else
variant=dev
filename="envoy.tar"
fi
echo "Download docker image (https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/docker/${filename}) ..."
curl -sLO "https://storage.googleapis.com/${{ parameters.bucketGCP }}/${DOWNLOAD_PATH}/docker/${filename}"
echo "Copy oci image: oci-archive:${filename} docker-daemon:envoyproxy/envoy:${variant}"
skopeo copy -q "oci-archive:${filename}" "docker-daemon:envoyproxy/envoy:${variant}"
rm "$filename"
done
docker images | grep envoy
- task: DownloadSecureFile@1
name: WorkflowTriggerKey
displayName: 'Download workflow trigger key'
inputs:
secureFile: '${{ parameters.authGithubWorkflow }}'

- bash: |
set -e
export DEBIAN_FRONTEND=noninteractive
sudo apt-get -qq update -y
sudo apt-get -qq install -y --no-install-recommends expect
KEY="$(cat $(WorkflowTriggerKey.secureFilePath) | base64 -w0)"
echo "##vso[task.setvariable variable=value;isoutput=true]$KEY"
name: key
- bash: ./ci/do_ci.sh verify_examples
env:
ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
NO_BUILD_SETUP: 1
# ENVOY_EXAMPLES_DEBUG: 1
- template: ../bazel.yml
parameters:
ciTarget: trigger_verify
authGithub: "$(key.value)"
env:
${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
ENVOY_BRANCH: "$(System.PullRequest.TargetBranch)"
ENVOY_COMMIT: "$(System.PullRequest.SourceCommitId)"
ENVOY_HEAD_REF: "$(Build.SourceBranch)"
${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
ENVOY_BRANCH: "$(Build.SourceBranch)"
GITHUB_APP_ID: ${{ parameters.authGithubWorkflowAppId }}
GITHUB_INSTALL_ID: ${{ parameters.authGithubWorkflowInstallId }}
ENVOY_REPO: phlax/envoy

- job: packages_x64
displayName: Debs (x64)
Expand All @@ -83,6 +87,7 @@ jobs:
BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
displayName: "Verify packages"

- job: packages_arm64
displayName: Debs (arm64)
condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.examplesOnly'], 'true'))
Expand Down
6 changes: 6 additions & 0 deletions .azure-pipelines/stages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -169,11 +169,17 @@ stages:
- stage: verify
displayName: Verify
dependsOn: ["env", "publish"]
variables:
RUN_DOCKER: $[stageDependencies.env.repo.outputs['run.docker']]
jobs:
- template: stage/verify.yml
parameters:
authGCP: $(GcpServiceAccountKey)
bucketGCP: $(GcsArtifactBucket)
authGithubWorkflow: $(GitHubPublicRepoWorkflowKey)
authGithubWorkflowAppId: $(GitHubPublicRepoWorkflowAppId)
authGithubWorkflowInstallId: $(GitHubPublicRepoWorkflowInstallId)
runDocker: variables['RUN_DOCKER']

- stage: macos
displayName: macOS
Expand Down
8 changes: 6 additions & 2 deletions .github/workflows/check-deps.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,19 @@ name: Check dependencies
on:
schedule:
- cron: '0 8 * * *'

workflow_dispatch:

permissions: read-all

jobs:
build:
runs-on: ubuntu-20.04
if: github.repository == 'envoyproxy/envoy'
if: |
${{
github.repository == 'envoyproxy/envoy'
&& (github.event.schedule
|| !contains(github.actor, '[bot]'))
}}
permissions:
contents: read # to fetch code (actions/checkout)
issues: write # required to open/close dependency issues
Expand Down
87 changes: 87 additions & 0 deletions .github/workflows/envoy-verify.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
name: Verify/examples

on:
# This runs untrusted code, do not expose secrets in the verify job
workflow_dispatch:
inputs:
ref:
description: "Git SHA ref to checkout"
sha:
description: "Git SHA of commit HEAD (ie last commit of PR)"
head_ref:
description: "Ref for grouping PRs"

concurrency:
group: ${{ github.event.inputs.head_ref || github.run_id }}-${{ github.workflow }}
cancel-in-progress: true

permissions:
contents: read

jobs:
check:
if: |
${{
github.repository == 'phlax/envoy'
&& (!contains(github.actor, '[bot]')
|| github.actor == trigger-workflow-verify-examples[bot])
}}
uses: ./.github/workflows/workflow-start.yml
permissions:
contents: read
statuses: write
with:
workflowName: ${{ github.workflow }}

# Runs untrusted code
verify-examples:
runs-on: ubuntu-20.04
needs: check
steps:
# Checkout the repo at provided commit
- name: 'Checkout Repository'
uses: actions/checkout@v3
with:
ref: "${{ inputs.ref }}"

- run: |
set -e
BUCKET="envoy-pr"
if [[ "${{ github.actor }}" == "envoy-cibot" ]]; then
BUCKET="envoy-postsubmit"
fi
DOWNLOAD_PATH="$(echo "${{ github.event.inputs.sha }}" | head -c 7)"
tmpdir=$(mktemp -d)
cd "$tmpdir"
images=("" "contrib" "google-vrp")
for image in "${images[@]}"; do
if [[ -n "$image" ]]; then
variant="${image}-dev"
filename="envoy-${image}.tar"
else
variant=dev
filename="envoy.tar"
fi
fileurl="https://storage.googleapis.com/${BUCKET}/${DOWNLOAD_PATH}/docker/${filename}"
echo "Download docker image (${fileurl}) ..."
curl -sLO "$fileurl"
echo "Copy oci image: oci-archive:${filename} docker-daemon:envoyproxy/envoy:${variant}"
skopeo copy -q "oci-archive:${filename}" "docker-daemon:envoyproxy/envoy:${variant}"
rm "$filename"
done
docker images | grep envoy
- run: |
set -e
export DEBIAN_FRONTEND=noninteractive
sudo apt-get -qq update -y
sudo apt-get -qq install -y --no-install-recommends expect
- run: |
./ci/do_ci.sh verify_examples
env:
NO_BUILD_SETUP: 1
# ENVOY_EXAMPLES_DEBUG: 1
7 changes: 6 additions & 1 deletion .github/workflows/mobile_release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@ on:

jobs:
android_release_artifacts:
if: github.repository == 'envoyproxy/envoy'
if: |
${{
github.repository == 'envoyproxy/envoy'
&& (github.event.schedule
|| !contains(github.actor, '[bot]'))
}}
name: android_release_artifacts
runs-on: ubuntu-20.04
timeout-minutes: 120
Expand Down
7 changes: 6 additions & 1 deletion .github/workflows/pr_notifier.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@ jobs:
pull-requests: read # for pr_notifier.py
name: PR Notifier
runs-on: ubuntu-20.04
if: github.repository == 'envoyproxy/envoy'
if: |
${{
github.repository == 'envoyproxy/envoy'
&& (github.event.schedule
|| !contains(github.actor, '[bot]'))
}}
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.8
Expand Down
7 changes: 6 additions & 1 deletion .github/workflows/stale.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,12 @@ jobs:
pull-requests: write # for actions/stale to close stale PRs
name: Prune Stale
runs-on: ubuntu-20.04
if: github.repository == 'envoyproxy/envoy'
if: |
${{
github.repository == 'envoyproxy/envoy'
&& (github.event.schedule
|| !contains(github.actor, '[bot]'))
}}
steps:
- name: Prune Stale
Expand Down
61 changes: 61 additions & 0 deletions .github/workflows/workflow-complete.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
name: Workflow complete
# This workflow is only required for externally triggered jobs that have manually
# set the check status for a commit/PR

on:
# Do not run untrusted code here
workflow_run:
workflows:
- Verify/examples
types:
- completed

permissions:
contents: read

jobs:
complete:
runs-on: ubuntu-20.04
permissions:
statuses: write
steps:
- name: 'Download artifact'
uses: actions/github-script@v6
with:
script: |
let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.payload.workflow_run.id,
});
let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
return artifact.name == "state_sha"
})[0];
let download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: 'zip',
});
let fs = require('fs');
fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/state_sha.zip`, Buffer.from(download.data));
- run: |
set -e
unzip state_sha.zip
STATE_SHA="$(cat state_sha)"
echo "state_sha=$STATE_SHA" >> "$GITHUB_OUTPUT"
STATE="${{ github.event.workflow_run.conclusion }}"
if [[ ${STATE} != "success" ]]; then
STATE=failure
fi
echo "state=${STATE}" >> "$GITHUB_OUTPUT"
id: job
- name: Complete status check
uses: envoyproxy/toolshed/gh-actions/status@a6e1c951217efae1ac6b2bf32c5a9729976442b8
with:
authToken: ${{ secrets.GITHUB_TOKEN }}
context: ${{ github.event.workflow.name }}
state: ${{ steps.job.outputs.state }}
sha: ${{ steps.job.outputs.state_sha }}
target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}
38 changes: 38 additions & 0 deletions .github/workflows/workflow-start.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: Workflow start
# This workflow is only required for externally triggered jobs that need to manually
# set the check status for a commit/PR

on:
workflow_call:
inputs:
workflowName:
required: true
type: string

permissions:
contents: read

jobs:
start:
runs-on: ubuntu-20.04
permissions:
statuses: write
steps:
- name: Start status check
uses: envoyproxy/toolshed/gh-actions/status@a6e1c951217efae1ac6b2bf32c5a9729976442b8
with:
authToken: ${{ secrets.GITHUB_TOKEN }}
context: ${{ inputs.workflowName }}
state: 'pending'
sha: ${{ inputs.sha }}
target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- name: Save the SHA
env:
STATE_SHA: ${{ inputs.sha }}
run: |
mkdir -p ./sha
echo $STATE_SHA > ./sha/state_sha
- uses: actions/upload-artifact@v3
with:
name: state_sha
path: sha/

0 comments on commit c7bd38e

Please sign in to comment.