-
Notifications
You must be signed in to change notification settings - Fork 2.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
workflows: add external workload conformance test
This was ported and adapted from `cilium-cli`. Signed-off-by: Nicolas Busseneau <nicolas@isovalent.com>
- Loading branch information
1 parent
94f94e5
commit 5726e9f
Showing
3 changed files
with
720 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
#!/bin/bash | ||
|
||
apt-get update | ||
apt-get install -y --no-install-recommends \ | ||
apt-transport-https \ | ||
ca-certificates \ | ||
curl \ | ||
gnupg \ | ||
lsb-release | ||
|
||
HOSTNAME=$(curl --silent http://metadata.google.internal/computeMetadata/v1/instance/attributes/hostname -H "Metadata-Flavor: Google") | ||
echo "Setting hostname $HOSTNAME" | ||
hostname $HOSTNAME | ||
|
||
echo "Installing docker" | ||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg | ||
echo \ | ||
"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ | ||
$(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null | ||
apt-get update | ||
apt-get install -y --no-install-recommends \ | ||
docker-ce \ | ||
docker-ce-cli \ | ||
containerd.io | ||
|
||
echo "Adding user $USER to group docker" | ||
usermod -aG docker $USER |
345 changes: 345 additions & 0 deletions
345
.github/workflows/conformance-externalworkloads-v1.10.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,345 @@ | ||
name: External workloads (ci-external-workloads-v1.10) | ||
|
||
# Any change in triggers needs to be reflected in the concurrency group. | ||
on: | ||
issue_comment: | ||
types: | ||
- created | ||
### FOR TESTING PURPOSES | ||
# This workflow runs in the context of `master`, and ignores changes to | ||
# workflow files in PRs. For testing changes to this workflow from a PR: | ||
# - Make sure the PR uses a branch from the base repository (requires write | ||
# privileges). It will not work with a branch from a fork (missing secrets). | ||
# - Uncomment the `pull_request` event below, commit separately with a `DO | ||
# NOT MERGE` message, and push to the PR. As long as the commit is present, | ||
# any push to the PR will trigger this workflow. | ||
# - Don't forget to remove the `DO NOT MERGE` commit once satisfied. The run | ||
# will disappear from the PR checks: please provide a direct link to the | ||
# successful workflow run (can be found from Actions tab) in a comment. | ||
# | ||
# pull_request: {} | ||
### | ||
|
||
# By specifying the access of one of the scopes, all of those that are not | ||
# specified are set to 'none'. | ||
permissions: | ||
# To be able to access the repository with actions/checkout | ||
contents: read | ||
# To allow retrieving information from the PR API | ||
pull-requests: read | ||
# So that Sibz/github-status-action can write into the status API | ||
statuses: write | ||
|
||
concurrency: | ||
# Structure: | ||
# - Workflow name | ||
# - Event type | ||
# - A unique identifier depending on event type: | ||
# - schedule: SHA | ||
# - issue_comment: PR number | ||
# - pull_request: PR number | ||
# | ||
# This structure ensures a unique concurrency group name is generated for each | ||
# type of testing: | ||
# - schedule: {name} schedule {SHA} | ||
# - issue_comment: {name} issue_comment {PR number} | ||
# - pull_request: {name} pull_request {PR number} | ||
# | ||
# Note: for `issue_comment` triggers, we additionally need to filter out based | ||
# on comment content, otherwise any comment will interrupt workflow runs. | ||
group: | | ||
${{ github.workflow }} | ||
${{ github.event_name }} | ||
${{ | ||
(github.event_name == 'schedule' && github.sha) || | ||
(github.event_name == 'issue_comment' && | ||
(startsWith(github.event.comment.body, 'ci-external-workloads-v1.10') || | ||
startsWith(github.event.comment.body, 'test-backport-1.10')) && | ||
github.event.issue.number) || | ||
(github.event_name == 'pull_request' && github.event.pull_request.number) | ||
}} | ||
cancel-in-progress: true | ||
|
||
env: | ||
clusterName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-vm | ||
vmName: ${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-vm | ||
zone: us-west2-a | ||
vmStartupScript: .github/gcp-vm-startup.sh | ||
check_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | ||
|
||
jobs: | ||
check_changes: | ||
name: Deduce required tests from code changes | ||
if: | | ||
(github.event_name == 'issue_comment' && ( | ||
(startsWith(github.event.comment.body, 'ci-external-workloads-v1.10') || | ||
startsWith(github.event.comment.body, 'test-backport-1.10')) && | ||
)) || | ||
(github.event_name == 'schedule' && github.repository == 'cilium/cilium') || | ||
github.event_name == 'pull_request' | ||
runs-on: ubuntu-latest | ||
outputs: | ||
tested: ${{ steps.tested-tree.outputs.src }} | ||
steps: | ||
- name: Retrieve pull request's base and head | ||
if: ${{ github.event.issue.pull_request }} | ||
id: pr | ||
run: | | ||
curl ${{ github.event.issue.pull_request.url }} > pr.json | ||
echo "::set-output name=base::$(jq -r '.base.sha' pr.json)" | ||
echo "::set-output name=head::$(jq -r '.head.sha' pr.json)" | ||
# Because we run on issue comments, we need to checkout the code for | ||
# paths-filter to work. | ||
- name: Checkout code | ||
if: ${{ github.event.issue.pull_request }} | ||
uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f | ||
with: | ||
persist-credentials: false | ||
- name: Check code changes | ||
if: ${{ github.event.issue.pull_request }} | ||
uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 | ||
id: tested-tree | ||
with: | ||
base: ${{ steps.pr.outputs.base }} | ||
ref: ${{ steps.pr.outputs.head }} | ||
filters: | | ||
src: | ||
- '!(test|Documentation)/**' | ||
# When the test-me-please trigger is used, this job is skipped if the only | ||
# modified files were under test/ or Documentation/. | ||
installation-and-connectivity: | ||
needs: check_changes | ||
if: | | ||
(github.event_name == 'issue_comment' && ( | ||
startsWith(github.event.comment.body, 'ci-external-workloads') || | ||
(startsWith(github.event.comment.body, 'test-me-please') && (needs.check_changes.outputs.tested == 'true')) | ||
)) || | ||
(github.event_name == 'schedule' && github.repository == 'cilium/cilium') || | ||
github.event_name == 'pull_request' | ||
runs-on: ubuntu-latest | ||
timeout-minutes: 20 | ||
steps: | ||
- name: Set up job variables | ||
id: vars | ||
run: | | ||
if [ ${{ github.event.issue.pull_request || github.event.pull_request }} ]; then | ||
PR_API_JSON=$(curl \ | ||
-H "Accept: application/vnd.github.v3+json" \ | ||
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | ||
${{ github.event.issue.pull_request.url || github.event.pull_request.url }}) | ||
SHA=$(echo "$PR_API_JSON" | jq -r ".head.sha") | ||
OWNER=$(echo "$PR_API_JSON" | jq -r ".number") | ||
else | ||
SHA=${{ github.sha }} | ||
OWNER=${{ github.sha }} | ||
fi | ||
CILIUM_INSTALL_DEFAULTS="--cluster-name=${{ env.clusterName }} \ | ||
--agent-image=quay.io/${{ github.repository_owner }}/cilium-ci \ | ||
--operator-image=quay.io/${{ github.repository_owner }}/operator-generic-ci \ | ||
--version=${SHA} \ | ||
--wait=false \ | ||
--config monitor-aggregation=none \ | ||
--config tunnel=vxlan \ | ||
--kube-proxy-replacement=strict" | ||
HUBBLE_ENABLE_DEFAULTS="--relay-image=quay.io/${{ github.repository_owner }}/hubble-relay-ci \ | ||
--relay-version=${SHA}" | ||
CLUSTERMESH_ENABLE_DEFAULTS="--apiserver-image=quay.io/${{ github.repository_owner }}/clustermesh-apiserver-ci \ | ||
--apiserver-version=${SHA}" | ||
echo ::set-output name=cilium_install_defaults::${CILIUM_INSTALL_DEFAULTS} | ||
echo ::set-output name=hubble_enable_defaults::${HUBBLE_ENABLE_DEFAULTS} | ||
echo ::set-output name=clustermesh_enable_defaults::${CLUSTERMESH_ENABLE_DEFAULTS} | ||
echo ::set-output name=sha::${SHA} | ||
echo ::set-output name=owner::${OWNER} | ||
- name: Set commit status to pending | ||
uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d | ||
with: | ||
authToken: ${{ secrets.GITHUB_TOKEN }} | ||
sha: ${{ steps.vars.outputs.sha }} | ||
context: ${{ github.workflow }} | ||
description: Connectivity test in progress... | ||
state: pending | ||
target_url: ${{ env.check_url }} | ||
|
||
- name: Checkout code | ||
uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f | ||
with: | ||
persist-credentials: false | ||
|
||
- name: Install Cilium CLI | ||
run: | | ||
export CILUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt) | ||
curl -sSL --remote-name-all https://github.com/cilium/cilium-cli/releases/download/$CILUM_CLI_VERSION/cilium-linux-amd64.tar.gz{,.sha256sum} | ||
sha256sum --check cilium-linux-amd64.tar.gz.sha256sum | ||
sudo tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin | ||
rm cilium-linux-amd64.tar.gz{,.sha256sum} | ||
cilium version | ||
- name: Set up gcloud CLI | ||
uses: google-github-actions/setup-gcloud@daadedc81d5f9d3c06d2c92f49202a3cc2b919ba | ||
with: | ||
project_id: ${{ secrets.GCP_PROJECT_ID }} | ||
service_account_key: ${{ secrets.GCP_PR_SA_KEY }} | ||
export_default_credentials: true | ||
|
||
- name: Display gcloud CLI info | ||
run: | | ||
gcloud info | ||
- name: Create GCP VM | ||
run: | | ||
gcloud compute instances create ${{ env.vmName }} \ | ||
--labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \ | ||
--zone ${{ env.zone }} \ | ||
--machine-type e2-custom-2-4096 \ | ||
--boot-disk-type pd-standard \ | ||
--boot-disk-size 10GB \ | ||
--preemptible \ | ||
--image-project ubuntu-os-cloud \ | ||
--image-family ubuntu-2004-lts \ | ||
--metadata hostname=${{ env.vmName }} \ | ||
--metadata-from-file startup-script=${{ env.vmStartupScript}} | ||
- name: Create GKE cluster | ||
run: | | ||
gcloud container clusters create ${{ env.clusterName }} \ | ||
--labels "usage=${{ github.repository_owner }}-${{ github.event.repository.name }},owner=${{ steps.vars.outputs.owner }}" \ | ||
--zone ${{ env.zone }} \ | ||
--node-taints node.cilium.io/agent-not-ready=true:NoSchedule \ | ||
--image-type COS_CONTAINERD \ | ||
--num-nodes 2 \ | ||
--machine-type e2-custom-2-4096 \ | ||
--disk-type pd-standard \ | ||
--disk-size 10GB \ | ||
--preemptible | ||
- name: Get cluster credentials | ||
run: | | ||
gcloud container clusters get-credentials ${{ env.clusterName }} --zone ${{ env.zone }} | ||
- name: Wait for images to be available | ||
timeout-minutes: 10 | ||
shell: bash | ||
run: | | ||
for image in cilium-ci operator-generic-ci hubble-relay-ci clustermesh-apiserver-ci ; do | ||
until curl --silent -f -lSL "https://quay.io/api/v1/repository/${{ github.repository_owner }}/$image/tag/${{ steps.vars.outputs.sha }}/images" &> /dev/null; do sleep 45s; done | ||
done | ||
- name: Install Cilium in cluster | ||
run: | | ||
cilium install ${{ steps.vars.outputs.cilium_install_defaults }} | ||
- name: Wait for Cilium status to be ready | ||
run: | | ||
cilium status --wait | ||
- name: Enable cluster mesh | ||
run: | | ||
cilium clustermesh enable ${{ steps.vars.outputs.clustermesh_enable_defaults }} | ||
- name: Wait for cluster mesh status to be ready | ||
run: | | ||
cilium clustermesh status --wait | ||
- name: Add VM to cluster mesh | ||
run: | | ||
cilium clustermesh vm create ${{ env.vmName }} -n default --ipv4-alloc-cidr 10.192.1.0/30 | ||
cilium clustermesh vm status | ||
- name: Install Cilium on VM | ||
run: | | ||
cilium clustermesh vm install install-external-workload.sh --config debug | ||
gcloud compute scp install-external-workload.sh ${{ env.vmName }}:~/ --zone ${{ env.zone }} | ||
gcloud compute ssh ${{ env.vmName }} --zone ${{ env.zone }} \ | ||
--command "~/install-external-workload.sh" | ||
sleep 5s | ||
gcloud compute ssh ${{ env.vmName }} --zone ${{ env.zone }} \ | ||
--command "cilium status" | ||
- name: Verify cluster DNS on VM | ||
run: | | ||
gcloud compute ssh ${{ env.vmName }} --zone ${{ env.zone }} \ | ||
--command "nslookup -norecurse clustermesh-apiserver.kube-system.svc.cluster.local" | ||
- name: Ping clustermesh-apiserver from VM | ||
run: | | ||
gcloud compute ssh ${{ env.vmName }} --zone ${{ env.zone }} \ | ||
--command "ping -c 3 \$(cilium service list get -o jsonpath='{[?(@.spec.flags.name==\"clustermesh-apiserver\")].spec.backend-addresses[0].ip}')" | ||
- name: Run connectivity test | ||
run: | | ||
cilium connectivity test --flow-validation=disabled | ||
- name: Post-test information gathering | ||
if: ${{ !success() }} | ||
run: | | ||
cilium status | ||
cilium clustermesh status | ||
cilium clustermesh vm status | ||
gcloud compute ssh ${{ env.vmName }} --zone ${{ env.zone }} --command "cilium status" | ||
gcloud compute ssh ${{ env.vmName }} --zone ${{ env.zone }} --command "sudo docker logs cilium --timestamps" | ||
kubectl get pods --all-namespaces -o wide | ||
kubectl get cew --all-namespaces -o wide | ||
kubectl get cep --all-namespaces -o wide | ||
curl -sLO https://github.com/cilium/cilium-sysdump/releases/latest/download/cilium-sysdump.zip | ||
python cilium-sysdump.zip --output cilium-sysdump-out | ||
shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently | ||
|
||
- name: Clean up GKE cluster and VM | ||
if: ${{ always() }} | ||
run: | | ||
gcloud container clusters delete ${{ env.clusterName }} --zone ${{ env.zone }} --quiet --async | ||
gcloud compute instances delete ${{ env.vmName }} --zone ${{ env.zone }} --quiet | ||
shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently | ||
|
||
- name: Upload artifacts | ||
if: ${{ !success() }} | ||
uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074 | ||
with: | ||
name: cilium-sysdump-out.zip | ||
path: cilium-sysdump-out.zip | ||
retention-days: 5 | ||
|
||
- name: Set commit status to success | ||
if: ${{ success() }} | ||
uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d | ||
with: | ||
authToken: ${{ secrets.GITHUB_TOKEN }} | ||
sha: ${{ steps.vars.outputs.sha }} | ||
context: ${{ github.workflow }} | ||
description: Connectivity test successful | ||
state: success | ||
target_url: ${{ env.check_url }} | ||
|
||
- name: Set commit status to failure | ||
if: ${{ failure() }} | ||
uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d | ||
with: | ||
authToken: ${{ secrets.GITHUB_TOKEN }} | ||
sha: ${{ steps.vars.outputs.sha }} | ||
context: ${{ github.workflow }} | ||
description: Connectivity test failed | ||
state: failure | ||
target_url: ${{ env.check_url }} | ||
|
||
- name: Set commit status to cancelled | ||
if: ${{ cancelled() }} | ||
uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d | ||
with: | ||
authToken: ${{ secrets.GITHUB_TOKEN }} | ||
sha: ${{ steps.vars.outputs.sha }} | ||
context: ${{ github.workflow }} | ||
description: Connectivity test cancelled | ||
state: error | ||
target_url: ${{ env.check_url }} | ||
|
||
- name: Send slack notification | ||
if: ${{ !success() && (github.event_name == 'schedule' || github.event_name == 'push') }} | ||
uses: 8398a7/action-slack@dcc8c8e9dd8802e21a712dc0c003db97b42efe43 | ||
with: | ||
status: ${{ job.status }} | ||
fields: repo,message,commit,author,action,eventName,ref,workflow,job,took # selectable (default: repo,message) | ||
env: | ||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} |
Oops, something went wrong.