From 9562fa03f796394a7ce929ae13b90bdeaef85fa0 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 25 Mar 2024 15:23:55 +0000 Subject: [PATCH 1/4] CI(trigger-e2e-tests): set e2e-platforms --- .github/workflows/trigger-e2e-tests.yml | 34 ++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/trigger-e2e-tests.yml b/.github/workflows/trigger-e2e-tests.yml index ae34cbffe0d0..9895f5c7a5a9 100644 --- a/.github/workflows/trigger-e2e-tests.yml +++ b/.github/workflows/trigger-e2e-tests.yml @@ -79,7 +79,38 @@ jobs: fi done + - uses: actions/checkout@v4 + if: github.event_name == 'pull_request' + with: + submodules: true + fetch-depth: 0 + + - name: Set e2e-platforms + id: e2e-platforms + env: + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.sha }} + run: | + # Default set of plafforms to run e2e tests on + platforms='["docker", "k8s"]' + + # If the PR changes vendor/ or pgxn/ directories, or Dockerfile.compute-node, add k8s-neonvm to the list of platforms. + # If the workflow run is not a pull request, add k8s-neonvm to the list. + if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + for f in $(git diff --name-only ${BASE_SHA} ${COMMIT_SHA}); do + if [[ $f == "vendor/"* ]] || [[ $f == "pgxn/*" ]] || [[ $f == "Dockerfile.compute-node" ]]; then + platforms=$(echo "${platforms}" | jq --compact-output '. += ["k8s-neonvm"] | unique') + fi + done + else + platforms=$(echo "${platforms}" | jq --compact-output '. += ["k8s-neonvm"] | unique') + fi + + echo "e2e-platforms=${platforms}" | tee -a $GITHUB_OUTPUT + - name: Set PR's status to pending and request a remote CI test + env: + E2E_PLATFORMS: ${{ steps.e2e-platforms.outputs.e2e-platforms }} run: | # For pull requests, GH Actions set "github.sha" variable to point at a fake merge commit # but we need to use a real sha of a latest commit in the PR's branch for the e2e job, @@ -114,6 +145,7 @@ jobs: \"remote_repo\": \"${{ github.repository }}\", \"storage_image_tag\": \"${TAG}\", \"compute_image_tag\": \"${TAG}\", - \"concurrency_group\": \"${{ env.E2E_CONCURRENCY_GROUP }}\" + \"concurrency_group\": \"${{ env.E2E_CONCURRENCY_GROUP }}\", + \"e2e-platforms": \""'"${E2E_PLATFORMS}"'"\" } }" From daff7891ae5ea5b82b156fc2fa6b84f080b1ec3a Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 25 Mar 2024 15:24:36 +0000 Subject: [PATCH 2/4] CI(trigger-e2e-tests): use ubuntu-latest and gh cli --- .github/workflows/trigger-e2e-tests.yml | 63 +++++++++---------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/.github/workflows/trigger-e2e-tests.yml b/.github/workflows/trigger-e2e-tests.yml index 9895f5c7a5a9..800664a71b07 100644 --- a/.github/workflows/trigger-e2e-tests.yml +++ b/.github/workflows/trigger-e2e-tests.yml @@ -62,14 +62,14 @@ jobs: trigger-e2e-tests: needs: [ tag ] - runs-on: [ self-hosted, gen3, small ] + runs-on: ubuntu-latest env: TAG: ${{ needs.tag.outputs.build-tag }} - container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned - options: --init steps: - name: check if ecr image are present + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} run: | for REPO in neon compute-tools compute-node-v14 vm-compute-node-v14 compute-node-v15 vm-compute-node-v15 compute-node-v16 vm-compute-node-v16; do OUTPUT=$(aws ecr describe-images --repository-name ${REPO} --region eu-central-1 --query "imageDetails[?imageTags[?contains(@, '${TAG}')]]" --output text) @@ -91,7 +91,7 @@ jobs: COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} BASE_SHA: ${{ github.event.pull_request.base.sha || github.sha }} run: | - # Default set of plafforms to run e2e tests on + # Default set of platforms to run e2e tests on platforms='["docker", "k8s"]' # If the PR changes vendor/ or pgxn/ directories, or Dockerfile.compute-node, add k8s-neonvm to the list of platforms. @@ -111,41 +111,24 @@ jobs: - name: Set PR's status to pending and request a remote CI test env: E2E_PLATFORMS: ${{ steps.e2e-platforms.outputs.e2e-platforms }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} run: | - # For pull requests, GH Actions set "github.sha" variable to point at a fake merge commit - # but we need to use a real sha of a latest commit in the PR's branch for the e2e job, - # to place a job run status update later. - COMMIT_SHA=${{ github.event.pull_request.head.sha }} - # For non-PR kinds of runs, the above will produce an empty variable, pick the original sha value for those - COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}} - REMOTE_REPO="${{ github.repository_owner }}/cloud" - curl -f -X POST \ - https://api.github.com/repos/${{ github.repository }}/statuses/$COMMIT_SHA \ - -H "Accept: application/vnd.github.v3+json" \ - --user "${{ secrets.CI_ACCESS_TOKEN }}" \ - --data \ - "{ - \"state\": \"pending\", - \"context\": \"neon-cloud-e2e\", - \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\" - }" - - curl -f -X POST \ - https://api.github.com/repos/$REMOTE_REPO/actions/workflows/testing.yml/dispatches \ - -H "Accept: application/vnd.github.v3+json" \ - --user "${{ secrets.CI_ACCESS_TOKEN }}" \ - --data \ - "{ - \"ref\": \"main\", - \"inputs\": { - \"ci_job_name\": \"neon-cloud-e2e\", - \"commit_hash\": \"$COMMIT_SHA\", - \"remote_repo\": \"${{ github.repository }}\", - \"storage_image_tag\": \"${TAG}\", - \"compute_image_tag\": \"${TAG}\", - \"concurrency_group\": \"${{ env.E2E_CONCURRENCY_GROUP }}\", - \"e2e-platforms": \""'"${E2E_PLATFORMS}"'"\" - } - }" + gh api /repos/${{ github.repository }}/statuses/$COMMIT_SHA \ + --method POST \ + --raw-field "state=pending" \ + --raw-field "description=[$REMOTE_REPO] Remote CI job is about to start" \ + --raw-field "context=neon-cloud-e2e" + + gh workflow --repo $REMOTE_REPO \ + run testing.yml \ + --ref "main" \ + --raw-field "ci_job_name=neon-cloud-e2e" \ + --raw-field "commit_hash=$COMMIT_SHA" \ + --raw-field "remote_repo=${{ github.repository }}" \ + --raw-field "storage_image_tag=${TAG}" \ + --raw-field "compute_image_tag=${TAG}" \ + --raw-field "concurrency_group=${{ env.E2E_CONCURRENCY_GROUP }}" \ + --raw-field "e2e-platforms=${E2E_PLATFORMS}" From 6c2b4fc2266bba2ce9fc964e8d08836b4f87d90f Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Tue, 26 Mar 2024 15:24:30 +0000 Subject: [PATCH 3/4] CI(trigger-e2e-tests): get list of changed files from API --- .github/workflows/trigger-e2e-tests.yml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/trigger-e2e-tests.yml b/.github/workflows/trigger-e2e-tests.yml index 800664a71b07..4beaebcf3352 100644 --- a/.github/workflows/trigger-e2e-tests.yml +++ b/.github/workflows/trigger-e2e-tests.yml @@ -79,17 +79,11 @@ jobs: fi done - - uses: actions/checkout@v4 - if: github.event_name == 'pull_request' - with: - submodules: true - fetch-depth: 0 - - name: Set e2e-platforms id: e2e-platforms env: - COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha || github.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # Default set of platforms to run e2e tests on platforms='["docker", "k8s"]' @@ -97,7 +91,7 @@ jobs: # If the PR changes vendor/ or pgxn/ directories, or Dockerfile.compute-node, add k8s-neonvm to the list of platforms. # If the workflow run is not a pull request, add k8s-neonvm to the list. if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then - for f in $(git diff --name-only ${BASE_SHA} ${COMMIT_SHA}); do + for f in $(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename'); do if [[ $f == "vendor/"* ]] || [[ $f == "pgxn/*" ]] || [[ $f == "Dockerfile.compute-node" ]]; then platforms=$(echo "${platforms}" | jq --compact-output '. += ["k8s-neonvm"] | unique') fi @@ -114,21 +108,21 @@ jobs: COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} run: | - REMOTE_REPO="${{ github.repository_owner }}/cloud" + REMOTE_REPO="${GITHUB_REPOSITORY_OWNER}/cloud" - gh api /repos/${{ github.repository }}/statuses/$COMMIT_SHA \ + gh api "/repos/${GITHUB_REPOSITORY}/statuses/${COMMIT_SHA}" \ --method POST \ --raw-field "state=pending" \ --raw-field "description=[$REMOTE_REPO] Remote CI job is about to start" \ --raw-field "context=neon-cloud-e2e" - gh workflow --repo $REMOTE_REPO \ + gh workflow --repo ${REMOTE_REPO} \ run testing.yml \ --ref "main" \ --raw-field "ci_job_name=neon-cloud-e2e" \ --raw-field "commit_hash=$COMMIT_SHA" \ - --raw-field "remote_repo=${{ github.repository }}" \ + --raw-field "remote_repo=${GITHUB_REPOSITORY}" \ --raw-field "storage_image_tag=${TAG}" \ --raw-field "compute_image_tag=${TAG}" \ - --raw-field "concurrency_group=${{ env.E2E_CONCURRENCY_GROUP }}" \ + --raw-field "concurrency_group=${E2E_CONCURRENCY_GROUP}" \ --raw-field "e2e-platforms=${E2E_PLATFORMS}" From 3e271314402cb69d409d2e4deb63badf4d8b0ad1 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Tue, 26 Mar 2024 17:54:41 +0000 Subject: [PATCH 4/4] CI(trigger-e2e-tests): trigger tests on neonvm for changes in libs/vm_monitor --- .github/workflows/trigger-e2e-tests.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/trigger-e2e-tests.yml b/.github/workflows/trigger-e2e-tests.yml index 4beaebcf3352..7111ee37faa5 100644 --- a/.github/workflows/trigger-e2e-tests.yml +++ b/.github/workflows/trigger-e2e-tests.yml @@ -88,13 +88,18 @@ jobs: # Default set of platforms to run e2e tests on platforms='["docker", "k8s"]' - # If the PR changes vendor/ or pgxn/ directories, or Dockerfile.compute-node, add k8s-neonvm to the list of platforms. + # If the PR changes vendor/, pgxn/ or libs/vm_monitor/ directories, or Dockerfile.compute-node, add k8s-neonvm to the list of platforms. # If the workflow run is not a pull request, add k8s-neonvm to the list. if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then for f in $(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename'); do - if [[ $f == "vendor/"* ]] || [[ $f == "pgxn/*" ]] || [[ $f == "Dockerfile.compute-node" ]]; then - platforms=$(echo "${platforms}" | jq --compact-output '. += ["k8s-neonvm"] | unique') - fi + case "$f" in + vendor/*|pgxn/*|libs/vm_monitor/*|Dockerfile.compute-node) + platforms=$(echo "${platforms}" | jq --compact-output '. += ["k8s-neonvm"] | unique') + ;; + *) + # no-op + ;; + esac done else platforms=$(echo "${platforms}" | jq --compact-output '. += ["k8s-neonvm"] | unique')