From 00d90e8f3d24a85fa5d9c5c3f66f670bf6459094 Mon Sep 17 00:00:00 2001 From: Aidan Oldershaw Date: Thu, 8 Jul 2021 23:29:20 -0400 Subject: [PATCH] set one pipeline per PR since we no longer have to deal with version: every + passed constraints, we can now build the PR images once and fetch the result in each of the downstream jobs. this also makes it easier to test PRs, since you can just point to the image concourse/dev:pr-1234 Signed-off-by: Aidan Oldershaw --- pipelines/{prs.yml => pr.yml} | 351 +++++++++++----------------- pipelines/reconfigure.yml | 53 ++++- pipelines/release.yml | 41 +++- tasks/scripts/check-migration-order | 12 +- tasks/scripts/validate-pr | 4 +- tasks/validate-pr.yml | 3 +- 6 files changed, 220 insertions(+), 244 deletions(-) rename pipelines/{prs.yml => pr.yml} (53%) diff --git a/pipelines/prs.yml b/pipelines/pr.yml similarity index 53% rename from pipelines/prs.yml rename to pipelines/pr.yml index ed06db32..4afaadb6 100644 --- a/pipelines/prs.yml +++ b/pipelines/pr.yml @@ -1,10 +1,30 @@ # the following vars must be specified: # +# ((number)) the PR number # ((branch)) only PRs against this branch will be run # ((dev_image_tag)) the tag to use for the dev image # ((concourse_image_tag)) the tag to use for the concourse image for upgrade/downgrade test in integration --- + +pending_status_notification: &pending_status_notification + put: concourse-pr + inputs: [concourse-pr] + params: {path: concourse-pr, status: pending, context: $BUILD_JOB_NAME} + tags: [pr] + +status_notifications: &status_notifications + on_success: + put: concourse-pr + inputs: [concourse-pr] + params: {path: concourse-pr, status: success, context: $BUILD_JOB_NAME} + tags: [pr] + on_failure: + put: concourse-pr + inputs: [concourse-pr] + params: {path: concourse-pr, status: failure, context: $BUILD_JOB_NAME} + tags: [pr] + resource_types: - name: bosh-deployment type: registry-image @@ -13,25 +33,29 @@ resource_types: - name: pull-request type: registry-image source: - repository: teliaoss/github-pr-resource - tag: v0.21.0 + repository: aoldershaw/github-pr-resource + +# Remove this when a new version of Concourse is shipped with +# https://github.com/concourse/git-resource/pull/363 +- name: git + type: registry-image + source: {repository: aoldershaw/git-resource} resources: - name: concourse-base type: git icon: github source: - uri: https://github.com/concourse/concourse + uri: https://github.com/concourse/concourse.git branch: ((branch)) - name: concourse-pr type: pull-request icon: source-pull - check_every: 2m source: repository: concourse/concourse access_token: ((pull_requests_access_token)) - base_branch: ((branch)) + number: ((number)) - name: validator type: github-release @@ -65,6 +89,15 @@ resources: username: ((docker.username)) password: ((docker.password)) +- name: pr-image + type: registry-image + icon: docker + source: + repository: concourse/dev + tag: pr-((number)) + username: ((docker.username)) + password: ((docker.password)) + - name: unit-image type: registry-image icon: docker @@ -77,7 +110,7 @@ resources: - name: vault-image type: registry-image - icon: registry-image + icon: docker source: {repository: vault} - name: oci-build-task @@ -86,26 +119,52 @@ resources: source: {repository: vito/oci-build-task} jobs: -- name: unit +- name: pr-image public: true - max_in_flight: 2 - on_failure: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: failure, context: unit} + serial: true + plan: + - in_parallel: + - get: concourse-pr + trigger: true + tags: [pr] + - get: oci-build-task + tags: [pr] + - get: dev-image + params: {format: oci} + tags: [pr] + - get: ci + tags: [pr] + - task: yarn-build + attempts: 3 + file: ci/tasks/yarn-build.yml + input_mapping: {concourse: concourse-pr} tags: [pr] - get_params: {skip_download: true} - on_success: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: success, context: unit} + - task: build + image: oci-build-task + privileged: true + config: + platform: linux + params: + CONTEXT: built-concourse + IMAGE_ARG_base_image: dev-image/image.tar + TARGET: with-fly + inputs: [{name: built-concourse}, {name: dev-image}] + outputs: [{name: image}] + caches: [{path: cache}] + run: {path: build} tags: [pr] + - put: pr-image + params: {image: image/image.tar} get_params: {skip_download: true} + +- name: unit + public: true + serial: true + <<: *status_notifications plan: - in_parallel: - get: concourse-pr trigger: true - version: every tags: [pr] - get: concourse-base tags: [pr] @@ -113,12 +172,7 @@ jobs: tags: [pr] - get: ci tags: [pr] - - put: concourse-status-update - resource: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: pending, context: unit} - tags: [pr] - get_params: {list_changed_files: true, skip_download: true} + - *pending_status_notification - task: check-migration-order timeout: 5m file: ci/tasks/check-migration-order.yml @@ -146,147 +200,42 @@ jobs: input_mapping: {concourse: built-concourse} tags: [pr] -- name: testflight - public: true - max_in_flight: 2 - on_failure: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: failure, context: testflight} - tags: [pr] - get_params: {skip_download: true} - on_success: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: success, context: testflight} - tags: [pr] - get_params: {skip_download: true} - plan: - - in_parallel: - - get: concourse-pr - trigger: true - version: every - tags: [pr] - - get: oci-build-task - tags: [pr] - - get: dev-image - params: {format: oci} - tags: [pr] - - get: postgres-image - params: {format: oci} - tags: [pr] - - get: ci - tags: [pr] - - put: concourse-status-update - resource: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: pending, context: testflight} - tags: [pr] - get_params: {list_changed_files: true, skip_download: true} - - task: yarn-build - attempts: 3 - file: ci/tasks/yarn-build.yml - input_mapping: {concourse: concourse-pr} - tags: [pr] - - task: build - image: oci-build-task - privileged: true - config: - platform: linux - params: - CONTEXT: built-concourse - IMAGE_ARG_base_image: dev-image/image.tar - TARGET: with-fly - inputs: [{name: built-concourse}, {name: dev-image}] - outputs: [{name: image}] - caches: [{path: cache}] - run: {path: build} - tags: [pr] - - across: - - var: runtime - values: - - guardian - - containerd - max_in_flight: all - task: testflight - timeout: 1h - privileged: true - file: ci/tasks/docker-compose-testflight.yml - input_mapping: {concourse: built-concourse, dev-image: image} - params: - RUNTIME: ((.:runtime)) - DOWNLOAD_CLI: false - tags: [pr] - - name: validate-labels public: true - max_in_flight: 2 - on_failure: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: failure, context: validate-labels} - tags: [pr] - get_params: {skip_download: true} - on_success: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: success, context: validate-labels} - tags: [pr] - get_params: {skip_download: true} + serial: true + <<: *status_notifications plan: - in_parallel: - get: concourse-pr trigger: true - version: every tags: [pr] - get: ci tags: [pr] - get: validator tags: [pr] - - put: concourse-status-update - resource: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: pending, context: validate-labels} - tags: [pr] - get_params: {list_changed_files: true, skip_download: true} + - *pending_status_notification - task: validate-pr timeout: 5m file: ci/tasks/validate-pr.yml tags: [pr] params: GITHUB_TOKEN: ((concourse_github_dummy.access_token)) + PR_NUMBER: ((number)) - name: containerd-integration public: true - max_in_flight: 2 - on_failure: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: failure, context: containerd-integration} - tags: [pr] - get_params: {skip_download: true} - on_success: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: success, context: containerd-integration} - tags: [pr] - get_params: {skip_download: true} + serial: true + <<: *status_notifications plan: - in_parallel: - get: concourse-pr trigger: true - version: every tags: [pr] - get: dev-image tags: [pr] - get: ci tags: [pr] - - put: concourse-status-update - resource: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: pending, context: containerd-integration} - tags: [pr] - get_params: {list_changed_files: true, skip_download: true} + - *pending_status_notification - task: integration image: dev-image privileged: true @@ -295,29 +244,54 @@ jobs: input_mapping: {concourse: concourse-pr} tags: [pr] +- name: testflight + public: true + serial: true + <<: *status_notifications + plan: + - in_parallel: + - get: concourse-pr + passed: [pr-image] + trigger: true + tags: [pr] + - get: pr-image + passed: [pr-image] + params: {format: oci} + tags: [pr] + - get: postgres-image + params: {format: oci} + tags: [pr] + - get: ci + tags: [pr] + - *pending_status_notification + - across: + - var: runtime + values: + - guardian + - containerd + max_in_flight: all + task: testflight + timeout: 1h + privileged: true + file: ci/tasks/docker-compose-testflight.yml + input_mapping: {concourse: concourse-pr, dev-image: pr-image} + params: + RUNTIME: ((.:runtime)) + DOWNLOAD_CLI: false + tags: [pr] - name: watsjs public: true - max_in_flight: 2 - on_failure: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: failure, context: watsjs} - tags: [pr] - get_params: {skip_download: true} - on_success: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: success, context: watsjs} - tags: [pr] - get_params: {skip_download: true} + serial: true + <<: *status_notifications plan: - in_parallel: - get: concourse-pr + passed: [pr-image] trigger: true - version: every tags: [pr] - - get: dev-image + - get: pr-image + passed: [pr-image] params: {format: oci} tags: [pr] - get: postgres-image @@ -325,66 +299,35 @@ jobs: tags: [pr] - get: ci tags: [pr] - - get: oci-build-task - tags: [pr] - - put: concourse-status-update - resource: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: pending, context: watsjs} - tags: [pr] - get_params: {list_changed_files: true, skip_download: true} + - *pending_status_notification - task: yarn-build attempts: 3 file: ci/tasks/yarn-build.yml input_mapping: {concourse: concourse-pr} tags: [pr] - - task: build - image: oci-build-task - privileged: true - config: - platform: linux - params: - CONTEXT: built-concourse - IMAGE_ARG_base_image: dev-image/image.tar - TARGET: with-fly - inputs: [{name: built-concourse}, {name: dev-image}] - outputs: [{name: image}] - caches: [{path: cache}] - run: {path: build} - tags: [pr] - task: watsjs attempts: 3 timeout: 1h privileged: true file: ci/tasks/docker-compose-watsjs.yml - input_mapping: {concourse: built-concourse, dev-image: image} + input_mapping: {concourse: built-concourse, dev-image: pr-image} tags: [pr] - name: integration public: true - max_in_flight: 2 - on_failure: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: failure, context: integration} - tags: [pr] - get_params: {skip_download: true} - on_success: - put: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: success, context: integration} - tags: [pr] - get_params: {skip_download: true} + serial: true + <<: *status_notifications plan: - in_parallel: - get: concourse-pr + passed: [pr-image] trigger: true - version: every tags: [pr] - get: concourse-image params: {format: oci} tags: [pr] - - get: dev-image + - get: pr-image + passed: [pr-image] params: {format: oci} tags: [pr] - get: postgres-image @@ -395,35 +338,9 @@ jobs: tags: [pr] - get: ci tags: [pr] - - get: oci-build-task - tags: [pr] - - put: concourse-status-update - resource: concourse-pr - inputs: [concourse-pr] - params: {path: concourse-pr, status: pending, context: integration} - tags: [pr] - get_params: {list_changed_files: true, skip_download: true} - - task: yarn-build - attempts: 3 - file: ci/tasks/yarn-build.yml - input_mapping: {concourse: concourse-pr} - tags: [pr] - - task: build - image: oci-build-task - privileged: true - config: - platform: linux - params: - CONTEXT: built-concourse - IMAGE_ARG_base_image: dev-image/image.tar - TARGET: with-fly - inputs: [{name: built-concourse}, {name: dev-image}] - outputs: [{name: image}] - caches: [{path: cache}] - run: {path: build} - tags: [pr] + - *pending_status_notification - task: integration timeout: 1h privileged: true - input_mapping: {concourse: built-concourse, dev-image: image} + input_mapping: {concourse: concourse-pr, dev-image: pr-image} file: ci/tasks/integration.yml diff --git a/pipelines/reconfigure.yml b/pipelines/reconfigure.yml index b8836726..2b8abaf9 100644 --- a/pipelines/reconfigure.yml +++ b/pipelines/reconfigure.yml @@ -3,6 +3,10 @@ resource_types: type: registry-image source: {repository: mockersf/concourse-slack-notifier} +- name: pull-request + type: registry-image + source: {repository: aoldershaw/github-pr-resource} + resources: - name: pipelines-and-tasks type: git @@ -13,6 +17,14 @@ resources: - pipelines - tasks +- name: pr-pipeline + type: git + icon: github + source: + uri: https://github.com/concourse/ci + paths: + - pipelines/pr.yml + - name: teams type: git icon: github @@ -106,6 +118,14 @@ resources: repository: concourse access_token: ((concourse_github_dummy.access_token)) +- name: concourse-prs + type: pull-requests + checK_every: 2m + source: + repository: concourse/concourse + access_token: ((pull_requests_access_token)) + base_branch: master + jobs: - name: reconfigure-self plan: @@ -246,14 +266,6 @@ jobs: file: pipelines-and-tasks/pipelines/concourse.yml - set_pipeline: helm-chart file: pipelines-and-tasks/pipelines/helm-chart.yml - - set_pipeline: prs - team: contributor - file: pipelines-and-tasks/pipelines/prs.yml - instance_vars: - branch: master - vars: - dev_image_tag: latest - concourse_image_tag: latest - set_pipeline: helm-prs file: pipelines-and-tasks/pipelines/helm-prs.yml instance_vars: @@ -421,3 +433,28 @@ jobs: - set_pipeline: set-pipelines team: examples file: examples/pipelines/set-pipelines.yml + +- name: reconfigure-prs + plan: + - in_parallel: + - get: ignored + resource: pipelines-and-tasks + passed: [reconfigure-self] + - get: pr-pipeline + trigger: true + - get: concourse-prs + trigger: true + - load_var: concourse_prs + file: concourse-prs/prs.json + - across: + - var: pr_number + values: ((.:concourse_prs)) + set_pipeline: pr + team: contributor + file: pr-pipeline/pipelines/pr.yml + instance_vars: + number: ((.:pr_number)) + vars: + branch: master + dev_image_tag: latest + concourse_image_tag: latest diff --git a/pipelines/release.yml b/pipelines/release.yml index d24ed1b3..faca2339 100644 --- a/pipelines/release.yml +++ b/pipelines/release.yml @@ -53,10 +53,14 @@ resource_types: type: registry-image source: {repository: aoldershaw/github-release-resource} +- name: pull-request + type: registry-image + source: {repository: aoldershaw/github-pr-resource} + groups: - name: develop jobs: - - set-prs-pipeline + - reconfigure-prs - unit - integration - resource-types-images @@ -99,19 +103,28 @@ groups: - "*" jobs: -- name: set-prs-pipeline +- name: reconfigure-prs public: true serial: true plan: - - get: ci - resource: ci-prs-pipeline - trigger: true - - set_pipeline: prs - file: ci/pipelines/prs.yml + - in_parallel: + - get: ci + resource: ci-pr-pipeline + trigger: true + - get: concourse-prs + trigger: true + - load_var: concourse_prs + file: concourse-prs/prs.json + - across: + - var: pr_number + values: ((.:concourse_prs)) + set_pipeline: pr team: contributor + file: ci/pipelines/pr.yml instance_vars: - branch: release/((release_minor)).x + number: ((.:pr_number)) vars: + branch: release/((release_minor)).x dev_image_tag: release-((release_minor)) concourse_image_tag: ((release_minor)) @@ -1560,13 +1573,13 @@ resources: branch: master private_key: ((deploy_keys.ci-repo-push)) -- name: ci-prs-pipeline +- name: ci-pr-pipeline type: git icon: github source: uri: https://github.com/concourse/ci paths: - - pipelines/prs.yml + - pipelines/pr.yml - name: infrastructure type: git @@ -2111,3 +2124,11 @@ resources: owner: aquasecurity repository: trivy-db access_token: ((concourse_github_dummy.access_token)) + +- name: concourse-prs + type: pull-request + checK_every: 2m + source: + repository: concourse/concourse + access_token: ((pull_requests_access_token)) + base_branch: release/((release_minor)).x diff --git a/tasks/scripts/check-migration-order b/tasks/scripts/check-migration-order index 529b8e24..62a95b09 100755 --- a/tasks/scripts/check-migration-order +++ b/tasks/scripts/check-migration-order @@ -4,11 +4,13 @@ migrations_path=atc/db/migration/migrations/ base_dir=concourse-base/$migrations_path pr_dir=concourse-pr/$migrations_path -if grep skip-migrations-check concourse-pr/.git/resource/title >/dev/null; then - echo "skipping migration check at PR's request" - echo "pr title: $(cat concourse-pr/.git/resource/title)" - exit 0 -fi +# Disabling for now as the current PR approach doesn't provide the PR title +# +# if grep skip-migrations-check concourse-pr/.git/resource/title >/dev/null; then +# echo "skipping migration check at PR's request" +# echo "pr title: $(cat concourse-pr/.git/resource/title)" +# exit 0 +# fi migrations_on_base=$(mktemp) find $base_dir -name '[0-9]*' -exec basename {} \; | diff --git a/tasks/scripts/validate-pr b/tasks/scripts/validate-pr index 567d76be..4fef2643 100755 --- a/tasks/scripts/validate-pr +++ b/tasks/scripts/validate-pr @@ -2,12 +2,10 @@ set -e -u -PRNUMBER=$(cat concourse-pr/.git/resource/pr) - chmod +x validator/releaseme validator/releaseme validate \ --github-token=$GITHUB_TOKEN \ --github-owner=concourse \ --github-repo=concourse \ - --pr-number=$PRNUMBER \ \ No newline at end of file + --pr-number=$PR_NUMBER diff --git a/tasks/validate-pr.yml b/tasks/validate-pr.yml index c6344753..50c65148 100644 --- a/tasks/validate-pr.yml +++ b/tasks/validate-pr.yml @@ -12,6 +12,7 @@ inputs: params: GITHUB_TOKEN: + PR_NUMBER: run: - path: ci/tasks/scripts/validate-pr \ No newline at end of file + path: ci/tasks/scripts/validate-pr