pipelines(br-upload): add a retrigger-all-uploads task

orange-cloudfoundry · Jan 15, 2019 · e577677 · e577677
1 parent 86bf20c
commit e577677
Show file tree

Hide file tree

Showing 4 changed files with 173 additions and 9 deletions.
diff --git a/concourse/pipelines/template/s3-br-upload-pipeline.yml.erb b/concourse/pipelines/template/s3-br-upload-pipeline.yml.erb
@@ -78,11 +78,12 @@ resources:
 jobs:
 
 - name: init-concourse-boshrelease-for-<%= depls %>
+<% jobs = ["init-concourse-boshrelease-for-#{depls}"] %>
   on_failure:
     put: failure-alert
     params:
       channel: ((slack-channel))
-      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/main/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
       icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
       username: Concourse
   plan:
@@ -124,15 +125,58 @@ jobs:
       FLY_PASSWORD: "((concourse-<%= depls %>-password))"
       FLY_TEAM: <%= current_team || 'main' %>
 
+- name: retrigger-all-uploads
+  <% jobs << "retrigger-all-uploads" %>
+  on_failure:
+    put: failure-alert
+    params:
+      channel: ((slack-channel))
+      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
+      username: Concourse
+  plan:
+  - aggregate:
+    - get: cf-ops-automation
+      params: { submodules: none}
+  - task: generate-<%= depls %>-pipeline
+    output_mapping: {result-dir: trigger-<%= depls %>-plan}
+    config:
+      platform: linux
+      image_resource:
+        type: docker-image
+        source: {repository: concourse/busyboxplus, tag: "git"}
+      outputs:
+        - name: result-dir
+      run:
+        path: sh
+        args:
+        - -exc
+        - |
+  <% uniq_releases.keys.sort.each do |release, _|  %>
+          echo "trigger-job -j $BUILD_PIPELINE_NAME/upload-current-<%= release %>" >> result-dir/flight-plan
+  <% end %>
+      params:
+        BUILD_PIPELINE_NAME: <%= depls %>-s3-br-upload-generated
+  - task: fly-into-concourse
+    input_mapping: {fly-cmd: trigger-<%= depls %>-plan}
+    output_mapping: {flight-report: concourse-<%= depls %>-trigger-report}
+    file: cf-ops-automation/concourse/tasks/fly_execute_commands.yml
+    params:
+      ATC_EXTERNAL_URL: ((concourse-<%= depls %>-target))
+      FLY_USERNAME: ((concourse-<%= depls %>-username))
+      FLY_PASSWORD: ((concourse-<%= depls %>-password))
+      FLY_TEAM: upload
+
 <% uniq_releases.sort.each do |release, info|  %>
   <% current_serial_group = configurer.serial_group_strategy.generate(release, info) %>
 - name: upload-current-<%= release %>
+  <% jobs << "upload-current-#{release}" %>
   serial_groups: [<%= release %><%= ",#{current_serial_group}" if enabled_parallel_execution_limit%>]
   on_failure:
     put: failure-alert
     params:
       channel: ((slack-channel))
-      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/main/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
       icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
       username: Concourse
   plan:
@@ -181,12 +225,13 @@ jobs:
         acl: public-read
 
 - name: upload-latest-<%= release %>
+  <% jobs << "upload-latest-#{release}" %>
   serial_groups: [<%= release %><%= ",#{current_serial_group}" if enabled_parallel_execution_limit%>]
   on_failure:
     put: failure-alert
     params:
       channel: ((slack-channel))
-      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/main/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
       icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
       username: Concourse
   plan:
@@ -231,3 +276,19 @@ jobs:
         file: <%= release %>/*.tgz
         acl: public-read
 <% end %>
+
+groups:
+- name: <%= depls.capitalize %>
+  jobs:
+<% jobs.each do |job| %>
+  - <%= job %>
+<% end %>
+
+<% uniq_releases.keys.group_by {|name| name[0] }.each do |first_letter, releases| %>
+- name: "<%= first_letter %>*"
+  jobs:
+  <% releases.each do |release| %>
+  - <%= "upload-current-#{release}" %>
+  - <%= "upload-latest-#{release}" %>
+  <% end %>
+<% end %>
diff --git a/scripts/bootstrap_coa_env.rb b/scripts/bootstrap_coa_env.rb
@@ -20,8 +20,8 @@
   else
     [
       "ci/bootstrap_coa_env/*-prereqs.yml",
-      "ci/bootstrap_coa_env/bucc/*-prereqs.yml",
-      "ci/bootstrap_coa_env/virtualbox/*-prereqs.yml" # NOTE: maybe make this an arg? make iaas an arg?
+      "ci/bootstrap_coa_env/virtualbox/*-prereqs.yml", # NOTE: maybe make this an arg? make iaas an arg?
+      "ci/bootstrap_coa_env/bucc/*-prereqs.yml"
     ]
   end
 

diff --git a/spec/scripts/generate-depls/fixtures/references/empty-s3-br-upload.yml b/spec/scripts/generate-depls/fixtures/references/empty-s3-br-upload.yml
@@ -33,7 +33,7 @@ jobs:
     put: failure-alert
     params:
       channel: "((slack-channel))"
-      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/main/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
       icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
       username: Concourse
   plan:
@@ -74,3 +74,51 @@ jobs:
       FLY_USERNAME: "((concourse-dummy-depls-username))"
       FLY_PASSWORD: "((concourse-dummy-depls-password))"
       FLY_TEAM: main
+- name: retrigger-all-uploads
+  on_failure:
+    put: failure-alert
+    params:
+      channel: "((slack-channel))"
+      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
+      username: Concourse
+  plan:
+  - aggregate:
+    - get: cf-ops-automation
+      params:
+        submodules: none
+  - task: generate-dummy-depls-pipeline
+    output_mapping:
+      result-dir: trigger-dummy-depls-plan
+    config:
+      platform: linux
+      image_resource:
+        type: docker-image
+        source:
+          repository: concourse/busyboxplus
+          tag: git
+      outputs:
+      - name: result-dir
+      run:
+        path: sh
+        args:
+        - "-exc"
+        - ''
+      params:
+        BUILD_PIPELINE_NAME: dummy-depls-s3-br-upload-generated
+  - task: fly-into-concourse
+    input_mapping:
+      fly-cmd: trigger-dummy-depls-plan
+    output_mapping:
+      flight-report: concourse-dummy-depls-trigger-report
+    file: cf-ops-automation/concourse/tasks/fly_execute_commands.yml
+    params:
+      ATC_EXTERNAL_URL: "((concourse-dummy-depls-target))"
+      FLY_USERNAME: "((concourse-dummy-depls-username))"
+      FLY_PASSWORD: "((concourse-dummy-depls-password))"
+      FLY_TEAM: upload
+groups:
+- name: Dummy-depls
+  jobs:
+  - init-concourse-boshrelease-for-dummy-depls
+  - retrigger-all-uploads
diff --git a/spec/scripts/generate-depls/fixtures/references/simple-depls-s3-br-upload-ref.yml b/spec/scripts/generate-depls/fixtures/references/simple-depls-s3-br-upload-ref.yml
@@ -47,7 +47,7 @@ jobs:
     put: failure-alert
     params:
       channel: "((slack-channel))"
-      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/main/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
       icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
       username: Concourse
   plan:
@@ -89,6 +89,50 @@ jobs:
       FLY_USERNAME: "((concourse-simple-depls-username))"
       FLY_PASSWORD: "((concourse-simple-depls-password))"
       FLY_TEAM: custom-team-name
+- name: retrigger-all-uploads
+  on_failure:
+    put: failure-alert
+    params:
+      channel: "((slack-channel))"
+      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
+      username: Concourse
+  plan:
+  - aggregate:
+    - get: cf-ops-automation
+      params:
+        submodules: none
+  - task: generate-simple-depls-pipeline
+    output_mapping:
+      result-dir: trigger-simple-depls-plan
+    config:
+      platform: linux
+      image_resource:
+        type: docker-image
+        source:
+          repository: concourse/busyboxplus
+          tag: git
+      outputs:
+      - name: result-dir
+      run:
+        path: sh
+        args:
+        - "-exc"
+        - |
+          echo "trigger-job -j $BUILD_PIPELINE_NAME/upload-current-ntp_boshrelease" >> result-dir/flight-plan
+      params:
+        BUILD_PIPELINE_NAME: simple-depls-s3-br-upload-generated
+  - task: fly-into-concourse
+    input_mapping:
+      fly-cmd: trigger-simple-depls-plan
+    output_mapping:
+      flight-report: concourse-simple-depls-trigger-report
+    file: cf-ops-automation/concourse/tasks/fly_execute_commands.yml
+    params:
+      ATC_EXTERNAL_URL: "((concourse-simple-depls-target))"
+      FLY_USERNAME: "((concourse-simple-depls-username))"
+      FLY_PASSWORD: "((concourse-simple-depls-password))"
+      FLY_TEAM: upload
 - name: upload-current-ntp_boshrelease
   serial_groups:
   - ntp_boshrelease
@@ -97,7 +141,7 @@ jobs:
     put: failure-alert
     params:
       channel: "((slack-channel))"
-      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/main/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
       icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
       username: Concourse
   plan:
@@ -152,7 +196,7 @@ jobs:
     put: failure-alert
     params:
       channel: "((slack-channel))"
-      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/main/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
+      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
       icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
       username: Concourse
   plan:
@@ -198,3 +242,14 @@ jobs:
     params:
       file: ntp_boshrelease/*.tgz
       acl: public-read
+groups:
+- name: Simple-depls
+  jobs:
+  - init-concourse-boshrelease-for-simple-depls
+  - retrigger-all-uploads
+  - upload-current-ntp_boshrelease
+  - upload-latest-ntp_boshrelease
+- name: n*
+  jobs:
+  - upload-current-ntp_boshrelease
+  - upload-latest-ntp_boshrelease