s3-br-upload-pipeline.yml.erb

<%
  require './lib/pipeline_helpers'

  uniq_releases= {}
  all_dependencies.sort.each do |name, boshrelease|
    boshrelease["releases"]&.each do |release, info|
      uniq_releases[release] = info
    end
  end

  pipeline_options = PipelineHelpers::PipelineConfigurerOptions.new.with_config(config).with_root_deployment(depls).build
  configurer = PipelineHelpers::PipelineConfigurer.new(pipeline_options)

  enabled_parallel_execution_limit = configurer.parallel_execution_limit.overridden?
%>

---
resource_types:
  - name: slack-notification
    type: docker-image
    source:
      repository: cfcommunity/slack-notification-resource
      tag: v1.4.2

  - name: cron-resource
    type: docker-image
    source:
      repository: cftoolsmiths/cron-resource
      # use latest as no other recent tag available

resources:
- name: weekday-morning
  type: cron-resource
  source:
    expression: 40 8 * * 1-5
    location: "Europe/Paris"
    fire_immediately: true

- name: failure-alert
  type: slack-notification
  source:
    url: ((slack-webhook))

- name: cf-ops-automation
  type: git
  source:
    uri: ((cf-ops-automation-uri))
    branch: ((cf-ops-automation-branch))
    tag_filter: ((cf-ops-automation-tag-filter))
    skip_ssl_verification: true

<% uniq_releases.sort.each do |release, info| %>
- name: <%= release %>
  <% if PipelineHelpers.bosh_io_hosted?(info) %>
  type: bosh-io-release
  source:
    repository: <%= info["repository"] %>
  <% else %>
  type: github-release
  source:
    user: <%= info["repository"].split('/').first %>
    repository: <%= info["repository"].split('/').last %>
    access_token: ((bot-github-access-token))
  <% end %>

- name: <%= release %>-s3
  type: s3
  source:
    bucket: ((s3-br-bucket))
    region_name: ((s3-br-region-name))
    regexp: <%= info['repository']&.split('/')&.first %>/<%= release %>-(.*).tgz
    access_key_id: ((s3-br-access-key-id))
    secret_access_key: ((s3-br-secret-key))
    endpoint: ((s3-br-endpoint))
    skip_ssl_verification: ((s3-br-skip-ssl-verification))
<% end %>

jobs:

- name: init-concourse-boshrelease-for-<%= depls %>
<% jobs = ["init-concourse-boshrelease-for-#{depls}"] %>
  on_failure:
    put: failure-alert
    params:
      channel: ((slack-channel))
      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
      icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
      username: Concourse
  plan:
  - aggregate:
    - get: cf-ops-automation
      params: { submodules: none}
      attempts: 2
    - get: weekday-morning
      trigger: true
  - task: generate-<%= depls %>-flight-plan
    output_mapping: {result-dir: init-<%= depls %>-plan}
    config:
      platform: linux
      image_resource:
        type: docker-image
        source: {repository: concourse/busyboxplus, tag: "git"}
      outputs:
        - name: result-dir
      run:
        path: sh
        args:
        - -exc
        - |
          <% uniq_releases.sort.each do |name, info| %>
            <% from_prefix = PipelineHelpers.bosh_io_hosted?(info) ? "version" : "tag" %>
          echo "check-resource -r $BUILD_PIPELINE_NAME/<%= name %> --from <%= from_prefix %>:((<%= name %>-version))" >> result-dir/flight-plan
          <% end %>
      params:
        BUILD_PIPELINE_NAME: <%= depls %>-s3-br-upload-generated

  <% current_team = CiDeployment.team(all_ci_deployments, depls, "#{depls}-s3-br-upload-generated") %>
  - task: fly-into-concourse
    input_mapping: {fly-cmd: init-<%= depls %>-plan}
    output_mapping: {flight-report: concourse-<%= depls %>-init-report}
    file: cf-ops-automation/concourse/tasks/fly_execute_commands.yml
    params:
      ATC_EXTERNAL_URL: ((concourse-<%= depls %>-target))
      FLY_USERNAME: ((concourse-<%= depls %>-username))
      FLY_PASSWORD: "((concourse-<%= depls %>-password))"
      FLY_TEAM: <%= current_team || 'main' %>

- name: retrigger-all-uploads
  <% jobs << "retrigger-all-uploads" %>
  on_failure:
    put: failure-alert
    params:
      channel: ((slack-channel))
      text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
      icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
      username: Concourse
  plan:
  - aggregate:
    - get: cf-ops-automation
      params: { submodules: none}
  - task: generate-<%= depls %>-pipeline
    output_mapping: {result-dir: trigger-<%= depls %>-plan}
    config:
      platform: linux
      image_resource:
        type: docker-image
        source: {repository: concourse/busyboxplus, tag: "git"}
      outputs:
        - name: result-dir
      run:
        path: sh
        args:
        - -exc
        - |
  <% uniq_releases.keys.sort.each do |release, _|  %>
          echo "trigger-job -j $BUILD_PIPELINE_NAME/upload-current-<%= release %>" >> result-dir/flight-plan
  <% end %>
      params:
        BUILD_PIPELINE_NAME: <%= depls %>-s3-br-upload-generated
  - task: fly-into-concourse
    input_mapping: {fly-cmd: trigger-<%= depls %>-plan}
    output_mapping: {flight-report: concourse-<%= depls %>-trigger-report}
    file: cf-ops-automation/concourse/tasks/fly_execute_commands.yml
    params:
      ATC_EXTERNAL_URL: ((concourse-<%= depls %>-target))
      FLY_USERNAME: ((concourse-<%= depls %>-username))
      FLY_PASSWORD: ((concourse-<%= depls %>-password))
      FLY_TEAM: upload

<% uniq_releases.sort.each do |release, info|  %>
  <% current_serial_group = configurer.serial_group_strategy.generate(release, info) %>
- name: upload-current-<%= release %>
  <% jobs << "upload-current-#{release}" %>
  serial_groups: [<%= release %><%= ",#{current_serial_group}" if enabled_parallel_execution_limit%>]
  on_failure:
    put: failure-alert
    params:
      channel: ((slack-channel))
      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
      icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
      username: Concourse
  plan:
    - aggregate:
      - get: <%= release %>
        attempts: 2
  <% if PipelineHelpers.bosh_io_hosted?(info) %>
        version: { version: "((<%= release %>-version))" }
  <% else %>
        version: { tag: "((<%= release %>-version))" }
  <% end %>
        trigger: true
        params: {tarball: true}
      - get: cf-ops-automation
        params: { submodules: none}
        attempts: 2
        passed: [ init-concourse-boshrelease-for-<%= depls %> ]
    - task: generate-<%= release %>-name
      input_mapping: {release: <%= release %>}
      output_mapping: {result-dir: <%= release %>}
      config:
        platform: linux
        image_resource:
          type: docker-image
          source: {repository: concourse/busyboxplus, tag: "git"}
        inputs:
          - name: release
        outputs:
          - name: result-dir
        run:
          path: sh
          args:
          - -exc
          - |
            VERSION=$(cat release/version)
  <% if PipelineHelpers.bosh_io_hosted?(info) %>
            cp release/release.tgz result-dir/${RELEASE_PREFIX}-${VERSION}.tgz
  <% else %>
            cp release/${RELEASE_PREFIX}-${VERSION}.tgz result-dir/${RELEASE_PREFIX}-${VERSION}.tgz
  <% end %>
        params:
          RELEASE_PREFIX: <%= release %>
    - put: <%= release %>-s3
      params:
        file: <%= release %>/*.tgz
        acl: public-read

- name: upload-latest-<%= release %>
  <% jobs << "upload-latest-#{release}" %>
  serial_groups: [<%= release %><%= ",#{current_serial_group}" if enabled_parallel_execution_limit%>]
  on_failure:
    put: failure-alert
    params:
      channel: ((slack-channel))
      text: Failed [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
      icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
      username: Concourse
  plan:
    - aggregate:
      - get: <%= release %>
        attempts: 2
        version: every
        trigger: true
        params: {tarball: true}
      - get: cf-ops-automation
        params: { submodules: none}
        attempts: 2
        passed: [ upload-current-<%= release %> ]
    - task: generate-<%= release %>-name
      input_mapping: {release: <%= release %>}
      output_mapping: {result-dir: <%= release %>}
      config:
        platform: linux
        image_resource:
          type: docker-image
          source: {repository: concourse/busyboxplus, tag: "git"}
        inputs:
          - name: release
        outputs:
          - name: result-dir
        run:
          path: sh
          args:
          - -exc
          - |
            VERSION=$(cat release/version)
  <% if PipelineHelpers.bosh_io_hosted?(info) %>
            cp release/release.tgz result-dir/${RELEASE_PREFIX}-${VERSION}.tgz
  <% else %>
            cp release/${RELEASE_PREFIX}-${VERSION}.tgz result-dir/${RELEASE_PREFIX}-${VERSION}.tgz
  <% end %>

        params:
          RELEASE_PREFIX: <%= release %>
    - put: <%= release %>-s3
      params:
        file: <%= release %>/*.tgz
        acl: public-read
<% end %>

groups:
- name: <%= depls.capitalize %>
  jobs:
<% jobs.each do |job| %>
  - <%= job %>
<% end %>

<% uniq_releases.keys.group_by {|name| name[0] }.each do |first_letter, releases| %>
- name: "<%= first_letter %>*"
  jobs:
  <% releases.each do |release| %>
  - <%= "upload-current-#{release}" %>
  - <%= "upload-latest-#{release}" %>
  <% end %>
<% end %>