Rework logging mechanisms

concourse/pipelines/bootstrap-all-init-pipelines.yml

@@ -255,6 +255,7 @@ jobs:
     params:
       IAAS_TYPE: ((iaas-type))
       PIPELINE_TYPE: control-plane
+      DEBUG: true
   - task: set-control-plane
     input_mapping: {scripts-resource: cf-ops-automation,templates-resource: paas-templates-full,secrets-resource: secrets-full, pipelines-resource: all-pipelines}
     file: cf-ops-automation/concourse/tasks/bootstrap_shared_pipelines.yml

concourse/pipelines/shared/control-plane-pipeline.yml.erb

@@ -10,6 +10,7 @@
   concourse_retry = configurer.concourse_retry
   jobs = Hash.new {|h,k| h[k]=[]}
   control_plane_jobs = []
+  coa_debug_mode = configurer.coa_debug_mode.get
 %>
 ---
 display:
@@ -308,6 +309,7 @@ jobs:
         params:
           IAAS_TYPE: ((iaas-type))
           PROFILES: ((profiles))
+          DEBUG: <%= coa_debug_mode %>
       - task: copy-and-format-pipelines
         config:
           platform: linux

concourse/pipelines/template/bosh-pipeline.yml.erb

@@ -48,6 +48,8 @@
 %>
 ---
 meta:
+  vars:
+    - &custom_ca_cert <%= bosh_cert[root_deployment_name]&.dump %>
   tasks:
     - &on_failure
       put: failure-alert
@@ -56,6 +58,43 @@ meta:
         text: Failed to run [[$BUILD_PIPELINE_NAME/$BUILD_JOB_NAME ($BUILD_NAME)]($ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME)].
         icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
         username: Concourse
+    - &curl_ssl_image
+      type: registry-image
+      source:
+        repository: elpaasoci/curl-ssl
+        tag: e3520d05ad3ff8564810ba75f148d9426a7ebe8b
+    - &generate_coa_ops_and_vars_files |
+        mkdir -p result-dir/operators
+        mkdir -p result-dir/vars
+        find final-release-manifest/ -name "*-operators.yml" -type f -exec cp {} result-dir/operators \;
+        find final-release-manifest/ -name "*-vars.yml" -type f -exec cp {} result-dir/vars \;
+        echo "Generating '0-coa-templates-reference-operators.yml'"
+        {
+          echo '- type: replace'
+          echo '  path: /tags?/coa-templates-commit-id?'
+          echo '  value: "((''coa-templates-commit-id))"'
+
+          echo '- type: replace'
+          echo '  path: /tags?/coa-templates-commit-author?'
+          echo '  value: "((''coa-templates-commit-author))"'
+        } > result-dir/operators/0-coa-templates-reference-operators.yml
+        echo "Done - file '0-coa-templates-reference-operators.yml'"
+
+        echo "Generating '0-coa-templates-reference-vars.yml'"
+        {
+          echo 'coa-templates-commit-id: "'"$(cat  template-resource/.git/ref)"'"'
+          echo 'coa-templates-commit-author: "'"$(cat template-resource/.git/committer)"'"'
+        } > result-dir/vars/0-coa-templates-reference-vars.yml
+        echo "Done - file '0-coa-templates-reference-vars.yml'"
+        echo "Generating '0-coa-metadata-vars.yml'"
+        {
+          echo 'paas_templates_commit_id: "'"$(cat  template-resource/.git/ref)"'"'
+          echo 'coa_root_deployment_name: "'"${ROOT_DEPLOYMENT_NAME}"'"'
+          echo 'coa_deployment_name: "'"${DEPLOYMENT_NAME}"'"'
+          echo 'iaas_type: "'"${IAAS_TYPE}"'"'
+          echo 'profiles: "'"${PROFILES}"'"'
+        } > result-dir/vars/0-coa-metadata-vars.yml
+        echo "Done - file '0-coa-metadata-vars.yml'"
 display:
   background_image: ((background-image-url))
 resource_types:
@@ -200,7 +239,7 @@ resources:
     client: ((bosh-username))
     client_secret: "((bosh-password))"
     deployment: <%= name %>
-    ca_cert: <%= bosh_cert[root_deployment_name]&.dump %>
+    ca_cert: *custom_ca_cert
 
   <% if deployment_details.errands? || deployment_details.manual_errands? %>
 - name: errand-<%= name %>
@@ -211,7 +250,7 @@ resources:
     client: ((bosh-username))
     client_secret: "((bosh-password))"
     deployment: <%= name %>
-    ca_cert: <%= bosh_cert[root_deployment_name]&.dump %>
+    ca_cert: *custom_ca_cert
   <% end %>
 <% end %>
 
@@ -345,11 +384,7 @@ jobs:
     - task: check-repackaging-errors
       config:
         platform: linux
-        image_resource:
-          type: registry-image
-          source:
-            repository: elpaasoci/curl-ssl
-            tag: e3520d05ad3ff8564810ba75f148d9426a7ebe8b
+        image_resource: *curl_ssl_image
         inputs:
           - name: repackaged-releases-fallback
         run:
@@ -705,7 +740,8 @@ jobs:
 <% current_serial_group = configurer.serial_group_strategy.generate(name, boshrelease) %>
 <% deployment_details = PipelineHelpers::DeploymentDetails.new(name, boshrelease) %>
 - name: deploy-<%= name %>
-  <% jobs["deploy-#{name[0]}"] << "deploy-#{name}" %>
+  <% jobs["deploy-#{name[0]}"] << "deploy-#{name[0]}*" unless jobs["deploy-#{name[0]}"].include?("deploy-#{name[0]}*") %>
+
   serial: true
   <%= "serial_groups: [#{current_serial_group}]" if enabled_parallel_execution_limit%>
   on_failure: *on_failure
@@ -816,11 +852,7 @@ jobs:
     output_mapping: {result-dir: ops-and-vars-files}
     config:
       platform: linux
-      image_resource:
-        type: registry-image
-        source:
-          repository: elpaasoci/curl-ssl
-          tag: e3520d05ad3ff8564810ba75f148d9426a7ebe8b
+      image_resource: *curl_ssl_image
       inputs:
         - name: final-release-manifest
         - name: template-resource
@@ -830,38 +862,7 @@ jobs:
         path: sh
         args:
         - -ec
-        - |
-          mkdir -p result-dir/operators
-          mkdir -p result-dir/vars
-          find final-release-manifest/ -name "*-operators.yml" -type f -exec cp {} result-dir/operators \;
-          find final-release-manifest/ -name "*-vars.yml" -type f -exec cp {} result-dir/vars \;
-          echo "Generating '0-coa-templates-reference-operators.yml'"
-          {
-            echo '- type: replace'
-            echo '  path: /tags?/coa-templates-commit-id?'
-            echo '  value: "((''coa-templates-commit-id))"'
-
-            echo '- type: replace'
-            echo '  path: /tags?/coa-templates-commit-author?'
-            echo '  value: "((''coa-templates-commit-author))"'
-          } > result-dir/operators/0-coa-templates-reference-operators.yml
-          echo "Done - file '0-coa-templates-reference-operators.yml'"
-
-          echo "Generating '0-coa-templates-reference-vars.yml'"
-          {
-            echo 'coa-templates-commit-id: "'"$(cat  template-resource/.git/ref)"'"'
-            echo 'coa-templates-commit-author: "'"$(cat template-resource/.git/committer)"'"'
-          } > result-dir/vars/0-coa-templates-reference-vars.yml
-          echo "Done - file '0-coa-templates-reference-vars.yml'"
-          echo "Generating '0-coa-metadata-vars.yml'"
-          {
-            echo 'paas_templates_commit_id: "'"$(cat  template-resource/.git/ref)"'"'
-            echo 'coa_root_deployment_name: "'"${ROOT_DEPLOYMENT_NAME}"'"'
-            echo 'coa_deployment_name: "'"${DEPLOYMENT_NAME}"'"'
-            echo 'iaas_type: "'"${IAAS_TYPE}"'"'
-            echo 'profiles: "'"${PROFILES}"'"'
-          } > result-dir/vars/0-coa-metadata-vars.yml
-          echo "Done - file '0-coa-metadata-vars.yml'"
+        - *generate_coa_ops_and_vars_files
       params:
         PROFILES: ((profiles))
         IAAS_TYPE: ((iaas-type))
@@ -986,7 +987,7 @@ jobs:
     <% deployment_details.errands.each do |errand_name, errand_info| %>
     <% errand_display_name = errand_info&.dig('display-name') || errand_name %>
 - name: run-errand-<%= name %>-<%= errand_display_name %>
-  <% jobs["deploy-#{name[0]}"] << "run-errand-#{name}-#{errand_display_name}" %>
+  <% jobs["deploy-#{name[0]}"] << "run-errand-#{name[0]}*" unless jobs["deploy-#{name[0]}"].include?("run-errand-#{name[0]}*") %>
   serial_groups: [auto-errand-<%= name %>]
   on_failure: *on_failure
   plan:
@@ -1005,8 +1006,7 @@ jobs:
     <% deployment_details.manual_errands.each do |errand_name, errand_info| %>
     <% errand_display_name = errand_info&.dig('display-name') || errand_name %>
 - name: run-manual-errand-<%= name %>-<%= errand_display_name %>
-  <% jobs["deploy-#{name[0]}"] << "run-manual-errand-#{name}-#{errand_display_name}" %>
-
+  <% jobs["deploy-#{name[0]}"] << "run-manual-errand-#{name[0]}*" unless jobs["deploy-#{name[0]}"].include?("run-manual-errand-#{name[0]}*") %>
   serial: true
   on_failure: *on_failure
   plan:
@@ -1034,11 +1034,7 @@ jobs:
     output_mapping: {result-dir: trigger-<%= root_deployment_name %>-plan}
     config:
       platform: linux
-      image_resource:
-        type: registry-image
-        source:
-          repository: elpaasoci/curl-ssl
-          tag: e3520d05ad3ff8564810ba75f148d9426a7ebe8b
+      image_resource: *curl_ssl_image
       outputs:
         - name: result-dir
       run:
@@ -1083,11 +1079,7 @@ jobs:
     output_mapping: {result-dir: init-<%= root_deployment_name %>-plan}
     config:
       platform: linux
-      image_resource:
-        type: registry-image
-        source:
-          repository: elpaasoci/curl-ssl
-          tag: e3520d05ad3ff8564810ba75f148d9426a7ebe8b
+      image_resource: *curl_ssl_image
       outputs:
         - name: result-dir
       run:

concourse/pipelines/template/bosh-precompile-pipeline.yml.erb

@@ -47,7 +47,6 @@
   releases_per_root_deployments_reference = releases_per_root_deployments.dup
   releases_per_root_deployments.each do |root_depl, releases_info|
     depends_on = config&.dig(root_depl,'precompile', 'depends-on') || []
-    puts "#{root_depl} depends_on: #{depends_on}"
     releases_info.each do |release_name, details|
       already_defined = false
       depends_on.each do |depend_root_depl|

concourse/tasks/generate-all-pipelines.yml

@@ -26,4 +26,5 @@ params:
   IAAS_TYPE:
   PIPELINE_TYPE:
   PROFILES:
-  PROFILES_AUTOSORT: true
+  PROFILES_AUTOSORT: true
+  DEBUG: false

concourse/tasks/generate-shared-pipelines.yml

@@ -26,4 +26,5 @@ params:
   IAAS_TYPE:
   PIPELINE_TYPE:
   PROFILES:
-  PROFILES_AUTOSORT: true
+  PROFILES_AUTOSORT: true
+  DEBUG: false

concourse/tasks/generate_depls/run.sh

@@ -14,6 +14,16 @@
 set -C
 set -o pipefail # error on first command failure when using pipe
 
+DEBUG_OPTIONS="--no-dump"
+if [ "${DEBUG}" = "true" ]
+then
+   echo "INFO: Enabling debug options: dump pipelines"
+   DEBUG_OPTIONS="--dump"
+else
+    echo "INFO: debug mode disabled"
+fi
+
+
 if [ -n "${PROFILES}" ]
 then
    echo "Profiles detected: ${PROFILES}"
@@ -53,7 +63,7 @@ cp -r templates/. result-dir
 cp -r scripts-resource/. result-dir
 cp -rf secrets/. result-dir
 cd result-dir
-./scripts/generate-depls.rb --depls "${ROOT_DEPLOYMENT}" -t ../templates -p . -o concourse --iaas "${IAAS_TYPE}" ${PROFILES_OPTION} ${PIPELINES_RESTRICTIONS} > >(tee generate-depls.log) 2> >(tee -a error.log >&2) # tee generate-depls.log
+./scripts/generate-depls.rb --depls "${ROOT_DEPLOYMENT}" -t ../templates -p . -o concourse ${DEBUG_OPTIONS} --iaas "${IAAS_TYPE}" ${PROFILES_OPTION} ${PIPELINES_RESTRICTIONS} > >(tee generate-depls.log) 2> >(tee -a error.log >&2) # tee generate-depls.log
 if [ -s 'error.log' ]; then
     cat error.log
     exit 1

concourse/tasks/generate_depls/task.yml

@@ -29,4 +29,5 @@ params:
   IAAS_TYPE: ((iaas-type))
   EXCLUDE_PIPELINES:
   PROFILES:
-  PROFILES_AUTOSORT: true
+  PROFILES_AUTOSORT: true
+  DEBUG: false

docs/reference_dataset/config_repository/private-config.yml

@@ -1,4 +1,9 @@
 ---
+# Log configuration is only available in private-config.yml
+log:
+  output: STDOUT # Use stdout to have log displayed in concourse, instead of a file. Default: File
+  # level: Debug # We don't need to customize this value, as it is set to debug by default. Valid values: Debug, Info, Warn and Error
+  date-format: "-" # Disable date as we already have a timestamp in concourse. Default: "%Y-%m-%dT%H:%M:%S.%6N"
 # you can override values from template_repository/shared-config.yml
 #offline-mode:
 #   boshreleases: false # Default: false

lib/cf_apps.rb

@@ -1,22 +1,24 @@
 require 'yaml'
+require_relative 'coa_run_logger'
 
 class CfApps
   attr_reader :base_path, :root_deployment_name
 
+  include CoaRunLogger
   def initialize(path, root_deployment_name)
     @base_path = path
     @root_deployment_name = root_deployment_name
   end
 
   def overview
     cf_apps = {}
-    puts "Path CF App: #{base_path}"
+    logger.info "Path CF App: #{base_path}"
 
     Dir[base_path].select { |file| File.directory? file }.each do |base_subdir|
       collect_cf_apps_details(base_subdir, cf_apps)
     end
 
-    puts "cf_apps: \n#{YAML.dump(cf_apps)}"
+    logger.debug "cf_apps: \n#{YAML.dump(cf_apps)}"
     cf_apps
   end
 
@@ -28,7 +30,7 @@ def self.enable_cf_app_files(dir)
 
   def collect_cf_apps_details(base_subdir, cf_apps)
     subdir_name = File.basename(base_subdir)
-    puts "Processing CF App: #{subdir_name}"
+    logger.info "Processing CF App: #{subdir_name}"
 
     self.class.enable_cf_app_files(base_subdir).each do |enable_cf_app_file|
       load_cf_apps_details_from_file(cf_apps, enable_cf_app_file, base_subdir)
@@ -38,7 +40,7 @@ def collect_cf_apps_details(base_subdir, cf_apps)
   end
 
   def load_cf_apps_details_from_file(cf_apps, file, subdir)
-    puts "Cf App detected: #{subdir} - #{file}"
+    logger.info "Cf App detected: #{subdir} - #{file}"
     dir = File.dirname(file)
     cf_apps_description = YAML.load_file(file, aliases: true)
 
@@ -48,7 +50,7 @@ def load_cf_apps_details_from_file(cf_apps, file, subdir)
   end
 
   def load_cf_app_details(cf_app_name, cf_app_details, file, dir, cf_apps)
-    puts "processing cf-app: #{cf_app_name} from #{file}"
+    logger.debug "processing cf-app: #{cf_app_name} from #{file}"
     raise "cannot process #{file}, an application named #{cf_app_name} already exists" if cf_apps.key?(cf_app_name)
 
     #   raise "#{dependency_file} - Invalid deployment: expected <#{dirname}> - Found <#{deployment_name}>" if deployment_name != dirname

lib/ci_deployment.rb

@@ -1,9 +1,11 @@
 require 'yaml'
+require_relative 'coa_run_logger'
 
 class CiDeployment
   attr_reader :base_path
   attr_accessor :content
 
+  include CoaRunLogger
   def initialize(path)
     @base_path = path
     @content = {}
@@ -34,13 +36,13 @@ def initialize(path)
   #       ops-depls-cf-apps-generated:
 
   def overview
-    puts "Path CI deployment overview: #{base_path}"
+    logger.info "Path CI deployment overview: #{base_path}"
 
     Dir[base_path].select { |file| File.directory? file }.each do |path|
       load_ci_deployment_from_dir(path)
     end
 
-    puts "ci_deployment loaded: \n#{YAML.dump(content)}"
+    logger.debug "ci_deployment loaded: \n#{YAML.dump(content)}"
     content
   end
 
@@ -66,15 +68,15 @@ def self.team(overview, root_deployment, pipeline_name)
 
   def load_ci_deployment_from_dir(path)
     dir_basename = File.basename(path)
-    puts "Processing #{dir_basename}"
+    logger.debug "Processing #{dir_basename}"
 
     Dir[path + '/ci-deployment-overview.yml'].each do |deployment_file|
       load_ci_deployment_from_file(deployment_file, dir_basename)
     end
   end
 
   def load_ci_deployment_from_file(deployment_file, dir_basename)
-    puts "CI deployment detected in #{dir_basename}"
+    logger.info "CI deployment detected in #{dir_basename}"
 
     deployment = YAML.load_file(deployment_file, aliases: true)
     raise "#{deployment} - Invalid deployment: expected 'ci-deployment' key as yaml root" unless deployment && deployment['ci-deployment']
@@ -105,7 +107,7 @@ def processes_pipeline_definitions(deployment_details)
     deployment_details['pipelines'].each do |pipeline_name, pipeline_details|
       next unless pipeline_details
       unless pipeline_details_config_file?(pipeline_details)
-        puts "Generating default value for key config_file in #{pipeline_name}"
+        logger.debug "Generating default value for key config_file in #{pipeline_name}"
         pipeline_details['config_file'] = "concourse/pipelines/#{pipeline_name}.yml"
       end
     end