Merge pull request #2460 from cloudfoundry/on-stemcell-change-spike

Introduce on-stemcell-change variable strategy

src/bosh-director/lib/bosh/director/api/controllers/deployments_controller.rb

@@ -416,6 +416,7 @@ def initialize(config)
         options['canaries'] = params[:canaries] if params['canaries']
         options['max_in_flight'] = params[:max_in_flight] if params['max_in_flight']
         options['scopes'] = token_scopes
+        options['force_latest_variables'] = true if params['force_latest_variables'] == 'true'
 
         # since authorizer does not look at manifest payload for deployment name
         @deployment = Models::Deployment[name: deployment_name]

src/bosh-director/lib/bosh/director/config_server/client.rb

@@ -6,6 +6,9 @@ class ConfigServerClient
     GENERATION_MODE_CONVERGE = 'converge'.freeze
     GENERATION_MODE_NO_OVERWRITE = 'no-overwrite'.freeze
 
+    ON_DEPLOY_UPDATE_STRATEGY = 'on-deploy'.freeze
+    ON_STEMCELL_CHANGE_UPDATE_STRATEGY = 'on-stemcell-change'.freeze
+
     def initialize(http_client, director_name, logger)
       @config_server_http_client = http_client
       @director_name = director_name
@@ -86,29 +89,53 @@ def interpolate_cross_deployment_link(link_properties_hash, consumer_variable_se
 
     # @param [DeploymentPlan::Variables] variables Object representing variables passed by the user
     # @param [String] deployment_name
-    def generate_values(variables, deployment_name, converge_variables = false, use_link_dns_names = false)
+    def generate_values(variables, deployment_name, converge_variables = false, use_link_dns_names = false, stemcell_change = false)
       deployment_model = @deployment_lookup.by_name(deployment_name)
 
       variables.spec.map do |variable|
         ConfigServerHelper.validate_variable_name(variable['name'])
+        constructed_name = ConfigServerHelper.add_prefix_if_not_absolute(variable['name'], @director_name, deployment_name)
+
+        strategy = variable.dig('update', 'strategy') || ON_DEPLOY_UPDATE_STRATEGY
+        use_latest_version =
+          strategy == ON_DEPLOY_UPDATE_STRATEGY ||
+          stemcell_change && strategy == ON_STEMCELL_CHANGE_UPDATE_STRATEGY ||
+          deployment_model.previous_variable_set&.find_variable_by_name(constructed_name).nil?
+
+        if use_latest_version
+          if variable['type'] == 'certificate'
+            has_ca = variable['options'] && variable['options']['ca']
+            generate_ca(variable, deployment_name) if has_ca
+            variable = generate_links(variable, deployment_model, use_link_dns_names)
+          end
 
-        if variable['type'] == 'certificate'
-          has_ca = variable['options'] && variable['options']['ca']
-          generate_ca(variable, deployment_name) if has_ca
-          variable = generate_links(variable, deployment_model, use_link_dns_names)
+          generation_mode = variable['update_mode']
+          generation_mode ||= converge_variables ? GENERATION_MODE_CONVERGE : GENERATION_MODE_NO_OVERWRITE
+
+          variable_id = generate_latest_version_id(
+            constructed_name,
+            variable['type'],
+            deployment_name,
+            deployment_model.current_variable_set,
+            variable['options'],
+            generation_mode,
+          )
+        else
+          previous_variable_version = deployment_model.previous_variable_set.find_variable_by_name(constructed_name)
+          variable_id = previous_variable_version[:variable_id]
         end
 
-        generation_mode = variable['update_mode']
-        generation_mode ||= converge_variables ? GENERATION_MODE_CONVERGE : GENERATION_MODE_NO_OVERWRITE
+        begin
+          save_variable(get_name_root(constructed_name), deployment_model.current_variable_set, variable_id)
+        rescue Sequel::UniqueConstraintViolation
+          @logger.debug("variable '#{get_name_root(constructed_name)}' was already added to set '#{deployment_model.current_variable_set.id}'")
+        end
 
-        constructed_name = ConfigServerHelper.add_prefix_if_not_absolute(variable['name'], @director_name, deployment_name)
-        generate_value_and_record_event(
-          constructed_name,
-          variable['type'],
-          deployment_name,
-          deployment_model.current_variable_set,
-          variable['options'],
-          generation_mode,
+        add_event(
+          action: 'create',
+          deployment_name: deployment_name,
+          object_name: constructed_name,
+          context: { 'update_strategy' => strategy, 'latest_version' => use_latest_version, 'name' => constructed_name, 'id' => variable_id },
         )
 
         variable
@@ -395,25 +422,6 @@ def save_variable(name_root, variable_set, variable_id)
       variable_set.add_variable(variable_name: name_root, variable_id: variable_id)
     end
 
-    def generate_and_save_value(name, type, variable_set, options, generation_mode)
-      unless variable_set.writable
-        raise Bosh::Director::ConfigServerGenerationError,
-              "Variable '#{get_name_root(name)}' cannot be generated. Variable generation allowed only during deploy action"
-      end
-
-      generated_variable = generate_value(name, type, options, generation_mode)
-
-      raise Bosh::Director::ConfigServerGenerationError, "Failed to version generated variable '#{name}'. Expected Config Server response to have key 'id'" unless generated_variable.key?('id')
-
-      begin
-        save_variable(get_name_root(name), variable_set, generated_variable['id'])
-      rescue Sequel::UniqueConstraintViolation
-        @logger.debug("variable '#{get_name_root(name)}' was already added to set '#{variable_set.id}'")
-      end
-
-      generated_variable
-    end
-
     def generate_value(name, type, options, mode)
       parameters = options.nil? ? {} : options
 
@@ -458,15 +466,17 @@ def add_event(options)
       )
     end
 
-    def generate_value_and_record_event(variable_name, variable_type, deployment_name, variable_set, options, generation_mode)
-      result = generate_and_save_value(variable_name, variable_type, variable_set, options, generation_mode)
-      add_event(
-        action: 'create',
-        deployment_name: deployment_name,
-        object_name: variable_name,
-        context: { 'name' => result['name'], 'id' => result['id'] },
-      )
-      result
+    def generate_latest_version_id(variable_name, variable_type, deployment_name, variable_set, options, generation_mode)
+      unless variable_set.writable
+        raise Bosh::Director::ConfigServerGenerationError,
+              "Variable '#{get_name_root(variable_name)}' cannot be generated. Variable generation allowed only during deploy action"
+      end
+
+      generated_variable = generate_value(variable_name, variable_type, options, generation_mode)
+
+      raise Bosh::Director::ConfigServerGenerationError, "Failed to version generated variable '#{variable_name}'. Expected Config Server response to have key 'id'" unless generated_variable.key?('id')
+
+      generated_variable['id']
     rescue Exception => e
       add_event(
         action: 'create',

src/bosh-director/lib/bosh/director/config_server/variables_interpolator.rb

@@ -49,8 +49,8 @@ def interpolate_with_versioning(raw_hash, variable_set, options = {})
       @config_server_client.interpolate_with_versioning(raw_hash, variable_set, options)
     end
 
-    def generate_values(variables, deployment_name, converge_variables = false, use_link_dns_names = false)
-      @config_server_client.generate_values(variables, deployment_name, converge_variables, use_link_dns_names)
+    def generate_values(variables, deployment_name, converge_variables = false, use_link_dns_names = false, stemcell_change = false)
+      @config_server_client.generate_values(variables, deployment_name, converge_variables, use_link_dns_names, stemcell_change)
     end
 
     def interpolated_versioned_variables_changed?(previous_raw_hash, next_raw_hash, previous_variable_set, target_variable_set)

src/bosh-director/lib/bosh/director/deployment_plan/assembler.rb

@@ -25,6 +25,7 @@ def bind_models(options = {})
       should_bind_links = is_deploy_action && options.fetch(:should_bind_links, true)
       should_bind_properties = options.fetch(:should_bind_properties, true)
       should_bind_new_variable_set = options.fetch(:should_bind_new_variable_set, false)
+      stemcell_change = options.fetch(:stemcell_change, false)
       deployment_options = @deployment_plan.deployment_wide_options
       fix = deployment_options.fetch(:fix, false)
       tags = deployment_options.fetch(:tags, {})
@@ -100,7 +101,7 @@ def bind_models(options = {})
       bind_instance_networks
       bind_dns
       bind_links if should_bind_links
-      generate_variables if is_deploy_action
+      generate_variables(stemcell_change) if is_deploy_action
     end
 
     private
@@ -169,12 +170,13 @@ def bind_links
       end
     end
 
-    def generate_variables
+    def generate_variables(stemcell_change)
       @variables_interpolator.generate_values(
         @deployment_plan.variables,
         @deployment_plan.name,
         @deployment_plan.features.converge_variables,
         @deployment_plan.features.use_link_dns_names,
+        stemcell_change,
       )
     end
 

src/bosh-director/lib/bosh/director/deployment_plan/planner.rb

@@ -26,7 +26,6 @@ class Planner
       #   Default job update configuration
       attr_accessor :update
 
-      # @return [Array<Bosh::Director::DeploymentPlan::Job>]
       #   All instance_groups in the deployment
       attr_reader :instance_groups
 

src/bosh-director/lib/bosh/director/jobs/update_deployment.rb

@@ -91,9 +91,15 @@ def prepare_deployment
           # the DNS encoder having an updated index before bind_models is called
           dns_encoder # TODO(ja): unit test that new_encoder_with_updated_index is called before bind_models
 
+          existing_stemcells = current_deployment.stemcells.map { |s| { os: s.operating_system, version: s.version } }.uniq
+          plan_stemcells = deployment_plan.stemcells.values.map { |s| { os: s.os || s.name, version: s.version } }.uniq
+
+          all_stemcell_versions_changed = (existing_stemcells.intersection(plan_stemcells)).empty?
+
           DeploymentPlan::Assembler.create(deployment_plan, @variables_interpolator).bind_models(
             is_deploy_action: deploy_action?,
             should_bind_new_variable_set: deploy_action?,
+            stemcell_change: @options['force_latest_variables'] || all_stemcell_versions_changed
           )
         end
       end

src/bosh-director/lib/bosh/director/models/deployment.rb

@@ -104,6 +104,10 @@ def current_variable_set
       variable_sets_dataset.order(Sequel.desc(:created_at)).limit(1).first
     end
 
+    def previous_variable_set
+      variable_sets_dataset.order(Sequel.desc(:created_at)).limit(2, 1).first
+    end
+
     def last_successful_variable_set
       variable_sets_dataset.where(deployed_successfully: true).order(Sequel.desc(:created_at)).limit(1).first
     end

src/bosh-director/spec/unit/api/controllers/deployments_controller_spec.rb

@@ -421,6 +421,17 @@ def manifest_with_errand(deployment_name='errand')
               post '/', spec_asset('test_manifest.yml'), { 'CONTENT_TYPE' => 'text/yaml' }
             end
           end
+
+          context 'with the "force_latest_variables" param which is needed because their BOSH DNS certs expire tomorrow and there is no new stemcell to trigger a cert rotation' do
+            it 'passes the parameter' do
+              expect_any_instance_of(DeploymentManager)
+                .to receive(:create_deployment)
+                      .with(anything(), anything(), anything(), anything(), anything(), hash_including('force_latest_variables' => true), anything())
+                      .and_return(OpenStruct.new(:id => 1))
+              post '/?force_latest_variables=true', spec_asset('test_conf.yaml'), {'CONTENT_TYPE' => 'text/yaml'}
+              expect(last_response).to be_redirect
+            end
+          end
         end
 
         describe 'deleting deployment' do