feat: Short Lived NATS credentials

When bootstraping a new VM, BOSH stores credentials in the metadata
services of the different IAASes. This includes the certificates the VM
uses to access NATS. This is a security concern for some environments
where a malicious user that can access the metadata service gets the
NATS credentials. He will be able to connect to NATS and listen to the
traffic between the the VMs and the director.

This feature removes that risk by making the credentials stored in
the metadata service ephemeral/short lived, so if a user gets access
to them, they won't be useful after the VM bootstrap. These credentials
will be used only while starting a new VM, and after that, they will be
replaced by permanent credentials that are not stored in the metadata
service.

This feature is enabled by default. To disable it a new parameter
called `enable_short_lived_nats_credentials` should be added with false as
its value in the director manifest during the create-env.
This feature will affect only newly created/recreated VMs.

When an installation is using this feature its a MUST to ensure that the
stemcells used in the director are compatible with it; stemcells that
are incompatible will result in unresponsive VMs.
The oldest compatible stemcell versions are:
Windows 2019 - 2019.41
Ubuntu Xenial - 621.171
Ubuntu Bionic - 1.36
Ubuntu Jammy - All versions are suported.

[#183316690] [Deadline: Before End of FYQ1] [AHA] Short-lived NATS bootstrap credential

Co-authored-by: Daniel Felipe Ochoa <danielfelipo@vmware.com>
Co-authored-by: Brian Upton <bupton@vmware.com>
Co-authored-by: Brian Cunnie <bcunnie@vmware.com>

jobs/director/spec

@@ -144,6 +144,9 @@ properties:
   director.enable_nats_delivered_templates:
     description: When true, rendered templates will be sent over NATs
     default: false
+  director.enable_short_lived_nats_bootstrap_credentials:
+    description: When true, NATS bootstrap credentials will be short lived on new VMs
+    default: true
   director.generate_vm_passwords:
     description: When true, a random unique password will be used for each vm if user has not specified a password
     default: false

jobs/director/templates/director.yml.erb

@@ -27,6 +27,7 @@ params = {
   },
   'max_vm_create_tries' => p('director.max_vm_create_tries'),
   'enable_nats_delivered_templates' => p('director.enable_nats_delivered_templates'),
+  'enable_short_lived_nats_bootstrap_credentials' => p('director.enable_short_lived_nats_bootstrap_credentials', true),
   'enable_cpi_resize_disk' => p('director.enable_cpi_resize_disk'),
   'generate_vm_passwords' => p('director.generate_vm_passwords'),
   'remove_dev_tools' => p('director.remove_dev_tools'),

src/bosh-director/db/migrations/director/20230103143246_add_permanent_nats_credentials_to_vms.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  up do
+    alter_table(:vms) do
+      add_column :permanent_nats_credentials, 'boolean', null: false, default: false
+    end
+  end
+end

src/bosh-director/db/migrations/migration_digests.json

@@ -159,5 +159,6 @@
   "20190318234554_add_criteria_columns_to_local_dns_aliases": "8f36cbe5cf9759ad6b903e8c08dc0cf388260cef",
   "20190325095716_remove_resurrection_paused": "546c855e7d2ee00a4f9867ab1d819847781a06de",
   "20190327222054_scale_dns_blob_version": "2460bacc06eae7368d9322a97bfe781a10c59d3f",
-  "20210902232124_add_blobstore_and_nats_shas_to_vms": "34aaaf22c8e5074a96b2666f1bd30a2f41652e24"
-}
+  "20210902232124_add_blobstore_and_nats_shas_to_vms": "34aaaf22c8e5074a96b2666f1bd30a2f41652e24",
+  "20230103143246_add_permanent_nats_credentials_to_vms": "d2752f1e16bddf57f882e898e172a72f481c87b7"
+}

src/bosh-director/lib/bosh/director/api/controllers/deployments_controller.rb

@@ -595,7 +595,7 @@ def create_vms_response(vms_instances_hash)
         results = []
         vms_instances_hash.each_pair do |instance, vms|
           vms.each do |vm|
-            results << create_vm_response(instance, vm).merge('active' => vm.active)
+            results << create_vm_response(instance, vm).merge('active' => vm.active).merge('permanent_nats_credentials' => vm.permanent_nats_credentials)
           end
         end
         results

src/bosh-director/lib/bosh/director/config.rb

@@ -26,6 +26,7 @@ class << self
         :dns,
         :dns_db,
         :enable_cpi_resize_disk,
+        :enable_short_lived_nats_bootstrap_credentials,
         :enable_post_deploy,
         :enable_snapshots,
         :enable_virtual_delete_vms,
@@ -61,6 +62,7 @@ class << self
         :db_config,
         :director_ips,
         :enable_nats_delivered_templates,
+        :enable_short_lived_nats_bootstrap_credentials,
         :allow_errands_on_stopped_instances,
         :ignore_missing_gateway,
         :director_certificate_expiry_json_path,
@@ -210,6 +212,7 @@ def configure(config, preload_db_classes: true)
         @keep_unreachable_vms = config.fetch('keep_unreachable_vms', false)
         @enable_post_deploy = config.fetch('enable_post_deploy', true)
         @enable_nats_delivered_templates = config.fetch('enable_nats_delivered_templates', false)
+        @enable_short_lived_nats_bootstrap_credentials = config.fetch('enable_short_lived_nats_bootstrap_credentials', false)
         @allow_errands_on_stopped_instances = config.fetch('allow_errands_on_stopped_instances', false)
         @generate_vm_passwords = config.fetch('generate_vm_passwords', false)
         @remove_dev_tools = config['remove_dev_tools']

src/bosh-director/lib/bosh/director/deployment_plan/instance.rb

@@ -178,7 +178,7 @@ def add_state_to_model(state)
         @model.update(spec: @current_state)
       end
 
-      def update_instance_settings(vm)
+      def update_instance_settings(vm, force_nats_rotation = false)
         disk_associations = @model.reload.active_persistent_disks.collection.reject do |disk|
           disk.model.managed?
         end
@@ -201,7 +201,7 @@ def update_instance_settings(vm)
           end
         end
 
-        if nats_config_changed?
+        if nats_config_changed? || force_nats_rotation
           cert_generator = NatsClientCertGenerator.new(@logger)
           agent_cert_key_result = cert_generator.generate_nats_client_certificate "#{vm.agent_id}.agent.bosh-internal"
           settings['mbus'] = {

src/bosh-director/lib/bosh/director/deployment_plan/steps/create_vm_step.rb

@@ -118,9 +118,10 @@ def create(instance, stemcell_cid, cloud_properties, network_settings, disks, en
             env['bosh']['mbus']['cert'] ||= {}
             env['bosh']['mbus']['cert']['ca'] = Config.nats_server_ca
             cert_generator = NatsClientCertGenerator.new(@logger)
-            agent_cert_key_result = cert_generator.generate_nats_client_certificate "#{agent_id}.agent.bosh-internal"
-            env['bosh']['mbus']['cert']['certificate'] = agent_cert_key_result[:cert].to_pem
-            env['bosh']['mbus']['cert']['private_key'] = agent_cert_key_result[:key].to_pem
+            agent_short_lived_creds = cert_generator.generate_nats_client_certificate "#{agent_id}.bootstrap.agent.bosh-internal"
+            env['bosh']['mbus']['cert']['certificate'] = agent_short_lived_creds[:cert].to_pem
+            env['bosh']['mbus']['cert']['private_key'] = agent_short_lived_creds[:key].to_pem
+
           end
 
           password = env.fetch('bosh', {}).fetch('password', "")

src/bosh-director/lib/bosh/director/deployment_plan/steps/update_instance_settings_step.rb

@@ -9,9 +9,10 @@ def initialize(instance)
         def perform(report)
           instance_model = @instance.model.reload
 
-          @instance.update_instance_settings(report.vm)
+          @instance.update_instance_settings(report.vm, Config.enable_short_lived_nats_bootstrap_credentials)
 
           instance_model.update(cloud_properties: JSON.dump(@instance.cloud_properties))
+          report.vm.update(permanent_nats_credentials: Config.enable_short_lived_nats_bootstrap_credentials)
         end
       end
     end

src/bosh-director/spec/unit/api/controllers/deployments_controller_spec.rb

@@ -1386,6 +1386,7 @@ def manifest_with_errand(deployment_name='errand')
                 'az' => {0 => 'az0', 1 => 'az1', nil => nil}[instance_idx],
                 'ips' => ["#{instance_idx}.#{instance_idx}.#{vm_by_instance}.#{vm_by_instance}"],
                 'vm_created_at' => time.utc.iso8601,
+                'permanent_nats_credentials' => false,
               )
             end
           end
@@ -1467,6 +1468,7 @@ def manifest_with_errand(deployment_name='errand')
                   'az' => { 0 => 'az0', 1 => 'az1', nil => nil }[instance_idx],
                   'ips' => ["1.2.#{instance_idx}.#{vm_by_instance}"],
                   'vm_created_at' => time.utc.iso8601,
+                  'permanent_nats_credentials' => false,
                 )
               end
             end
@@ -1516,7 +1518,8 @@ def manifest_with_errand(deployment_name='errand')
                   'active' => vm_is_active,
                   'az' => {0 => 'az0', 1 => 'az1', nil => nil}[i],
                   'ips' => ["1.2.3.#{i}"],
-                  'vm_created_at' => time.utc.iso8601
+                  'vm_created_at' => time.utc.iso8601,
+                  'permanent_nats_credentials' => false,
                 )
               end
             end
@@ -1575,6 +1578,7 @@ def manifest_with_errand(deployment_name='errand')
                 'ips' => [vip, network_spec_ip],
                 'job' => 'job',
                 'vm_created_at' => time.utc.iso8601,
+                'permanent_nats_credentials' => false,
               )
             end
           end

src/bosh-director/spec/unit/config_spec.rb

@@ -351,6 +351,33 @@
       end
     end
 
+    describe 'enable_short_lived_nats_bootstrap_credentials' do
+      it 'defaults to false' do
+        described_class.configure(test_config)
+        expect(described_class.enable_short_lived_nats_bootstrap_credentials).to be_falsey
+      end
+
+      context 'when explicitly set' do
+        context 'when set to true' do
+          before { test_config['enable_short_lived_nats_bootstrap_credentials'] = true }
+
+          it 'resolves to true' do
+            described_class.configure(test_config)
+            expect(described_class.enable_short_lived_nats_bootstrap_credentials).to be_truthy
+          end
+        end
+
+        context 'when set to false' do
+          before { test_config['enable_short_lived_nats_bootstrap_credentials'] = false }
+
+          it 'resolves to false' do
+            described_class.configure(test_config)
+            expect(described_class.enable_short_lived_nats_bootstrap_credentials).to be_falsey
+          end
+        end
+      end
+    end
+
     describe 'allow_errands_on_stopped_instances' do
       it 'defaults to false' do
         described_class.configure(test_config)

src/bosh-director/spec/unit/db/migrations/director/20230103143246_add_permanent_nats_credentials_to_vms_spec.rb

@@ -0,0 +1,96 @@
+require 'db_spec_helper'
+
+module Bosh::Director
+  describe '20230103143246_add_permanent_nats_credentials_to_vms.rb' do
+    subject(:migration) { '20230103143246_add_permanent_nats_credentials_to_vms.rb' }
+    let(:db) { DBSpecHelper.db }
+    let(:created_at_time) { Time.now }
+
+    before do
+      DBSpecHelper.migrate_all_before(subject)
+      db[:deployments] << { id: 1, name: 'fake-deployment-name', manifest: '{}' }
+      db[:variable_sets] << { deployment_id: db[:deployments].first[:id], created_at: Time.now }
+      db[:instances] << {
+        id: 1,
+        job: 'fake-instance-group',
+        uuid: 'uuid1',
+        index: 1,
+        deployment_id: 1,
+        state: 'started',
+        availability_zone: 'az1',
+        variable_set_id: 1,
+        spec_json: '{}',
+      }
+      attrs = {
+        id: 1,
+        instance_id: 1,
+        agent_id: 'fake-agent-id-1',
+        cid: 'fake-vm-cid-1',
+        env_json: 'fake-env-json',
+        trusted_certs_sha1: 'fake-trusted-certs-sha1',
+      }
+      db[:vms] << attrs
+
+      DBSpecHelper.migrate(subject)
+    end
+
+    it 'should add permanent_nats_credentials to the vms table' do
+      expect(db[:vms].columns).to include(:permanent_nats_credentials)
+    end
+
+    it 'should migrate existing vms records to permanent_nats_credentials equals false' do
+      expect(db[:vms].where(id: 1).first[:permanent_nats_credentials]).to eq(false)
+    end
+
+    it 'should add new vm records with permanent_nats_credentials equals false' do
+      db[:instances] << {
+        id: 2,
+        job: 'fake-instance-group',
+        uuid: 'uuid2',
+        index: 1,
+        deployment_id: 1,
+        state: 'started',
+        availability_zone: 'az1',
+        variable_set_id: 1,
+        spec_json: '{}',
+      }
+      attrs = {
+        id: 2,
+        instance_id: 2,
+        agent_id: 'fake-agent-id-2',
+        cid: 'fake-vm-cid-2',
+        env_json: 'fake-env-json',
+        trusted_certs_sha1: 'fake-trusted-certs-sha1',
+      }
+      db[:vms] << attrs
+
+      expect(db[:vms].where(id: 2).first[:permanent_nats_credentials]).to eq(false)
+    end
+
+    it 'should add new vm records with permanent_nats_credentials equals true.' do
+      db[:instances] << {
+        id: 2,
+        job: 'fake-instance-group',
+        uuid: 'uuid2',
+        index: 1,
+        deployment_id: 1,
+        state: 'started',
+        availability_zone: 'az1',
+        variable_set_id: 1,
+        spec_json: '{}',
+      }
+      attrs = {
+        id: 2,
+        instance_id: 2,
+        agent_id: 'fake-agent-id-2',
+        cid: 'fake-vm-cid-2',
+        env_json: 'fake-env-json',
+        trusted_certs_sha1: 'fake-trusted-certs-sha1',
+        permanent_nats_credentials: true,
+      }
+      db[:vms] << attrs
+
+      expect(db[:vms].where(id: 2).first[:permanent_nats_credentials]).to eq(true)
+    end
+  end
+end

src/bosh-director/spec/unit/db/migrations/director/add_data_migration_spec.rb

@@ -9,7 +9,7 @@ module Bosh::Director
       # populated with data. This test will fail every time a new migration script is added. Change
       # the file name below to the latest when a test is added.
       # Look at tests in this directory for similar examples: bosh-director/spec/unit/db/migrations/director
-      expect(latest_db_migration_file).to eq('20210902232124_add_blobstore_and_nats_shas_to_vms.rb')
+      expect(latest_db_migration_file).to eq('20230103143246_add_permanent_nats_credentials_to_vms.rb')
     end
   end
 end

src/bosh-director/spec/unit/deployment_plan/compilation_instance_pool_spec.rb

@@ -166,6 +166,10 @@ module Bosh::Director
       allow(deployment_plan).to receive(:network).with('a').and_return(network)
       allow(instance_deleter).to receive(:delete_instance_plan)
       allow(Config).to receive(:current_job).and_return(update_job)
+      allow(Config).to receive(:enable_short_lived_nats_bootstrap_credentials).and_return(true)
+      director_config = SpecHelper.spec_get_director_config
+      allow(Config).to receive(:nats_client_ca_private_key_path).and_return(director_config['nats']['client_ca_private_key_path'])
+      allow(Config).to receive(:nats_client_ca_certificate_path).and_return(director_config['nats']['client_ca_certificate_path'])
       allow(deployment_model).to receive(:current_variable_set).and_return(Models::VariableSet.make)
       allow(MetadataUpdater).to receive(:new).and_return(metadata_updater)
       allow(metadata_updater).to receive(:update_vm_metadata)

src/bosh-director/spec/unit/deployment_plan/instance_spec.rb

@@ -636,6 +636,33 @@ module Bosh::Director::DeploymentPlan
 
           instance.update_instance_settings(vm)
         end
+
+        context 'when forcing the rotation of nats credentials' do
+          before do
+            allow(Bosh::Director::Config).to receive(:nats_config_fingerprint).and_return('new-nats-sha')
+            allow(Bosh::Director::NatsClientCertGenerator).to receive(:new).and_return(cert_generator)
+
+            allow(cert_generator).to receive(:generate_nats_client_certificate).with(
+              /^#{vm.agent_id}\.agent\.bosh-internal/,
+            ).and_return(
+              cert: double(to_pem: 'new nats cert'),
+              key: double(to_pem: 'new nats key'),
+            )
+            allow(agent_client).to receive(:update_settings)
+          end
+
+          it 'should include the nats config' do
+            expect(agent_client).to receive(:update_settings).with(hash_including('mbus' => {
+              'cert' => {
+                'ca' => Bosh::Director::Config.nats_server_ca,
+                'certificate' => 'new nats cert',
+                'private_key' => 'new nats key',
+              }
+            }))
+
+            instance.update_instance_settings(vm, true)
+          end
+        end
       end
     end
 

src/bosh-director/spec/unit/deployment_plan/stages/package_compile_stage_spec.rb

@@ -144,6 +144,10 @@ module Bosh::Director
       allow(Config).to receive(:current_job).and_return(update_job)
       allow(Config).to receive(:name).and_return('fake-director-name')
       allow(Config).to receive(:cloud_options).and_return('provider' => { 'path' => '/path/to/default/cpi' })
+      allow(Config).to receive(:enable_short_lived_nats_bootstrap_credentials).and_return(true)
+      director_config = SpecHelper.spec_get_director_config
+      allow(Config).to receive(:nats_client_ca_private_key_path).and_return(director_config['nats']['client_ca_private_key_path'])
+      allow(Config).to receive(:nats_client_ca_certificate_path).and_return(director_config['nats']['client_ca_certificate_path'])
       allow(Bosh::Director::Config).to receive(:event_log).and_return(event_log)
       allow(cloud).to receive(:info)
       allow(cloud).to receive(:request_cpi_api_version=)

src/bosh-director/spec/unit/deployment_plan/stages/update_stage_functional_spec.rb

@@ -72,6 +72,10 @@ module Bosh::Director::DeploymentPlan::Stages
       allow(Bosh::Director::Config).to receive(:uuid).and_return('meow-uuid')
       allow(Bosh::Director::Config).to receive(:cloud_options).and_return('provider' => { 'path' => '/path/to/default/cpi' })
       allow(Bosh::Director::Config).to receive(:preferred_cpi_api_version).and_return(1)
+      allow(Bosh::Director::Config).to receive(:enable_short_lived_nats_bootstrap_credentials).and_return(true)
+      director_config = SpecHelper.spec_get_director_config
+      allow(Bosh::Director::Config).to receive(:nats_client_ca_private_key_path).and_return(director_config['nats']['client_ca_private_key_path'])
+      allow(Bosh::Director::Config).to receive(:nats_client_ca_certificate_path).and_return(director_config['nats']['client_ca_certificate_path'])
       allow(Bosh::Clouds::ExternalCpiResponseWrapper).to receive(:new).with(anything, anything).and_return(cloud)
       allow(variables_interpolator).to receive(:interpolate_template_spec_properties).and_return({})
       allow(variables_interpolator).to receive(:interpolated_versioned_variables_changed?).and_return(false)

src/bosh-director/spec/unit/deployment_plan/steps/create_vm_step_spec.rb

@@ -596,7 +596,7 @@ module Steps
                   allow(NatsClientCertGenerator).to receive(:new).and_return(cert_generator)
 
                   expect(cert_generator).to receive(:generate_nats_client_certificate).with(
-                    /^([0-9a-f\-]*)\.agent\.bosh-internal/,
+                    /^([0-9a-f\-]*)\.bootstrap\.agent\.bosh-internal/,
                   ).and_return(
                     cert: cert,
                     key: private_key,