multi CPI support (#52)

* multi cpi: allowing openstack properties to by overwritten by context (refactored cpi lambda to test this)

* multi-cpi: supporting ca_cert for context properties

* multi-cpi: separating writing ca file to disk from setting it in options

* multi-cpi: next try to make cert handling cleaner

* multi-cpi: bosh_cpi now merges cpi properties in context so we can use context['openstack']

* multi-cpi: don't just overwrite openstack properties but merge, so that defaults from bosh release spec are respected

* multi-cpi: no need to check for context.nil? since it's always provided as not-nil by bosh

* multi-cpi: updating upstream bosh gems to contain context passing

* multi-cpi: vendored updated upstream bosh gems

* multi-cpi: cpi properties are no longer in openstack subkey of context, but in toplevel

* Fix context cacert handling

Vcap user is not allowed to write to
`/var/vcap/jobs/openstack_cpi/config/cacert_context.pem`.
Instead of using this fixed path, a tmpdir is created for each cpi
call. The lambda writes the cacert into this tmpdir. The tmpdir is
deleted after the cpi call has finished.

[#134693605](https://www.pivotaltracker.com/story/show/134693605)

Signed-off-by: Jan von Loewenstein <jan.von.loewenstein@sap.com>

src/bosh_openstack_cpi/Gemfile.lock

@@ -2,10 +2,11 @@ GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.4.0)
-    bosh_common (1.3215.3.0)
+    bosh_common (1.3262.24.0)
       logging (~> 1.8.2)
-      semi_semantic (~> 1.1.0)
-    bosh_cpi (1.3215.3.1)
+      semi_semantic (~> 1.2.0)
+    bosh_cpi (2.0.1)
+      bosh_common (~> 1.3262.24.0)
       logging (~> 1.8.2)
       membrane (~> 1.1.0)
     builder (3.2.2)
@@ -53,7 +54,7 @@ GEM
       rspec-support (~> 3.3.0)
     rspec-support (3.3.0)
     safe_yaml (1.0.4)
-    semi_semantic (1.1.0)
+    semi_semantic (1.2.0)
     timecop (0.7.4)
     webmock (2.1.0)
       addressable (>= 2.3.6)

src/bosh_openstack_cpi/bin/openstack_cpi

@@ -14,23 +14,9 @@ cloud_config = OpenStruct.new(:logger => Logger.new(STDERR))
 Bosh::Clouds::Config.configure(cloud_config)
 
 cpi_log = StringIO.new
+Dir.mktmpdir do |dir|
+  cpi_lambda = Bosh::OpenStackCloud::CpiLambda.create(cpi_config, cpi_log, ssl_ca_file, File.join(dir, 'cacert_context.pem'))
+  cli = Bosh::Cpi::Cli.new(cpi_lambda, cpi_log, STDOUT)
 
-cpi_lambda = lambda do
-  unless cpi_config.has_key?('cloud') && cpi_config['cloud'].has_key?('properties')
-    raise "Could not find cloud properties in the configuration"
-  end
-
-  cloud_properties = cpi_config['cloud']['properties']
-  cloud_properties['cpi_log'] = cpi_log
-  connection_options = cloud_properties['openstack']['connection_options']
-  # If 'ca_cert' is set we render non-empty `config/openstack.crt`
-  if connection_options && connection_options['ca_cert']
-      connection_options['ssl_ca_file'] = ssl_ca_file
-      connection_options.delete('ca_cert')
-  end
-  Bosh::Clouds::Openstack.new(cloud_properties)
-end
-
-cli = Bosh::Cpi::Cli.new(cpi_lambda, cpi_log, STDOUT)
-
-cli.run(ARGF.read)
+  cli.run(ARGF.read)
+end

src/bosh_openstack_cpi/lib/cloud/openstack.rb

@@ -19,6 +19,7 @@ module OpenStackCloud; end
 require "cloud"
 require "cloud/openstack/helpers"
 require "cloud/openstack/cloud"
+require "cloud/openstack/cpi_lambda"
 require "cloud/openstack/openstack"
 require "cloud/openstack/tag_manager"
 

src/bosh_openstack_cpi/lib/cloud/openstack/cpi_lambda.rb

@@ -0,0 +1,32 @@
+module Bosh::OpenStackCloud
+  class CpiLambda
+    def self.create(cpi_config, cpi_log, ca_cert_from_config, ca_cert_from_context)
+      lambda do |context|
+        unless cpi_config.has_key?('cloud') && cpi_config['cloud'].has_key?('properties')
+          raise 'Could not find cloud properties in the configuration'
+        end
+
+        cloud_properties = cpi_config['cloud']['properties']
+        cloud_properties['cpi_log'] = cpi_log
+
+        # If 'ca_cert' is set in job config we render non-empty `config/openstack.crt` (excon needs it as a file)
+        connection_options = cloud_properties['openstack']['connection_options']
+        if connection_options && connection_options.delete('ca_cert')
+          connection_options['ssl_ca_file'] = ca_cert_from_config
+        end
+
+        # allow openstack config to be overwritten dynamically by context
+        cloud_properties['openstack'].merge!(context)
+
+        # write ca cert to disk if given in context
+        connection_options = cloud_properties['openstack']['connection_options']
+        if connection_options && (ca_cert = connection_options.delete('ca_cert'))
+          File.write(ca_cert_from_context, ca_cert)
+          connection_options['ssl_ca_file'] = ca_cert_from_context
+        end
+
+        Bosh::Clouds::Openstack.new(cloud_properties)
+      end
+    end
+  end
+end

src/bosh_openstack_cpi/spec/unit/cpi_lambda_spec.rb

@@ -0,0 +1,90 @@
+require "spec_helper"
+
+describe Bosh::OpenStackCloud::CpiLambda do
+  subject { described_class.create(cpi_config, cpi_log, ssl_ca_file, ca_cert_from_context) }
+  let(:cpi_config) {
+    {
+        'cloud' => {
+            'properties' => {
+                'openstack' => {
+                    'key1' => 'value1',
+                    'key2' => 'value2'
+                }
+            }
+        }
+    }
+  }
+  let(:ssl_ca_file) { 'feel-free-to-change' }
+  let(:cpi_log) { StringIO.new }
+  let(:ca_cert_from_context) { Tempfile.new('ca_cert').path }
+
+  describe 'when creating a cloud' do
+    it 'passes parts of the cpi config to openstack' do
+      expect(Bosh::Clouds::Openstack).to receive(:new).with({'openstack' => cpi_config['cloud']['properties']['openstack'],
+                                                             'cpi_log' => cpi_log})
+      subject.call({})
+    end
+
+    context 'if invalid cpi config is given' do
+      let(:cpi_config) {{'empty' => 'config'}}
+
+      it 'raises an error' do
+        expect {
+          subject.call({})
+        }.to raise_error /Could not find cloud properties in the configuration/
+      end
+    end
+
+    context 'if using ca_certs in config' do
+      let(:cpi_config) {{ 'cloud' => {'properties' => { 'openstack' => {'connection_options' => {'ca_cert' => 'xyz'}}}}}}
+
+      it 'sets ssl_ca_file that is passed and removes ca_certs' do
+        expect(Bosh::Clouds::Openstack).to receive(:new).with({'openstack' => {'connection_options' => {'ssl_ca_file' => ssl_ca_file}},
+                                                               'cpi_log' => cpi_log})
+        subject.call({})
+      end
+    end
+
+    context 'if openstack properties are provided in the context' do
+      it 'merges the openstack properties' do
+        context = {
+            'newkey' => 'newvalue',
+            'newkey2' => 'newvalue2',
+        }
+
+        expect(Bosh::Clouds::Openstack).to receive(:new).with({'openstack' => { 'key1' => 'value1',
+                                                                                'key2' => 'value2',
+                                                                                'newkey' => 'newvalue',
+                                                                                'newkey2' => 'newvalue2'},
+                                                               'cpi_log' => cpi_log})
+        subject.call(context)
+      end
+
+      it 'writes the given ca_cert to the disk and sets ssl_ca_file to its path' do
+        context = {
+            'newkey' => 'newvalue',
+            'connection_options' => {'ca_cert' => 'xyz'}
+        }
+
+        expect(Bosh::Clouds::Openstack).to receive(:new).with({'openstack' => { 'newkey' => 'newvalue',
+                                                                                'key1' => 'value1',
+                                                                                'key2' => 'value2',
+                                                                                'connection_options' => {'ssl_ca_file' => ca_cert_from_context}},
+                                                               'cpi_log' => cpi_log})
+
+        subject.call(context)
+        expect(File.read(ca_cert_from_context)).to eq('xyz')
+      end
+
+      context 'when the context does not include a ca_cert' do
+        it 'does not write into the file' do
+          allow(Bosh::Clouds::Openstack).to receive(:new)
+
+          subject.call({})
+
+          expect(File.read(ca_cert_from_context)).to eq('')
+        end
+      end
+    end
+  end
+end