Merge pull request #24 from ITV/eyaml_support

Eyaml support

.rubocop.yml

@@ -9,3 +9,6 @@ LineLength:
   Description: 'Limit lines to a more generous 120 characters.'
   Enabled: true
   Max: 120
+
+Metrics/MethodLength:
+  Max: 15

CHANGELOG.md

@@ -1,3 +1,26 @@
+# 3.1.0
+
+Added hiera-eyaml support.
+
+This allows us to use encrypted Terraform variables via hiera lookups (the `hiera.yaml` is consumed).
+
+It also allows us to decrypt and extract SSL certificates or SSH keys which can then be used as appropriate.
+
+In order to utilise these two improvements, you must update your `itv.yaml` e.g.:
+
+```
+dome:
+  hiera_keys:
+    artifactory_password: 'deirdre::artifactory_password'
+  certs:
+    sit.phoenix.itv.com.pem: 'phoenix::sit_wildcard_cert'
+    phoenix.key: 'phoenix::certificate_key'
+```
+
+This release also containes:
+- Improved debugging/output messages.
+- More tests.
+
 # 3.0.1
 
 Forcibly unsetting environment variables `AWS_ACCESS_KEY` and `AWS_SECRET_KEY`.

dome.gemspec

@@ -25,4 +25,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'trollop', '~> 2.1'
   spec.add_dependency 'aws-sdk', '~> 2.1'
   spec.add_dependency 'colorize', '~> 0.7'
+  spec.add_dependency 'hiera', '~> 1.3'
+  spec.add_dependency 'hiera-eyaml', '~> 2.1'
 end

lib/dome.rb

@@ -3,10 +3,13 @@
 require 'colorize'
 require 'fileutils'
 require 'yaml'
+require 'hiera'
 
 require 'dome/settings'
 require 'dome/version'
 require 'dome/helpers/shell'
 require 'dome/environment'
 require 'dome/state'
 require 'dome/terraform'
+require 'dome/hiera_lookup'
+require 'dome/secrets'

lib/dome/environment.rb

@@ -1,10 +1,11 @@
 module Dome
   class Environment
-    attr_reader :environment, :account, :settings
+    attr_reader :environment, :account, :ecosystem, :settings
 
     def initialize(directories = Dir.pwd.split('/'))
       @environment = directories[-1]
       @account     = directories[-2]
+      @ecosystem   = directories[-2].split('-')[-1]
       @settings    = Dome::Settings.new
     end
 
@@ -60,7 +61,6 @@ def invalid_environment_message
 
     private
 
-    # rubocop:disable Metrics/MethodLength
     # rubocop:disable Metrics/AbcSize
     def generic_error_message
       puts "The 'account' and 'environment' variables are assigned based on your current directory.\n".colorize(:red)

lib/dome/hiera_lookup.rb

@@ -0,0 +1,91 @@
+module Dome
+  class HieraLookup
+    def initialize(environment)
+      @environment = environment.environment
+      @account     = environment.account
+      @ecosystem   = environment.ecosystem
+      @settings    = Dome::Settings.new
+    end
+
+    def config
+      @config ||= YAML.load_file(File.join(puppet_dir, 'hiera.yaml')).merge(default_config)
+    end
+
+    def default_config
+      {
+        logger: 'noop',
+        yaml: {
+          datadir: "#{puppet_dir}/hieradata"
+        },
+        eyaml: {
+          datadir: "#{puppet_dir}/hieradata",
+          pkcs7_private_key: eyaml_private_key,
+          pkcs7_public_key: eyaml_public_key
+        }
+      }
+    end
+
+    def puppet_dir
+      directory = File.join(@settings.project_root, 'puppet')
+      puts "The configured Puppet directory is: #{directory.colorize(:green)}" unless @directory
+      @directory ||= directory
+    end
+
+    def eyaml_private_key
+      private_key = File.join(puppet_dir, 'keys/private_key.pkcs7.pem')
+      raise "Cannot find eyaml private key! Make sure it exists at #{private_key}" unless File.exist?(private_key)
+      puts "Found eyaml private key: #{private_key.colorize(:green)}"
+      private_key
+    end
+
+    def eyaml_public_key
+      public_key = File.join(puppet_dir, 'keys/public_key.pkcs7.pem')
+      raise "Cannot find eyaml public key! Make sure it exists at #{public_key}" unless File.exist?(public_key)
+      puts "Found eyaml public key: #{public_key.colorize(:green)}"
+      public_key
+    end
+
+    def lookup(key, default = nil, order_override = nil, resolution_type = :priority)
+      hiera = Hiera.new(config: config)
+
+      hiera_scope = {}
+      hiera_scope['ecosystem'] = @ecosystem
+      hiera_scope['location']  = 'aeuw1'
+      hiera_scope['env']       = @environment
+
+      hiera.lookup(key.to_s, default, hiera_scope, order_override, resolution_type)
+    end
+
+    def secret_env_vars(secret_vars)
+      secret_vars.each_pair do |key, val|
+        hiera_lookup = lookup(val)
+        terraform_env_var = "TF_VAR_#{key}"
+        ENV[terraform_env_var] = hiera_lookup
+        if hiera_lookup
+          puts "Setting #{terraform_env_var.colorize(:green)}."
+        else
+          puts "Hiera lookup failed for '#{val}', so #{terraform_env_var} was not set.".colorize(:red)
+        end
+      end
+    end
+
+    def extract_certs(certs)
+      create_certificate_directory
+
+      certs.each_pair do |key, val|
+        directory = "#{certificate_directory}/#{key}"
+        puts "Extracting certificate #{key.colorize(:green)} into #{directory.colorize(:green)}"
+        File.open(directory, 'w') { |f| f.write(lookup(val)) }
+      end
+    end
+
+    def create_certificate_directory
+      puts "Creating certificate directory at #{certificate_directory.colorize(:green)}"
+      FileUtils.mkdir_p certificate_directory
+    end
+
+    def certificate_directory
+      "#{@settings.project_root}/terraform/certs"
+    end
+  end
+end

lib/dome/secrets.rb

@@ -0,0 +1,38 @@
+module Dome
+  class Secrets
+    attr_reader :settings, :hiera
+
+    def initialize(environment)
+      @environment = environment
+      @settings    = Dome::Settings.new
+      @hiera       = Dome::HieraLookup.new(@environment)
+    end
+
+    def secret_env_vars
+      return if dome_config.nil? || hiera_keys_config.nil?
+      @hiera.secret_env_vars(hiera_keys_config)
+    end
+
+    def extract_certs
+      return if dome_config.nil? || certs_config.nil?
+      @hiera.extract_certs(certs_config)
+    end
+
+    def dome_config
+      puts "No #{'dome'.colorize(:green)} key found in your itv.yaml." unless @settings.parse['dome']
+      @settings.parse['dome']
+    end
+
+    def hiera_keys_config
+      puts "No #{'hiera_keys'.colorize(:green)} sub-key under #{'dome'.colorize(:green)} key found "\
+      'in your itv.yaml.' unless @settings.parse['dome']['hiera_keys']
+      @settings.parse['dome']['hiera_keys']
+    end
+
+    def certs_config
+      puts "No #{'certs'.colorize(:green)} sub-key under #{'dome'.colorize(:green)} key found "\
+      'in your itv.yaml.' unless @settings.parse['dome']['certs']
+      @settings.parse['dome']['certs']
+    end
+  end
+end

lib/dome/settings.rb

@@ -12,5 +12,9 @@ def load_yaml
     def itv_yaml_path
       '../../../itv.yaml'
     end
+
+    def project_root
+      File.realpath(File.dirname(itv_yaml_path))
+    end
   end
 end

lib/dome/terraform.rb

@@ -6,6 +6,7 @@ class Terraform
 
     def initialize
       @environment = Dome::Environment.new
+      @secrets     = Dome::Secrets.new(@environment)
       @state       = Dome::State.new(@environment)
       @plan_file   = "plans/#{@environment.account}-#{@environment.environment}-plan.tf"
     end
@@ -45,12 +46,15 @@ def plan
     end
 
     def apply
+      @secrets.secret_env_vars
       command         = "terraform apply #{@plan_file}"
       failure_message = 'something went wrong when applying the TF plan'
       execute_command(command, failure_message)
     end
 
     def create_plan
+      @secrets.secret_env_vars
+      @secrets.extract_certs
       command         = "terraform plan -refresh=true -out=#{@plan_file} -var-file=params/env.tfvars"
       failure_message = 'something went wrong when creating the TF plan'
       execute_command(command, failure_message)

lib/dome/version.rb

@@ -1,3 +1,3 @@
 module Dome
-  VERSION = '3.0.1'.freeze
+  VERSION = '3.1.0'.freeze
 end

spec/hiera_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Dome do
+  let(:account_dir) { 'deirdre-dev' }
+  let(:environment_dir) { 'qa' }
+  let(:environment) { Dome::Environment.new([account_dir, environment_dir]) }
+  let(:hiera) { Dome::HieraLookup.new(environment) }
+
+  it 'outputs the correct message for a hiera lookup' do
+    vars = { 'foo' => 'bar' }
+    allow(hiera).to receive(:lookup).and_return('bar')
+    error_message = "Setting \e[0;32;49mTF_VAR_foo\e[0m.\n"
+    expect { hiera.secret_env_vars(vars) }.to output(error_message).to_stdout
+  end
+
+  it 'outputs the correct error message for a failed hiera lookup' do
+    vars = { 'foo' => 'bar' }
+    allow(hiera).to receive(:lookup).and_return(nil)
+    error_message = "\e[0;31;49mHiera lookup failed for 'bar', so TF_VAR_foo was not set.\e[0m\n"
+    expect { hiera.secret_env_vars(vars) }.to output(error_message).to_stdout
+  end
+end

spec/secrets_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+
+describe Dome do
+  let(:account_dir) { 'deirdre-dev' }
+  let(:environment_dir) { 'qa' }
+  let(:environment) { Dome::Environment.new([account_dir, environment_dir]) }
+  let(:secrets) { Dome::Secrets.new(environment) }
+
+  # to prevent a validation error
+  let(:itv_yaml_path) { 'spec/fixtures/itv.yaml' }
+  before(:each) { allow(secrets.settings).to receive(:itv_yaml_path) { itv_yaml_path } }
+
+  context 'if config is missing from itv.yaml' do
+    context 'outputs a debug message to STDOUT' do
+      it 'when missing the parent key dome' do
+        allow(secrets.settings).to receive(:load_yaml).and_return({})
+        expect { secrets.dome_config }.to output.to_stdout
+      end
+
+      it 'when missing the sub-key hiera_keys' do
+        yaml = { 'dome' => { 'foo' => 'bar' } }
+        allow(secrets.settings).to receive(:load_yaml).and_return(yaml)
+        expect { secrets.hiera_keys_config }.to output.to_stdout
+      end
+
+      it 'when missing the sub-key certs' do
+        yaml = { 'dome' => { 'foo' => 'bar' } }
+        allow(secrets.settings).to receive(:load_yaml).and_return(yaml)
+        expect { secrets.certs_config }.to output.to_stdout
+      end
+    end
+
+    it 'does not set secret environment variables' do
+      allow(secrets.settings).to receive(:load_yaml).and_return({})
+      expect(secrets.hiera).not_to receive(:secret_env_vars)
+      secrets.secret_env_vars
+    end
+
+    it 'does not extract certificates' do
+      allow(secrets.settings).to receive(:load_yaml).and_return({})
+      expect(secrets.hiera).not_to receive(:extract_certs)
+      secrets.extract_certs
+    end
+
+    xit 'only warns you once about missing parent key dome when performing a plan' do
+      # TODO
+    end
+  end
+
+  context 'with valid config' do
+    it 'sets secret environment variables' do
+      yaml = { 'dome' => { 'hiera_keys' => { 'artifactory_password' => 'artifactory::root-readonly::password' } } }
+      allow(secrets.settings).to receive(:load_yaml).and_return(yaml)
+      expect(secrets.hiera).to receive(:secret_env_vars)
+      secrets.secret_env_vars
+    end
+
+    it 'extracts certificates' do
+      yaml = { 'dome' => { 'certs' => { 'id_rsa' => 'aws::ssh_privkey_content' } } }
+      allow(secrets.settings).to receive(:load_yaml).and_return(yaml)
+      expect(secrets.hiera).to receive(:extract_certs)
+      secrets.extract_certs
+    end
+  end
+end