From cf3ac023ad0ca895adf11af42f62eee0f87c601e Mon Sep 17 00:00:00 2001 From: Devin Christensen Date: Sat, 15 Feb 2014 13:23:03 -0700 Subject: [PATCH] Re-factor the backend implementation This is an attempt to get deep_merge's working, which are currently throwing exceptons in my testing environment. See [this issue](https://github.com/TomPoulton/hiera-eyaml/issues/65). I copied the yaml_backend from hiera v1.2.1, which is the one used in puppet v3.4.2, and injected the decryption mechanics. I also needed to copy the `parse_answer` method from the `::Hiera::Backend` and modified it to use the decryption aware version of `parse_string`. It's still an unfortunately long implementation considering that the only real difference between the eyaml backend and the yaml backend is the change to `::Heira::Backend.parse_string` that allows it to process encrypted values. I feel like this backend could be trimmed down so that it inherits from the yaml backend, but the issues with doing that are tricky enough to solve that I'm not going to pursue it at this point. --- features/puppet.feature | 4 +- hiera-eyaml.gemspec | 5 +- lib/hiera/backend/eyaml.rb | 4 +- lib/hiera/backend/eyaml_backend.rb | 165 +++++++++++++++-------------- 4 files changed, 93 insertions(+), 85 deletions(-) diff --git a/features/puppet.feature b/features/puppet.feature index 7379f968..69135ba3 100644 --- a/features/puppet.feature +++ b/features/puppet.feature @@ -18,7 +18,7 @@ Feature: eyaml hiera integration Given I set FACTER_fact to "not-existcity" When I run `rm -f /tmp/eyaml_puppettest.* 2>/dev/null` - When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet/manifests/init.pp` + When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet-hiera-merge/manifests/init.pp` Then the file "/tmp/eyaml_puppettest.1" should match /^good night$/ Then the file "/tmp/eyaml_puppettest.2" should match /^great to see you$/ Then the file "/tmp/eyaml_puppettest.3" should match /good luck/ @@ -29,7 +29,7 @@ Feature: eyaml hiera integration Given I set FACTER_fact to "city" When I run `rm -f /tmp/eyaml_puppettest.* 2>/dev/null` - When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet/manifests/init.pp` + When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet-hiera-merge/manifests/init.pp` Then the file "/tmp/eyaml_puppettest.1" should match /^rise and shine$/ Then the file "/tmp/eyaml_puppettest.2" should match /^break a leg$/ Then the file "/tmp/eyaml_puppettest.3" should match /it'll be alright on the night/ diff --git a/hiera-eyaml.gemspec b/hiera-eyaml.gemspec index 767cf7cb..0411c1d7 100644 --- a/hiera-eyaml.gemspec +++ b/hiera-eyaml.gemspec @@ -17,6 +17,7 @@ Gem::Specification.new do |gem| gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.require_paths = ["lib"] - gem.add_dependency('trollop', '>=2.0') - gem.add_dependency('highline', '>=1.6.19') + gem.add_dependency('hiera', '>= 1.2.1') + gem.add_dependency('trollop', '>= 2.0') + gem.add_dependency('highline', '>= 1.6.19') end diff --git a/lib/hiera/backend/eyaml.rb b/lib/hiera/backend/eyaml.rb index df2956c7..10273936 100644 --- a/lib/hiera/backend/eyaml.rb +++ b/lib/hiera/backend/eyaml.rb @@ -15,7 +15,7 @@ def self.subcommand= command def self.subcommand @@subcommand end - + def self.default_encryption_scheme= new_encryption @@default_encryption_scheme = new_encryption end @@ -41,7 +41,7 @@ def self.subcommands= commands def self.subcommands @@subcommands end - + end end end diff --git a/lib/hiera/backend/eyaml_backend.rb b/lib/hiera/backend/eyaml_backend.rb index d6651734..9b58ece8 100644 --- a/lib/hiera/backend/eyaml_backend.rb +++ b/lib/hiera/backend/eyaml_backend.rb @@ -2,123 +2,130 @@ require 'hiera/backend/eyaml/utils' require 'hiera/backend/eyaml/options' require 'hiera/backend/eyaml/parser/parser' +require 'hiera/filecache' + require 'yaml' class Hiera module Backend class Eyaml_backend - def initialize - @extension = Config[:eyaml][:extension] ? Config[:eyaml][:extension] : "eyaml" + attr_reader :extension + + def initialize(cache = nil) + Hiera.debug("Hiera eYAML backend starting") + + @cache = cache || Filecache.new + @extension = Config[:eyaml][:extension] || "eyaml" end def lookup(key, scope, order_override, resolution_type) - - debug("Lookup called for key #{key}") answer = nil - Backend.datasources(scope, order_override) do |source| - eyaml_file = Backend.datafile(:eyaml, scope, source, @extension) || next + parse_options(scope) - debug("Processing datasource: #{eyaml_file}") + Hiera.debug("Looking up #{key} in eYAML backend") - data = YAML.load(File.read( eyaml_file )) + Backend.datasources(scope, order_override) do |source| + Hiera.debug("Looking for data source #{source}") + eyaml_file = Backend.datafile(:eyaml, scope, source, extension) || next - next if data.nil? or data.empty? - debug ("Data contains valid YAML") + next unless File.exists?(eyaml_file) - next unless data.include?(key) - debug ("Key #{key} found in YAML document") - - parsed_answer = parse_answer(key, data[key], scope) - - begin - case resolution_type - when :array - debug("Appending answer array") - raise Exception, "Hiera type mismatch: expected Array and got #{parsed_answer.class}" unless parsed_answer.kind_of? Array or parsed_answer.kind_of? String - answer ||= [] - answer << parsed_answer - when :hash - debug("Merging answer hash") - raise Exception, "Hiera type mismatch: expected Hash and got #{parsed_answer.class}" unless parsed_answer.kind_of? Hash - answer ||= {} - answer = Backend.merge_answer(parsed_answer,answer) - else - debug("Assigning answer variable") - answer = parsed_answer - break - end - rescue NoMethodError - raise Exception, "Resolution type is #{resolution_type} but parsed_answer is a #{parsed_answer.class}" + data = @cache.read(eyaml_file, Hash) do |data| + YAML.load(data) || {} end - end - answer - end + next if data.empty? + next unless data.include?(key) - def parse_answer(key, data, scope, extra_data={}) - if data.is_a?(Numeric) or data.is_a?(TrueClass) or data.is_a?(FalseClass) - # Can't be encrypted - data - elsif data.is_a?(String) - parsed_string = Backend.parse_string(data, scope) - decrypt(key, parsed_string, scope) - elsif data.is_a?(Hash) - answer = {} - data.each_pair do |key, val| - answer[key] = parse_answer(key, val, scope, extra_data) + # Extra logging that we found the key. This can be outputted + # multiple times if the resolution type is array or hash but that + # should be expected as the logging will then tell the user ALL the + # places where the key is found. + Hiera.debug("Found #{key} in #{source}") + + # for array resolution we just append to the array whatever + # we find, we then goes onto the next file and keep adding to + # the array + # + # for priority searches we break after the first found data item + new_answer = parse_answer(data[key], scope) + case resolution_type + when :array + raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String + answer ||= [] + answer << new_answer + when :hash + raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash + answer ||= {} + answer = Backend.merge_answer(new_answer,answer) + else + answer = new_answer + break end - answer - elsif data.is_a?(Array) - answer = [] - data.each do |item| - answer << parse_answer(key, item, scope, extra_data) - end - answer end - end - def deblock block_string - block_string.gsub(/[ \n]/, '') + return answer end - def decrypt(key, value, scope) - - if encrypted? value + private - debug "Attempting to decrypt: #{key}" - - Config[:eyaml].each do |config_key, config_value| - config_value = Backend.parse_string(Config[:eyaml][config_key], scope) - debug "Setting: #{config_key} = #{config_value}" - Eyaml::Options[config_key] = config_value - end - - Eyaml::Options[:source] = "hiera" + def decrypt(data) + if encrypted?(data) + Hiera.debug("Attempting to decrypt") parser = Eyaml::Parser::ParserFactory.hiera_backend_parser - tokens = parser.parse(value) + tokens = parser.parse(data) decrypted = tokens.map{ |token| token.to_plain_text } plaintext = decrypted.join plaintext.chomp - else - value + data end end - def encrypted?(value) - if value.match(/.*ENC\[.*?\]/) then true else false end + def encrypted?(data) + /.*ENC\[.*?\]/ =~ data ? true : false + end + + def parse_answer(data, scope, extra_data={}) + if data.is_a?(Numeric) or data.is_a?(TrueClass) or data.is_a?(FalseClass) + return data + elsif data.is_a?(String) + return parse_string(data, scope, extra_data) + elsif data.is_a?(Hash) + answer = {} + data.each_pair do |key, val| + interpolated_key = Backend.parse_string(key, scope, extra_data) + answer[interpolated_key] = parse_answer(val, scope, extra_data) + end + + return answer + elsif data.is_a?(Array) + answer = [] + data.each do |item| + answer << parse_answer(item, scope, extra_data) + end + + return answer + end end - def debug(msg) - Hiera.debug("[eyaml_backend]: #{msg}") + def parse_options(scope) + Config[:eyaml].each do |key, value| + parsed_value = Backend.parse_string(value, scope) + Eyaml::Options[key] = parsed_value + Hiera.debug("Set option: #{key} = #{parsed_value}") + end + + Eyaml::Options[:source] = "hiera" end - def warn(msg) - Hiera.warn("[eyaml_backend]: #{msg}") + def parse_string(data, scope, extra_data={}) + decrypted_data = decrypt(data) + Backend.parse_string(decrypted_data, scope, extra_data) end end end