Re-factor the backend implementation

This is an attempt to get deep_merge's working, which are currently
throwing exceptons in my testing environment. See [this
issue](#65).

I copied the yaml_backend from hiera v1.2.1, which is the one used in
puppet v3.4.2, and injected the decryption mechanics. I also needed to
copy the `parse_answer` method from the `::Hiera::Backend` and modified
it to use the decryption aware version of `parse_string`. It's still an
unfortunately long implementation considering that the only real
difference between the eyaml backend and the yaml backend is the change
to `::Heira::Backend.parse_string` that allows it to process encrypted
values.

I feel like this backend could be trimmed down so that it inherits from
the yaml backend, but the issues with doing that are tricky enough to
solve that I'm not going to pursue it at this point.

features/puppet.feature

@@ -18,7 +18,7 @@ Feature: eyaml hiera integration
     Given I set FACTER_fact to "not-existcity"
     When I run `rm -f /tmp/eyaml_puppettest.* 2>/dev/null`
 
-    When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet/manifests/init.pp`
+    When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet-hiera-merge/manifests/init.pp`
     Then the file "/tmp/eyaml_puppettest.1" should match /^good night$/
     Then the file "/tmp/eyaml_puppettest.2" should match /^great to see you$/
     Then the file "/tmp/eyaml_puppettest.3" should match /good luck/
@@ -29,7 +29,7 @@ Feature: eyaml hiera integration
     Given I set FACTER_fact to "city"
     When I run `rm -f /tmp/eyaml_puppettest.* 2>/dev/null`
 
-    When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet/manifests/init.pp`
+    When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet-hiera-merge/manifests/init.pp`
     Then the file "/tmp/eyaml_puppettest.1" should match /^rise and shine$/
     Then the file "/tmp/eyaml_puppettest.2" should match /^break a leg$/
     Then the file "/tmp/eyaml_puppettest.3" should match /it'll be alright on the night/

hiera-eyaml.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency('trollop', '>=2.0')
-  gem.add_dependency('highline', '>=1.6.19')
+  gem.add_dependency('hiera', '>= 1.2.1')
+  gem.add_dependency('trollop', '>= 2.0')
+  gem.add_dependency('highline', '>= 1.6.19')
 end

lib/hiera/backend/eyaml.rb

@@ -15,7 +15,7 @@ def self.subcommand= command
       def self.subcommand
         @@subcommand
       end
-      
+
       def self.default_encryption_scheme= new_encryption
         @@default_encryption_scheme = new_encryption
       end
@@ -41,7 +41,7 @@ def self.subcommands= commands
       def self.subcommands
         @@subcommands
       end
-      
+
     end
   end
 end

lib/hiera/backend/eyaml_backend.rb

@@ -2,123 +2,130 @@
 require 'hiera/backend/eyaml/utils'
 require 'hiera/backend/eyaml/options'
 require 'hiera/backend/eyaml/parser/parser'
+require 'hiera/filecache'
+
 require 'yaml'
 
 class Hiera
   module Backend
     class Eyaml_backend
 
-      def initialize
-        @extension = Config[:eyaml][:extension] ? Config[:eyaml][:extension] : "eyaml"
+      attr_reader :extension
+
+      def initialize(cache = nil)
+        Hiera.debug("Hiera eYAML backend starting")
+
+        @cache     = cache || Filecache.new
+        @extension = Config[:eyaml][:extension] || "eyaml"
       end
 
       def lookup(key, scope, order_override, resolution_type)
-
-        debug("Lookup called for key #{key}")
         answer = nil
 
-        Backend.datasources(scope, order_override) do |source|
-          eyaml_file = Backend.datafile(:eyaml, scope, source, @extension) || next
+        parse_options(scope)
 
-          debug("Processing datasource: #{eyaml_file}")
+        Hiera.debug("Looking up #{key} in eYAML backend")
 
-          data = YAML.load(File.read( eyaml_file ))
+        Backend.datasources(scope, order_override) do |source|
+          Hiera.debug("Looking for data source #{source}")
+          eyaml_file = Backend.datafile(:eyaml, scope, source, extension) || next
 
-          next if data.nil? or data.empty?
-          debug ("Data contains valid YAML")
+          next unless File.exists?(eyaml_file)
 
-          next unless data.include?(key)
-          debug ("Key #{key} found in YAML document")
-
-          parsed_answer = parse_answer(key, data[key], scope)
-
-          begin
-            case resolution_type
-            when :array
-              debug("Appending answer array")
-              raise Exception, "Hiera type mismatch: expected Array and got #{parsed_answer.class}" unless parsed_answer.kind_of? Array or parsed_answer.kind_of? String
-              answer ||= []
-              answer << parsed_answer
-            when :hash
-              debug("Merging answer hash")
-              raise Exception, "Hiera type mismatch: expected Hash and got #{parsed_answer.class}" unless parsed_answer.kind_of? Hash
-              answer ||= {}
-              answer = Backend.merge_answer(parsed_answer,answer)
-            else
-              debug("Assigning answer variable")
-              answer = parsed_answer
-              break
-            end
-          rescue NoMethodError
-            raise Exception, "Resolution type is #{resolution_type} but parsed_answer is a #{parsed_answer.class}"
+          data = @cache.read(eyaml_file, Hash) do |data|
+            YAML.load(data) || {}
           end
-        end
 
-        answer
-      end
+          next if data.empty?
+          next unless data.include?(key)
 
-      def parse_answer(key, data, scope, extra_data={})
-        if data.is_a?(Numeric) or data.is_a?(TrueClass) or data.is_a?(FalseClass)
-          # Can't be encrypted
-          data
-        elsif data.is_a?(String)
-          parsed_string = Backend.parse_string(data, scope)
-          decrypt(key, parsed_string, scope)
-        elsif data.is_a?(Hash)
-          answer = {}
-          data.each_pair do |key, val|
-            answer[key] = parse_answer(key, val, scope, extra_data)
+          # Extra logging that we found the key. This can be outputted
+          # multiple times if the resolution type is array or hash but that
+          # should be expected as the logging will then tell the user ALL the
+          # places where the key is found.
+          Hiera.debug("Found #{key} in #{source}")
+
+          # for array resolution we just append to the array whatever
+          # we find, we then goes onto the next file and keep adding to
+          # the array
+          #
+          # for priority searches we break after the first found data item
+          new_answer = parse_answer(data[key], scope)
+          case resolution_type
+          when :array
+            raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
+            answer ||= []
+            answer << new_answer
+          when :hash
+            raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
+            answer ||= {}
+            answer = Backend.merge_answer(new_answer,answer)
+          else
+            answer = new_answer
+            break
           end
-          answer
-        elsif data.is_a?(Array)
-          answer = []
-          data.each do |item|
-            answer << parse_answer(key, item, scope, extra_data)
-          end
-          answer
         end
-      end
 
-      def deblock block_string
-        block_string.gsub(/[ \n]/, '')
+        return answer
       end
 
-      def decrypt(key, value, scope)
-
-        if encrypted? value
+      private
 
-          debug "Attempting to decrypt: #{key}"
-
-          Config[:eyaml].each do |config_key, config_value|
-            config_value = Backend.parse_string(Config[:eyaml][config_key], scope)
-            debug "Setting: #{config_key} = #{config_value}"
-            Eyaml::Options[config_key] = config_value
-          end
-
-          Eyaml::Options[:source] = "hiera"
+      def decrypt(data)
+        if encrypted?(data)
+          Hiera.debug("Attempting to decrypt")
 
           parser = Eyaml::Parser::ParserFactory.hiera_backend_parser
-          tokens = parser.parse(value)
+          tokens = parser.parse(data)
           decrypted = tokens.map{ |token| token.to_plain_text }
           plaintext = decrypted.join
 
           plaintext.chomp
-
         else
-          value
+          data
         end
       end
 
-      def encrypted?(value)
-        if value.match(/.*ENC\[.*?\]/) then true else false end
+      def encrypted?(data)
+        /.*ENC\[.*?\]/ =~ data ? true : false
+      end
+
+      def parse_answer(data, scope, extra_data={})
+        if data.is_a?(Numeric) or data.is_a?(TrueClass) or data.is_a?(FalseClass)
+          return data
+        elsif data.is_a?(String)
+          return parse_string(data, scope, extra_data)
+        elsif data.is_a?(Hash)
+          answer = {}
+          data.each_pair do |key, val|
+            interpolated_key = Backend.parse_string(key, scope, extra_data)
+            answer[interpolated_key] = parse_answer(val, scope, extra_data)
+          end
+
+          return answer
+        elsif data.is_a?(Array)
+          answer = []
+          data.each do |item|
+            answer << parse_answer(item, scope, extra_data)
+          end
+
+          return answer
+        end
       end
 
-      def debug(msg)
-        Hiera.debug("[eyaml_backend]: #{msg}")
+      def parse_options(scope)
+        Config[:eyaml].each do |key, value|
+          parsed_value = Backend.parse_string(value, scope)
+          Eyaml::Options[key] = parsed_value
+          Hiera.debug("Set option: #{key} = #{parsed_value}")
+        end
+
+        Eyaml::Options[:source] = "hiera"
       end
 
-      def warn(msg)
-        Hiera.warn("[eyaml_backend]:  #{msg}")
+      def parse_string(data, scope, extra_data={})
+        decrypted_data = decrypt(data)
+        Backend.parse_string(decrypted_data, scope, extra_data)
       end
     end
   end