Merge branch 'refactor_backend' of https://github.com/devin-c/hiera-e…

…yaml into devin-c-refactor_backend

features/puppet.feature

@@ -18,7 +18,7 @@ Feature: eyaml hiera integration
     Given I set FACTER_fact to "not-existcity"
     When I run `rm -f /tmp/eyaml_puppettest.* 2>/dev/null`
 
-    When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet/manifests/init.pp`
+    When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet-hiera-merge/manifests/init.pp`
     Then the file "/tmp/eyaml_puppettest.1" should match /^good night$/
     Then the file "/tmp/eyaml_puppettest.2" should match /^great to see you$/
     Then the file "/tmp/eyaml_puppettest.3" should match /good luck/
@@ -29,7 +29,7 @@ Feature: eyaml hiera integration
     Given I set FACTER_fact to "city"
     When I run `rm -f /tmp/eyaml_puppettest.* 2>/dev/null`
 
-    When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet/manifests/init.pp`
+    When I run `puppet apply --confdir ./puppet-hiera-merge --node_name_value localhost puppet-hiera-merge/manifests/init.pp`
     Then the file "/tmp/eyaml_puppettest.1" should match /^rise and shine$/
     Then the file "/tmp/eyaml_puppettest.2" should match /^break a leg$/
     Then the file "/tmp/eyaml_puppettest.3" should match /it'll be alright on the night/

hiera-eyaml.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency('trollop', '>=2.0')
-  gem.add_dependency('highline', '>=1.6.19')
+  gem.add_dependency('hiera', '>= 1.2.1')
+  gem.add_dependency('trollop', '>= 2.0')
+  gem.add_dependency('highline', '>= 1.6.19')
 end

lib/hiera/backend/eyaml.rb

@@ -15,7 +15,7 @@ def self.subcommand= command
       def self.subcommand
         @@subcommand
       end
-      
+
       def self.default_encryption_scheme= new_encryption
         @@default_encryption_scheme = new_encryption
       end
@@ -41,7 +41,7 @@ def self.subcommands= commands
       def self.subcommands
         @@subcommands
       end
-      
+
     end
   end
 end

lib/hiera/backend/eyaml_backend.rb

@@ -2,123 +2,134 @@
 require 'hiera/backend/eyaml/utils'
 require 'hiera/backend/eyaml/options'
 require 'hiera/backend/eyaml/parser/parser'
+require 'hiera/filecache'
+
 require 'yaml'
 
 class Hiera
   module Backend
     class Eyaml_backend
 
-      def initialize
-        @extension = Config[:eyaml][:extension] ? Config[:eyaml][:extension] : "eyaml"
+      attr_reader :extension
+
+      def initialize(cache = nil)
+        debug("Hiera eYAML backend starting")
+
+        @cache     = cache || Filecache.new
+        @extension = Config[:eyaml][:extension] || "eyaml"
       end
 
       def lookup(key, scope, order_override, resolution_type)
-
-        debug("Lookup called for key #{key}")
         answer = nil
 
-        Backend.datasources(scope, order_override) do |source|
-          eyaml_file = Backend.datafile(:eyaml, scope, source, @extension) || next
+        parse_options(scope)
 
-          debug("Processing datasource: #{eyaml_file}")
+        debug("Looking up #{key} in eYAML backend")
 
-          data = YAML.load(File.read( eyaml_file ))
+        Backend.datasources(scope, order_override) do |source|
+          debug("Looking for data source #{source}")
+          eyaml_file = Backend.datafile(:eyaml, scope, source, extension) || next
 
-          next if data.nil? or data.empty?
-          debug ("Data contains valid YAML")
+          next unless File.exists?(eyaml_file)
 
-          next unless data.include?(key)
-          debug ("Key #{key} found in YAML document")
-
-          parsed_answer = parse_answer(key, data[key], scope)
-
-          begin
-            case resolution_type
-            when :array
-              debug("Appending answer array")
-              raise Exception, "Hiera type mismatch: expected Array and got #{parsed_answer.class}" unless parsed_answer.kind_of? Array or parsed_answer.kind_of? String
-              answer ||= []
-              answer << parsed_answer
-            when :hash
-              debug("Merging answer hash")
-              raise Exception, "Hiera type mismatch: expected Hash and got #{parsed_answer.class}" unless parsed_answer.kind_of? Hash
-              answer ||= {}
-              answer = Backend.merge_answer(parsed_answer,answer)
-            else
-              debug("Assigning answer variable")
-              answer = parsed_answer
-              break
-            end
-          rescue NoMethodError
-            raise Exception, "Resolution type is #{resolution_type} but parsed_answer is a #{parsed_answer.class}"
+          data = @cache.read(eyaml_file, Hash) do |data|
+            YAML.load(data) || {}
           end
-        end
 
-        answer
-      end
+          next if data.empty?
+          next unless data.include?(key)
 
-      def parse_answer(key, data, scope, extra_data={})
-        if data.is_a?(Numeric) or data.is_a?(TrueClass) or data.is_a?(FalseClass)
-          # Can't be encrypted
-          data
-        elsif data.is_a?(String)
-          parsed_string = Backend.parse_string(data, scope)
-          decrypt(key, parsed_string, scope)
-        elsif data.is_a?(Hash)
-          answer = {}
-          data.each_pair do |key, val|
-            answer[key] = parse_answer(key, val, scope, extra_data)
+          # Extra logging that we found the key. This can be outputted
+          # multiple times if the resolution type is array or hash but that
+          # should be expected as the logging will then tell the user ALL the
+          # places where the key is found.
+          debug("Found #{key} in #{source}")
+
+          # for array resolution we just append to the array whatever
+          # we find, we then goes onto the next file and keep adding to
+          # the array
+          #
+          # for priority searches we break after the first found data item
+          new_answer = parse_answer(data[key], scope)
+          case resolution_type
+          when :array
+            raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String
+            answer ||= []
+            answer << new_answer
+          when :hash
+            raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash
+            answer ||= {}
+            answer = Backend.merge_answer(new_answer,answer)
+          else
+            answer = new_answer
+            break
           end
-          answer
-        elsif data.is_a?(Array)
-          answer = []
-          data.each do |item|
-            answer << parse_answer(key, item, scope, extra_data)
-          end
-          answer
         end
-      end
 
-      def deblock block_string
-        block_string.gsub(/[ \n]/, '')
+        return answer
       end
 
-      def decrypt(key, value, scope)
-
-        if encrypted? value
+      private
 
-          debug "Attempting to decrypt: #{key}"
-
-          Config[:eyaml].each do |config_key, config_value|
-            config_value = Backend.parse_string(Config[:eyaml][config_key], scope)
-            debug "Setting: #{config_key} = #{config_value}"
-            Eyaml::Options[config_key] = config_value
-          end
+      def debug(message)
+        Hiera.debug("[eyaml_backend]: #{message}")
+      end
 
-          Eyaml::Options[:source] = "hiera"
+      def decrypt(data)
+        if encrypted?(data)
+          debug("Attempting to decrypt")
 
           parser = Eyaml::Parser::ParserFactory.hiera_backend_parser
-          tokens = parser.parse(value)
+          tokens = parser.parse(data)
           decrypted = tokens.map{ |token| token.to_plain_text }
           plaintext = decrypted.join
 
           plaintext.chomp
-
         else
-          value
+          data
         end
       end
 
-      def encrypted?(value)
-        if value.match(/.*ENC\[.*?\]/) then true else false end
+      def encrypted?(data)
+        /.*ENC\[.*?\]/ =~ data ? true : false
+      end
+
+      def parse_answer(data, scope, extra_data={})
+        if data.is_a?(Numeric) or data.is_a?(TrueClass) or data.is_a?(FalseClass)
+          return data
+        elsif data.is_a?(String)
+          return parse_string(data, scope, extra_data)
+        elsif data.is_a?(Hash)
+          answer = {}
+          data.each_pair do |key, val|
+            interpolated_key = Backend.parse_string(key, scope, extra_data)
+            answer[interpolated_key] = parse_answer(val, scope, extra_data)
+          end
+
+          return answer
+        elsif data.is_a?(Array)
+          answer = []
+          data.each do |item|
+            answer << parse_answer(item, scope, extra_data)
+          end
+
+          return answer
+        end
       end
 
-      def debug(msg)
-        Hiera.debug("[eyaml_backend]: #{msg}")
+      def parse_options(scope)
+        Config[:eyaml].each do |key, value|
+          parsed_value = Backend.parse_string(value, scope)
+          Eyaml::Options[key] = parsed_value
+          debug("Set option: #{key} = #{parsed_value}")
+        end
+
+        Eyaml::Options[:source] = "hiera"
       end
 
-      def warn(msg)
-        Hiera.warn("[eyaml_backend]:  #{msg}")
+      def parse_string(data, scope, extra_data={})
+        decrypted_data = decrypt(data)
+        Backend.parse_string(decrypted_data, scope, extra_data)
       end
     end
   end