Merge pull request #19 from crayfishx/pr/18

Cache feature originally submitted as #18

README.md

@@ -16,6 +16,7 @@ The following is an example hiera.yaml configuration for use with hiera-http
       :host: 127.0.0.1
       :port: 5984
       :output: json
+      :cache_timeout: 10
       :failure: graceful
       :paths:
         - /configuration/%{fqdn}
@@ -31,6 +32,10 @@ The following are optional configuration parameters
 
 `:http_read_timeout: ` : Timeout in seconds for waiting for a HTTP response (default 10)
 
+`:cache_timeout: ` : Timeout in seconds for HTTP requests to a same path (default 10)
+
+`:cache_clean_interval: ` : Interval (in secs) to clean the cache (default 3600), set to 0 to disable cache cleaning 
+
 `:failure: ` : When set to `graceful` will stop hiera-http from throwing an exception in the event of a connection error, timeout or invalid HTTP response and move on.  Without this option set hiera-http will throw an exception in such circumstances
 
 `:ignore_404: ` : If `failure` is _not_ set to `graceful` then any error code received from the HTTP response will throw an exception.  This option makes 404 responses exempt from exceptions.  This is useful if you expect to get 404's for data items not in a certain part of the hierarchy and need to fall back to the next level in the hierarchy, but you still want to bomb out on other errors.

lib/hiera/backend/http_backend.rb

@@ -11,6 +11,10 @@ def initialize
         @http.read_timeout = @config[:http_read_timeout] || 10
         @http.open_timeout = @config[:http_connect_timeout] || 10
 
+        @cache = {}
+        @cache_timeout = @config[:cache_timeout] || 10
+        @cache_clean_interval = @config[:cache_clean_interval] || 3600
+
         if @config[:use_ssl]
           @http.use_ssl = true
 
@@ -34,7 +38,6 @@ def initialize
       end
 
       def lookup(key, scope, order_override, resolution_type)
-
         answer = nil
 
         paths = @config[:paths].map { |p| Backend.parse_string(p, scope, { 'key' => key }) }
@@ -44,30 +47,9 @@ def lookup(key, scope, order_override, resolution_type)
         paths.each do |path|
 
           Hiera.debug("[hiera-http]: Lookup #{key} from #{@config[:host]}:#{@config[:port]}#{path}")
-          httpreq = Net::HTTP::Get.new(path)
-
-          if @config[:use_auth]
-              httpreq.basic_auth @config[:auth_user], @config[:auth_pass]
-          end
-
-          begin
-            httpres = @http.request(httpreq)
-          rescue Exception => e
-            Hiera.warn("[hiera-http]: Net::HTTP threw exception #{e.message}")
-            raise Exception, e.message unless @config[:failure] == 'graceful'
-            next
-          end
-
-          unless httpres.kind_of?(Net::HTTPSuccess)
-            Hiera.debug("[hiera-http]: bad http response from #{@config[:host]}:#{@config[:port]}#{path}")
-            Hiera.debug("HTTP response code was #{httpres.code}")
-            unless ( httpres.code == '404' && @config[:ignore_404] == true )
-              raise Exception, 'Bad HTTP response' unless @config[:failure] == 'graceful'
-            end
-            next
-          end
 
-          result = self.parse_response(key, httpres.body)
+          result = http_get_and_parse_with_cache(path)
+          result = result[key] if result.is_a?(Hash)
           next unless result
 
           parsed_result = Backend.parse_answer(result, scope)
@@ -88,50 +70,94 @@ def lookup(key, scope, order_override, resolution_type)
       end
 
 
-      def parse_response(key,answer)
+      private
 
-        return nil unless answer
+      def parse_response(answer)
+        return unless answer
 
-        Hiera.debug("[hiera-http]: Query returned data, parsing response as #{@config[:output] || 'plain'}")
+        format = @config[:output] || 'plain'
+        Hiera.debug("[hiera-http]: Query returned data, parsing response as #{format}")
 
-        case @config[:output]
-
-        when 'plain'
-          # When the output format is configured as plain we assume that if the
-          # endpoint URL returns an HTTP success then the contents of the response
-          # body is the value itself, or nil.
-          #
-          answer
+        case format
         when 'json'
-          # If JSON is specified as the output format, assume the output of the
-          # endpoint URL is a JSON document and return keypart that matched our
-          # lookup key
-          self.json_handler(key,answer)
+          parse_json answer
         when 'yaml'
-          # If YAML is specified as the output format, assume the output of the
-          # endpoint URL is a YAML document and return keypart that matched our
-          # lookup key
-          self.yaml_handler(key,answer)
+          parse_yaml answer
         else
           answer
         end
       end
 
       # Handlers
       # Here we define specific handlers to parse the output of the http request
-      # and return a value.  Currently we support YAML and JSON
+      # and return its structured representation.  Currently we support YAML and JSON
       #
-      def json_handler(key,answer)
+      def parse_json(answer)
         require 'rubygems'
         require 'json'
-        JSON.parse(answer)[key]
+        JSON.parse(answer)
       end
 
-      def yaml_handler(answer)
+      def parse_yaml(answer)
         require 'yaml'
-        YAML.parse(answer)[key]
+        YAML.parse(answer)
+      end
+
+      def http_get_and_parse_with_cache(path)
+        return http_get(path) if @cache_timeout <= 0
+
+        now = Time.now.to_i
+        expired_at = now + @cache_timeout
+
+        # Deleting all stale cache entries can be expensive. Do not do it every time
+        periodically_clean_cache(now, expired_at) unless @cache_clean_interval == 0
+
+        # Just refresh the entry being requested for performance
+        unless @cache[path] && @cache[path][:created_at] < expired_at
+          @cache[path] = {
+            :created_at => now,
+            :result => http_get_and_parse(path)
+          }
+        end
+        @cache[path][:result]
       end
 
+      def http_get_and_parse(path)
+        httpreq = Net::HTTP::Get.new(path)
+
+        if @config[:use_auth]
+          httpreq.basic_auth @config[:auth_user], @config[:auth_pass]
+        end
+
+        begin
+          httpres = @http.request(httpreq)
+        rescue Exception => e
+          Hiera.warn("[hiera-http]: Net::HTTP threw exception #{e.message}")
+          raise Exception, e.message unless @config[:failure] == 'graceful'
+          return
+        end
+
+        unless httpres.kind_of?(Net::HTTPSuccess)
+          Hiera.debug("[hiera-http]: bad http response from #{@config[:host]}:#{@config[:port]}#{path}")
+          Hiera.debug("HTTP response code was #{httpres.code}")
+          unless httpres.code == '404' && @config[:ignore_404]
+            raise Exception, 'Bad HTTP response' unless @config[:failure] == 'graceful'
+          end
+          return
+        end
+
+        parse_response httpres.body
+      end
+
+
+      def periodically_clean_cache(now, expired_at)
+        return if now < @clean_cache_at.to_i
+
+        @clean_cache_at = now + @cache_clean_interval
+        @cache.select! do |_, entry|
+          entry[:created_at] < expired_at
+        end
+      end
     end
   end
 end