Merge pull request #19 from balexx/master

Fixes #11214 - support node attribute whitelisting, blacklisting and caching

README.md

@@ -22,6 +22,13 @@ require 'chef_handler_foreman'
 foreman_server_options  :url => 'http://your.server/foreman'
 # add following line if you want to upload node attributes (facts in Foreman language)
 foreman_facts_upload    true
+## Facts whitelist / blacklisting
+# add following line if you want to upload only specific node attributes - only top-level attributes
+foreman_facts_whitelist ['lsb','network','cpu']
+# add following line if you want to avoid uploading specific node attributes - any part from the key will do
+foreman_facts_blacklist ['kernel','counters','interfaces::sit0']
+# enable caching of attributes - (full) upload will be performed only if attributes changed
+foreman_facts_cache_file '/var/cache/chef_foreman_cache.md5'
 # add following line if you want to upload reports
 foreman_reports_upload  true
 # add following line to manage reports verbosity. Allowed values are debug, notice and error
@@ -41,3 +48,16 @@ chef. The configuration line will look like this:
 ```ruby
 foreman_reports_upload  true, 2
 ```
+
+### Caching of facts
+
+Note that some attributes, such as network counters or used memory, change on every chef-client run.
+For caching to work, you would need to blacklist such attributes, otherwise facts will be uploaded
+on every run.
+
+## Facts whitelisting / blacklisting
+
+Cherry picking which facts to upload, coupled with caching, allows to scale the solution to many
+thousands of nodes. Note, however, that some attributes are expected by Foreman to exist, and thus
+should not be blacklisted. The whitelist and blacklist examples above include a minimal set of
+attributes known to work in a large scale production environment.

lib/chef_handler_foreman/foreman_facts.rb

@@ -12,15 +12,21 @@
 #along with this program.  If not, see <http://www.gnu.org/licenses/>
 
 require 'chef/handler'
+require 'digest/md5'
 
 module ChefHandlerForeman
   class ForemanFacts < Chef::Handler
     attr_accessor :uploader
+    attr_accessor :blacklist
+    attr_accessor :whitelist
+    attr_accessor :cache_file
+    attr_accessor :cache_expired
 
     def report
       send_attributes(prepare_facts)
     end
 
+
     private
 
     def prepare_facts
@@ -55,7 +61,10 @@ def normalize(os)
     def plain_attributes
       # chef 10 attributes can be access by to_hash directly, chef 11 uses attributes method
       attributes = node.respond_to?(:attributes) ? node.attributes : node.to_hash
-      plainify(attributes.to_hash).flatten.inject(&:merge)
+      attributes = attributes.select { |key, value| @whitelist.include?(key) } if @whitelist
+      attrs = plainify(attributes.to_hash).flatten.inject(&:merge)
+      verify_checksum(attrs) if @cache_file
+      attrs
     end
 
     def plainify(hash, prefix = nil)
@@ -67,8 +76,11 @@ def plainify(hash, prefix = nil)
           result.push plainify(array_to_hash(value), get_key(key, prefix))
         else
           new                       = {}
-          new[get_key(key, prefix)] = value
-          result.push new
+          full_key = get_key(key, prefix)
+          if @blacklist.nil? || !@blacklist.any? { |black_key| full_key.include?(black_key) }
+            new[full_key] = value
+            result.push new
+          end
         end
       end
       result
@@ -85,11 +97,31 @@ def get_key(key, prefix)
     end
 
     def send_attributes(attributes)
-      if uploader
-        uploader.foreman_request('/api/hosts/facts', attributes, node.name)
+      if @cache_file and !@cache_expired
+        Chef::Log.info "No attributes have changed - not uploading to foreman" 
       else
-        Chef::Log.error "No uploader registered for foreman facts, skipping facts upload"
+        if uploader
+          uploader.foreman_request('/api/hosts/facts', attributes, node.name)
+        else
+          Chef::Log.error "No uploader registered for foreman facts, skipping facts upload"
+        end
+      end
+    end
+
+    def verify_checksum(attributes)
+      @cache_expired = true
+      attrs_checksum = Digest::MD5.hexdigest(attributes.to_s)
+      if File.exist?(@cache_file)
+        contents = File.read(@cache_file)
+        if attrs_checksum == contents
+          @cache_expired = false
+        end
       end
+      File.open(@cache_file, 'w') { |f| f.write(attrs_checksum) }
+      rescue => e
+        @cache_expired = true
+        Chef::Log.info "unable to verify cache checksum - #{e.message}, facts will be sent"
+        Chef::Log.debug e.backtrace.join("\n")
     end
   end
 end

lib/chef_handler_foreman/foreman_hooks.rb

@@ -52,6 +52,24 @@ def foreman_reports_upload(upload, mode = 1)
       end
     end
 
+    def foreman_facts_blacklist(blacklist)
+      if @foreman_facts_handler
+        @foreman_facts_handler.blacklist = blacklist
+      end
+    end
+
+    def foreman_facts_whitelist(whitelist)
+      if @foreman_facts_handler
+        @foreman_facts_handler.whitelist = whitelist
+      end
+    end
+
+    def foreman_facts_cache_file(cache_file)
+      if @foreman_facts_handler
+        @foreman_facts_handler.cache_file = cache_file
+      end
+    end
+
     # level can be string error notice debug
     def reports_log_level(level)
       raise ArgumentError, 'unknown level: ' + level.to_s unless %w(error notice debug).include?(level)