(FACT-2486) Add facts cache

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config --exclude-limit 1000`
-# on 2020-04-15 10:09:42 +0300 using RuboCop version 0.74.0.
+# on 2020-04-23 16:16:39 +0300 using RuboCop version 0.74.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -71,8 +71,8 @@ RSpec/FilePath:
     - 'spec/facter/resolvers/windows/system32_resolver_spec.rb'
     - 'spec/facter/resolvers/windows/virtualization_resolver_spec.rb'
     - 'spec/facter/resolvers/windows/win_os_description_resolver_spec.rb'
-    - 'spec/framework/config/block_list_spec.rb'
     - 'spec/framework/config/config_reader_spec.rb'
+    - 'spec/framework/config/fact_groups_spec.rb'
     - 'spec/framework/core/fact/external/external_fact_manager_spec.rb'
     - 'spec/framework/core/fact/internal/internal_fact_manager_spec.rb'
     - 'spec/framework/core/fact_loaders/class_discoverer_spec.rb'
@@ -190,7 +190,7 @@ RSpec/SubjectStub:
     - 'spec/custom_facts/util/fact_spec.rb'
     - 'spec/custom_facts/util/resolution_spec.rb'
 
-# Offense count: 179
+# Offense count: 176
 # Configuration parameters: IgnoreNameless, IgnoreSymbolicNames.
 RSpec/VerifiedDoubles:
   Exclude:
@@ -247,7 +247,6 @@ RSpec/VerifiedDoubles:
     - 'spec/facter/resolvers/windows/uptime_resolver_spec.rb'
     - 'spec/facter/resolvers/windows/virtualization_resolver_spec.rb'
     - 'spec/facter/resolvers/windows/win_os_description_resolver_spec.rb'
-    - 'spec/framework/config/block_list_spec.rb'
     - 'spec/framework/core/fact_loaders/external_fact_loader_spec.rb'
     - 'spec/framework/core/fact_loaders/fact_loader_spec.rb'
     - 'spec/framework/core/fact_manager_spec.rb'

lib/custom_facts/util/config.rb

@@ -36,6 +36,10 @@ def self.external_facts_dirs
         @external_facts_dirs
       end
 
+      def self.facts_cache_dir
+        @facts_cache_dir ||= setup_default_cache_dir
+      end
+
       def self.setup_default_ext_facts_dirs
         if LegacyFacter::Util::Root.root?
           windows_dir = windows_data_dir
@@ -77,8 +81,6 @@ def self.setup_default_cache_dir
                            end
       end
 
-      setup_default_cache_dir
-
       def self.setup_default_override_binary_dir
         @override_binary_dir = if LegacyFacter::Util::Config.windows?
                                  nil

lib/facter.rb

@@ -327,7 +327,7 @@ def error_check(args, resolved_facts)
     #
     # @api private
     def log_blocked_facts
-      block_list = Facter::BlockList.new(Facter::Options[:config]).block_list
+      block_list = Facter::FactGroups.new(Facter::Options[:config]).block_list
       return unless block_list.any? && Facter::Options[:block]
 
       @logger.debug("blocking collection of #{block_list.join("\s")} facts")

lib/framework/cli/cli.rb

@@ -122,13 +122,13 @@ def version
     desc '--list-block-groups', 'List block groups'
     map ['--list-block-groups'] => :list_block_groups
     def list_block_groups(*_args)
-      puts Facter::BlockList.new.block_groups.to_yaml.lines[1..-1].join
+      puts Facter::FactGroups.new.groups.to_yaml.lines[1..-1].join
     end
 
     desc '--list-cache-groups', 'List cache groups'
     map ['--list-cache-groups'] => :list_cache_groups
     def list_cache_groups(*_args)
-      puts Facter::CacheList.instance.cache_groups.to_yaml.lines[1..-1].join
+      puts Facter::FactGroups.new.groups.to_yaml.lines[1..-1].join
     end
 
     def self.exit_on_failure?

lib/framework/config/fact_groups.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Facter
+  class FactGroups
+    attr_reader :groups, :block_list
+
+    @groups_ttls = []
+
+    STRING_TO_SECONDS = { 'seconds' => 1, 'minutes' => 60, 'hours' => 3600, 'days' => 3600 * 24 }.freeze
+
+    def initialize(group_list_path = nil)
+      @groups_file_path = group_list_path || File.join(ROOT_DIR, 'fact_groups.conf')
+      @groups ||= File.readable?(@groups_file_path) ? Hocon.load(@groups_file_path) : {}
+      load_groups
+    end
+
+    # Breakes down blocked groups in blocked facts
+    def blocked_facts
+      fact_list = []
+
+      @block_list.each do |group_name|
+        facts_for_block = @groups[group_name]
+
+        fact_list += facts_for_block || [group_name]
+      end
+
+      fact_list
+    end
+
+    # Get the group name a fact is part of
+    def get_fact_group(fact_name)
+      @groups.detect { |k, v| break k if Array(v).find { |f| fact_name =~ /^#{f}.*/ } }
+    end
+
+    # Get config ttls for a given group
+    def get_group_ttls(group_name)
+      return unless (ttls = @groups_ttls.find { |g| g[group_name] })
+
+      ttls_to_seconds(ttls[group_name])
+    end
+
+    private
+
+    def load_groups
+      config = ConfigReader.init(Options[:config])
+      @block_list = config.block_list || {}
+      @groups_ttls = ConfigReader.init(Options[:config]).ttls || {}
+    end
+
+    def ttls_to_seconds(ttls)
+      duration, unit = ttls.split(' ', 2)
+      duration.to_i * STRING_TO_SECONDS[unit]
+    end
+  end
+end

lib/framework/core/cache_manager.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Facter
+  class CacheManager
+    def initialize
+      @groups = {}
+      @log = Log.new(self)
+      @fact_groups = Facter::FactGroups.new
+    end
+
+    def cache_dir
+      LegacyFacter::Util::Config.facts_cache_dir
+    end
+
+    def resolve_facts(searched_facts)
+      return searched_facts, [] if !File.directory?(cache_dir) || !Options[:cache]
+
+      facts = []
+      searched_facts.each do |fact|
+        res = resolve_fact(fact)
+        facts << res if res
+      end
+      facts.each do |fact|
+        searched_facts.delete_if { |f| f.name == fact.name }
+      end
+      [searched_facts, facts]
+    end
+
+    def cache_facts(resolved_facts)
+      return unless Options[:cache]
+
+      resolved_facts.each do |fact|
+        cache_fact(fact)
+      end
+
+      begin
+        write_cache unless @groups.empty?
+      rescue Errno::EACCES => e
+        @log.warn("Could not write cache: #{e.message}")
+      end
+    end
+
+    private
+
+    def resolve_fact(searched_fact)
+      group_name = @fact_groups.get_fact_group(searched_fact.name)
+      return unless group_name
+
+      return unless group_cached?(group_name)
+
+      return if check_ttls(group_name).zero?
+
+      data = read_group_json(group_name)
+      return unless data
+
+      @log.debug("loading cached values for #{group_name} facts")
+      create_fact(searched_fact, data[searched_fact.name])
+    end
+
+    def create_fact(searched_fact, value)
+      Facter::ResolvedFact.new(searched_fact.name, value, searched_fact.type,
+                               searched_fact.user_query, searched_fact.filter_tokens)
+    end
+
+    def cache_fact(fact)
+      group_name = @fact_groups.get_fact_group(fact.name)
+      return if !group_name || fact.value.nil?
+
+      return unless group_cached?(group_name)
+
+      @groups[group_name] ||= {}
+      @groups[group_name][fact.name] = fact.value
+    end
+
+    def write_cache
+      unless File.directory?(cache_dir)
+        require 'fileutils'
+        FileUtils.mkdir_p(cache_dir)
+      end
+
+      @groups.each do |group_name, data|
+        next if check_ttls(group_name).zero?
+
+        @log.debug("caching values for #{group_name} facts")
+        cache_file_name = File.join(cache_dir, group_name)
+        File.write(cache_file_name, JSON.pretty_generate(data))
+      end
+    end
+
+    def read_group_json(group_name)
+      return @groups[group_name] if @groups.key?(group_name)
+
+      cache_file_name = File.join(cache_dir, group_name)
+      data = nil
+      file = Util::FileHelper.safe_read(cache_file_name)
+      begin
+        data = JSON.parse(file)
+      rescue JSON::ParserError
+        delete_cache(group_name)
+      end
+      @groups[group_name] = data
+    end
+
+    def group_cached?(group_name)
+      cached = @fact_groups.get_group_ttls(group_name) ? true : false
+      delete_cache(group_name) unless cached
+      cached
+    end
+
+    def check_ttls(group_name)
+      ttls = @fact_groups.get_group_ttls(group_name)
+      return 0 unless ttls
+
+      cache_file_name = File.join(cache_dir, group_name)
+      return ttls unless File.readable?(cache_file_name)
+
+      file_time = File.mtime(cache_file_name)
+      expire_date = file_time + ttls
+      if expire_date < Time.now
+        File.delete(cache_file_name)
+        return ttls
+      end
+      expire_date.to_i
+    end
+
+    def delete_cache(group_name)
+      cache_file_name = File.join(cache_dir, group_name)
+      File.delete(cache_file_name) if File.readable?(cache_file_name)
+    end
+  end
+end

lib/framework/core/fact_manager.rb

@@ -20,10 +20,16 @@ def resolve_facts(user_query = [])
       loaded_facts = @fact_loader.load(Options.get)
       searched_facts = QueryParser.parse(user_query, loaded_facts)
 
+      cache_manager = Facter::CacheManager.new
+      searched_facts, cached_facts = cache_manager.resolve_facts(searched_facts)
       internal_facts = @internal_fact_mgr.resolve_facts(searched_facts)
       external_facts = @external_fact_mgr.resolve_facts(searched_facts)
 
       resolved_facts = override_core_facts(internal_facts, external_facts)
+
+      cache_manager.cache_facts(resolved_facts)
+      resolved_facts = resolved_facts.concat(cached_facts)
+
       FactFilter.new.filter_facts!(resolved_facts)
 
       resolved_facts

lib/framework/core/file_loader.rb

@@ -31,9 +31,7 @@ def load_lib_dirs(*dirs)
 require "#{ROOT_DIR}/lib/framework/detector/os_detector"
 
 require "#{ROOT_DIR}/lib/framework/config/config_reader"
-require "#{ROOT_DIR}/lib/framework/config/group_list"
-require "#{ROOT_DIR}/lib/framework/config/block_list"
-require "#{ROOT_DIR}/lib/framework/config/cache_list"
+require "#{ROOT_DIR}/lib/framework/config/fact_groups"
 
 load_dir(['config'])
 

lib/framework/core/options/config_file_options.rb

@@ -71,7 +71,7 @@ def augment_show_legacy(global_conf)
       end
 
       def augment_facts(ttls)
-        blocked_facts = Facter::BlockList.new.blocked_facts
+        blocked_facts = Facter::FactGroups.new.blocked_facts
         @options[:blocked_facts] = blocked_facts unless blocked_facts.nil?
 
         @options[:ttls] = ttls unless ttls.nil?