Merge pull request #3341 from hlindberg/PUP-1640_data-in-modules-spi

(PUP-1640) Add agnostic mechanism for data in modules/environment

lib/puppet.rb

@@ -247,3 +247,4 @@ def self.rollback_context(name)
 require 'puppet/util/storage'
 require 'puppet/status'
 require 'puppet/file_bucket/file'
+require 'puppet/plugins'

lib/puppet/data_providers.rb

@@ -0,0 +1,34 @@
+module Puppet::DataProviders
+
+  # Stub to allow this module to be required before the actual implementation (which requires Puppet::Pops
+  # and Puppet::Pops cannot be loaded until Puppet is fully loaded.
+  #
+  class DataAdapters
+  end
+
+  def self.assert_loaded
+    unless @loaded
+      require 'puppet/pops'
+      require 'puppet/data_providers/data_adapter'
+    end
+    @loaded = true
+  end
+
+  def self.lookup_in_environment(name, scope)
+    assert_loaded()
+    adapter = Puppet::DataProviders::DataAdapter.adapt(Puppet.lookup(:current_environment))
+    adapter.env_provider.lookup(name,scope)
+  end
+
+  MODULE_NAME = 'module_name'.freeze
+
+  def self.lookup_in_module(name, scope)
+    # Do not attempt to do a lookup in a module if evaluated code is not in a module
+    # which is detected by checking if "MODULE_NAME" exists in scope
+    return nil unless scope.exist?(MODULE_NAME)
+
+    assert_loaded()
+    adapter = Puppet::DataProviders::DataAdapter.adapt(Puppet.lookup(:current_environment))
+    adapter.module_provider(scope[MODULE_NAME]).lookup(name,scope)
+  end
+end

lib/puppet/data_providers/data_adapter.rb

@@ -0,0 +1,86 @@
+# A DataAdapter adapts an object with a Hash of data
+#
+class Puppet::DataProviders::DataAdapter < Puppet::Pops::Adaptable::Adapter
+  attr_accessor :data
+  attr_accessor :env_provider
+
+  def initialize(env)
+    @env = env
+    @data = {}
+  end
+
+  def [](name)
+    @data[name]
+  end
+
+  def has_name?(name)
+    @data.has_key?
+  end
+
+  def []=(name, value)
+    unless value.is_a?(Hash)
+      raise ArgumentError, "Given value must be a Hash, got: #{value.class}."
+    end
+    @data[name] = value
+  end
+
+  def env_provider
+    @env_provider ||= initialize_env_provider
+  end
+
+  def module_provider(module_name)
+    @data[module_name] ||= initialize_module_provider(module_name)
+  end
+
+  def self.create_adapter(environment)
+    new(environment)
+  end
+
+  def initialize_module_provider(module_name)
+    # Support running tests without an injector being configured == using a null implementation
+    unless injector = Puppet.lookup(:injector) { nil }
+      return Puppet::Plugins::DataProviders::ModuleDataProvider.new()
+    end
+    # Get the registry of module to provider implementation name
+    module_service_type = Puppet::Plugins::DataProviders.hash_of_per_module_data_provider
+    module_service_name = Puppet::Plugins::DataProviders::PER_MODULE_DATA_PROVIDER_KEY
+    module_service = Puppet.lookup(:injector).lookup(nil, module_service_type, module_service_name)
+    provider_name = module_service[module_name] || 'none'
+
+    service_type = Puppet::Plugins::DataProviders.hash_of_module_data_providers
+    service_name = Puppet::Plugins::DataProviders::MODULE_DATA_PROVIDERS_KEY
+
+    # Get the service (registry of known implementations)
+    service = Puppet.lookup(:injector).lookup(nil, service_type, service_name)
+    provider = service[provider_name]
+    unless provider
+      raise Puppet::Error.new("Environment '#{@env.name}', cannot find module_data_provider '#{provider_name}'")
+    end
+    provider
+  end
+
+  def initialize_env_provider
+    # Get the environment's configuration since we need to know which data provider
+    # should be used (includes 'none' which gets a null implementation).
+    #
+    env_conf = Puppet.lookup(:environments).get_conf(@env.name)
+
+    # Get the data provider and find the bound implementation
+    # TODO: PUP-1640, drop the nil check when legacy env support is dropped
+    provider_name = env_conf.nil? ? 'none' : env_conf.environment_data_provider
+    service_type = Puppet::Plugins::DataProviders.hash_of_environment_data_providers
+    service_name = Puppet::Plugins::DataProviders::ENV_DATA_PROVIDERS_KEY
+
+    # Get the service (registry of known implementations)
+    # Support running tests without an injector being configured == using a null implementation
+    unless injector = Puppet.lookup(:injector) { nil }
+      return Puppet::Plugins::DataProviders::EnvironmentDataProvider.new()
+    end
+    service = Puppet.lookup(:injector).lookup(nil, service_type, service_name)
+    provider = service[provider_name]
+    unless provider
+      raise Puppet::Error.new("Environment '#{@env.name}', cannot find environment_data_provider '#{provider_name}'")
+    end
+    provider
+  end
+end

lib/puppet/data_providers/data_function_support.rb

@@ -0,0 +1,32 @@
+module Puppet::DataProviders::DataFunctionSupport
+  # Gets the data from the compiler, or initializes it from a function call if not present in the compiler.
+  # This means, that the function providing the data is called once per compilation, and the data is cached for
+  # as long as the compiler lives (which is for one catalog production).
+  # This makes it possible to return data that is tailored for the request.
+  # The class including this module must implement `loader(scope)` to return the apropriate loader.
+  #
+  def data(key, scope)
+    compiler = scope.compiler
+    adapter = Puppet::DataProviders::DataAdapter.get(compiler) || Puppet::DataProviders::DataAdapter.adapt(compiler)
+    adapter.data[key] ||= initialize_data_from_function("#{key}::data", scope)
+  end
+
+  def initialize_data_from_function(name, scope)
+    Puppet::Util::Profiler.profile("Called #{name}", [ :functions, name ]) do
+      loader = loader(scope)
+      if loader && func = loader.load(:function, name)
+        # function found, call without arguments, must return a Hash
+        # TODO: Validate the function - to ensure it does not contain unwanted side effects
+        #       That can only be done if the function is a puppet function
+        #
+        result = func.call(scope)
+        unless result.is_a?(Hash)
+          raise Puppet::Error.new("Expected '#{name}' function to return a Hash, got #{result.class}")
+        end
+      else
+        raise Puppet::Error.new("Data from 'function' cannot find the required '#{name}' function")
+      end
+      result
+    end
+  end
+end

lib/puppet/data_providers/function_env_data_provider.rb

@@ -0,0 +1,26 @@
+# This file is loaded by the autoloader, and it does not find the data function support unless required relative
+#
+require_relative 'data_function_support'
+module Puppet::DataProviders; end
+
+# The FunctionEnvDataProvider provides data from a function called 'environment::data()' that resides in a
+# directory environment (seen as a module with the name environment).
+# The function is called on demand, and is associated with the compiler via an Adapter. This ensures that the data
+# is only produced once per compilation.
+#
+class Puppet::DataProviders::FunctionEnvDataProvider < Puppet::Plugins::DataProviders::EnvironmentDataProvider
+  include Puppet::DataProviders::DataFunctionSupport
+
+  def lookup(name, scope)
+    begin
+      data('environment', scope)[name]
+    rescue *Puppet::Error => detail
+      raise Puppet::DataBinding::LookupError.new(detail.message, detail)
+    end
+  end
+
+  def loader(scope)
+    # This loader allows the data function to be private or public in the environment
+    scope.compiler.loaders.private_environment_loader
+  end
+end

lib/puppet/data_providers/function_module_data_provider.rb

@@ -0,0 +1,42 @@
+# This file is loaded by the autoloader, and it does not find the data function support unless required relative
+#
+require_relative 'data_function_support'
+module Puppet::DataProviders; end
+
+# The FunctionModuleDataProvider provides data from a function called 'environment::data()' that resides in a
+# directory environment (seen as a module with the name environment).
+# The function is called on demand, and is associated with the compiler via an Adapter. This ensures that the data
+# is only produced once per compilation.
+#
+class Puppet::DataProviders::FunctionModuleDataProvider < Puppet::Plugins::DataProviders::ModuleDataProvider
+  MODULE_NAME = 'module_name'.freeze
+  include Puppet::DataProviders::DataFunctionSupport
+
+  def lookup(name, scope)
+    # If the module name does not exist, this call is not from within a module, and should be ignored.
+    unless scope.exist?(MODULE_NAME)
+      return nil
+    end
+    # Get the module name. Calls to the lookup method should only be performed for modules that have opted in
+    # by specifying that they use the 'function' implementation as the module_data provider. Thus, this will error
+    # out if a module specified 'function' but did not provide a function called <module-name>::data
+    #
+    module_name = scope[MODULE_NAME]
+    begin
+      data(module_name, scope)[name]
+    rescue *Puppet::Error => detail
+      raise Puppet::DataBinding::LookupError.new(detail.message, detail)
+    end
+  end
+
+  def loader(scope)
+    loaders = scope.compiler.loaders
+    if scope.exist?(MODULE_NAME)
+      loaders.private_loader_for_module(scope[MODULE_NAME])
+    else
+      # Produce the environment's loader when not in a module
+      # This loader allows the data function to be private or public in the environment
+      loaders.private_environment_loader
+    end
+  end
+end

lib/puppet/defaults.rb

@@ -463,6 +463,14 @@ def self.default_diffargs
       This setting can also be set to `unlimited`, which causes the environment to
       be cached until the master is restarted."
     },
+    :environment_data_provider => {
+      :default    => "none",
+      :desc       => "The name of a registered environment data provider. The two built in
+      and registered providers are 'none' (no environment specific data), and 'function'
+      (environment specific data obtained by calling the function 'environment::data()').
+      Other environment data providers may be registered in modules on the module path. For such
+      custom data providers see the respective module documentation."
+    },
     :thin_storeconfigs => {
       :default  => false,
       :type     => :boolean,

lib/puppet/plugins/configuration.rb

@@ -10,6 +10,7 @@
 module Puppet::Plugins::Configuration
     require 'puppet/plugins/binding_schemes'
     require 'puppet/plugins/syntax_checkers'
+    require 'puppet/plugins/data_providers'
 
     # Extension-points are registered here:
     #
@@ -32,6 +33,7 @@ module Puppet::Plugins::Configuration
 
     extensions.multibind(checkers_name).name(checkers_name).hash_of(checkers_type)
     extensions.multibind(schemes_name).name(schemes_name).hash_of(schemes_type)
+    Puppet::Plugins::DataProviders::register_extensions(extensions)
 
     # Register injector boot bindings
     # -------------------------------
@@ -62,4 +64,5 @@ module Puppet::Plugins::Configuration
       in_multibind(checkers_name)
       to_instance('Puppet::SyntaxCheckers::Json')
     end
+    Puppet::Plugins::DataProviders::register_defaults(bindings)
 end

lib/puppet/plugins/data_providers.rb

@@ -0,0 +1,90 @@
+module Puppet::Plugins; end
+
+class Puppet::Plugins::DataProviders
+
+  # The lookup **key** for the multibind containing data provider name per module
+  # @api public
+  PER_MODULE_DATA_PROVIDER_KEY       = 'puppet::module_data'
+
+  # The lookup **type** for the name of the per module data provider.
+  # @api public
+  PER_MODULE_DATA_PROVIDER_TYPE      = String
+
+  # The lookup **key** for the multibind containing map of provider name to env data provider implementation.
+  # @api public
+  ENV_DATA_PROVIDERS_KEY             = 'puppet::environment_data_providers'
+
+  # The lookup **type** for the multibind containing map of provider name to env data provider implementation.
+  # @api public
+  ENV_DATA_PROVIDERS_TYPE            = 'Puppet::Plugins::DataProviders::EnvironmentDataProvider'
+
+  # The lookup **key** for the multibind containing map of provider name to module data provider implementation.
+  # @api public
+  MODULE_DATA_PROVIDERS_KEY          = 'puppet::module_data_providers'
+
+  # The lookup **type** for the multibind containing map of provider name to module data provider implementation.
+  # @api public
+  MODULE_DATA_PROVIDERS_TYPE         = 'Puppet::Plugins::DataProviders::ModuleDataProvider'
+
+  def self.register_extensions(extensions)
+    extensions.multibind(PER_MODULE_DATA_PROVIDER_KEY).name(PER_MODULE_DATA_PROVIDER_KEY).hash_of(PER_MODULE_DATA_PROVIDER_TYPE)
+    extensions.multibind(ENV_DATA_PROVIDERS_KEY).name(ENV_DATA_PROVIDERS_KEY).hash_of(ENV_DATA_PROVIDERS_TYPE)
+    extensions.multibind(MODULE_DATA_PROVIDERS_KEY).name(MODULE_DATA_PROVIDERS_KEY).hash_of(MODULE_DATA_PROVIDERS_TYPE)
+  end
+
+  def self.hash_of_per_module_data_provider
+    @@HASH_OF_PER_MODULE_DATA_PROVIDERS ||= Puppet::Pops::Types::TypeFactory.hash_of(PER_MODULE_DATA_PROVIDER_TYPE)
+  end
+
+  def self.hash_of_module_data_providers
+    @@HASH_OF_MODULE_DATA_PROVIDERS ||= Puppet::Pops::Types::TypeFactory.hash_of(
+      Puppet::Pops::Types::TypeFactory.type_of(MODULE_DATA_PROVIDERS_TYPE))
+  end
+
+  def self.hash_of_environment_data_providers
+    @@HASH_OF_ENV_DATA_PROVIDERS ||= Puppet::Pops::Types::TypeFactory.hash_of(
+      Puppet::Pops::Types::TypeFactory.type_of(ENV_DATA_PROVIDERS_TYPE))
+  end
+
+  # Registers a 'none' environment data provider, and a 'none' module data provider as the defaults.
+  # This is only done to allow that something binds to 'none' rather than removing the entire binding (which
+  # has the same effect).
+  #
+  def self.register_defaults(default_bindings)
+    default_bindings.bind do
+      name('none')
+      in_multibind(ENV_DATA_PROVIDERS_KEY)
+      to_instance(ENV_DATA_PROVIDERS_TYPE)
+    end
+
+    default_bindings.bind do
+      name('function')
+      in_multibind(ENV_DATA_PROVIDERS_KEY)
+      to_instance('Puppet::DataProviders::FunctionEnvDataProvider')
+    end
+
+    default_bindings.bind do
+      name('none')
+      in_multibind(MODULE_DATA_PROVIDERS_KEY)
+      to_instance(MODULE_DATA_PROVIDERS_TYPE)
+    end
+
+    default_bindings.bind do
+      name('function')
+      in_multibind(MODULE_DATA_PROVIDERS_KEY)
+      to_instance('Puppet::DataProviders::FunctionModuleDataProvider')
+    end
+  end
+
+  class ModuleDataProvider
+    def lookup(name, scope)
+      nil
+    end
+  end
+
+  class EnvironmentDataProvider
+    def lookup(name, scope)
+      nil
+    end
+  end
+end

lib/puppet/pops.rb

@@ -123,3 +123,4 @@ module Functions
   require 'puppet/bindings'
   require 'puppet/functions'
 end
+require 'puppet/plugins/data_providers'

lib/puppet/pops/binder/bindings_factory.rb

@@ -389,6 +389,10 @@ def to_instance(type, *args)
       else
         raise ArgumentError, "to_instance accepts String (a class name), or a Class.*args got: #{type.class}."
       end
+
+      # Help by setting the type - since if an to_instance is bound, the type is know. This avoids having
+      # to specify the same thing twice.
+      self.instance_of(type)
       model.producer = Puppet::Pops::Binder::BindingsFactory.instance_producer(class_name, *args)
     end
 

lib/puppet/pops/binder/bindings_loader.rb

@@ -42,11 +42,11 @@ def self.loadable?(basedir, name)
   private
 
   def self.loader()
-    unless Puppet.settings[:confdir] == @confdir
-      @confdir = Puppet.settings[:confdir] == @confdir
-      @autoloader = Puppet::Util::Autoload.new("BindingsLoader", "puppet/bindings")
-    end
-    @autoloader
+    @autoloader ||= Puppet::Util::Autoload.new("BindingsLoader", "puppet/bindings")
+#    unless Puppet.settings[:confdir] == @confdir
+#      @confdir = Puppet.settings[:confdir] == @confdir
+#    end
+#    @autoloader
   end
 
   def self.provide_from_string(scope, name)
@@ -71,7 +71,7 @@ def self.provide_from_name_path(scope, name, name_path)
   end
 
   def self.paths_for_name(fq_name)
-    [de_camel(fq_name), downcased_path(fq_name)]
+    [de_camel(fq_name), downcased_path(fq_name)].uniq
   end
 
   def self.downcased_path(fq_name)