(PUP-2520) implements 'manual' environment directories

lib/puppet/application/environment.rb

@@ -0,0 +1,4 @@
+require 'puppet/application/face_base'
+
+class Puppet::Application::Environment < Puppet::Application::FaceBase
+end

lib/puppet/environments.rb

@@ -48,6 +48,15 @@ def for(module_path, manifest)
   #     we are looking up
   #   @return [Puppet::Setting::EnvironmentConf, nil] the configuration for the
   #     requested environment, or nil if not found or no configuration is available
+  #
+  # @!macro [new] loader_get_environment_dir
+  #   Attempt to obtain the parent environment dir of a given environment.  Only the
+  #     directories environments can provide this value.
+  #
+  #   @param name [String,Symbol] The name of the environment whose configuration
+  #     we are looking up
+  #   @return [String, nil] the path to the environment directory for the
+  #     requested environment, or nil if not found or no directory is available
 
   # A source of pre-defined environments.
   #
@@ -88,6 +97,13 @@ def get_conf(name)
         nil
       end
     end
+
+    # @note There's no environment dir per definition in static environments
+    #
+    # @!macro loader_get_environment_dir
+    def get_environment_dir(name)
+      nil
+    end
   end
 
   # A source of unlisted pre-defined environments.
@@ -151,6 +167,13 @@ def get(name)
     def get_conf(name)
       nil
     end
+
+    # @note There's no environment dir per definition in legacy environments
+    #
+    # @!macro loader_get_environment_dir
+    def get_environment_dir(name)
+      nil
+    end
   end
 
   # Reads environments from a directory on disk. Each environment is
@@ -217,6 +240,17 @@ def get_conf(name)
       nil
     end
 
+    # @!macro loader_get_environment_dir
+    def get_environment_dir(name)
+      valid_directories.each do |envdir|
+        envname = Puppet::FileSystem.basename_string(envdir)
+        if envname == name.to_s
+          return envdir
+        end
+      end
+      nil
+    end
+
     private
 
     def valid_directories
@@ -269,10 +303,21 @@ def get_conf(name)
       nil
     end
 
+    # @!macro loader_get_environment_dir
+    def get_environment_dir(name)
+      @loaders.each do |loader|
+        if envdir = loader.get_environment_dir(name)
+          return envdir
+        end
+      end
+      nil
+    end
+
   end
 
   class Cached < Combined
     INFINITY = 1.0 / 0.0
+    MANUAL = -INFINITY
 
     def initialize(*loaders)
       super
@@ -290,13 +335,11 @@ def get(name)
     end
 
     # Clears the cache of the environment with the given name.
-    # (The intention is that this could be used from a MANUAL cache eviction command (TBD)
     def clear(name)
       @cache.delete(name)
     end
 
     # Clears all cached environments.
-    # (The intention is that this could be used from a MANUAL cache eviction command (TBD)
     def clear_all()
       @cache = {}
     end
@@ -319,6 +362,9 @@ def entry(env)
         NotCachedEntry.new(env)     # Entry that is always expired (avoids syscall to get time)
       when INFINITY
         Entry.new(env)              # Entry that never expires (avoids syscall to get time)
+      when MANUAL
+        # Entry that expires on demand (when the environment directory is touched)
+        ManualEntry.new(env, get_environment_dir(env.name))
       else
         TTLEntry.new(env, ttl)
       end
@@ -352,6 +398,48 @@ def expired?
       end
     end
 
+    # File based eviction policy entry
+    # when the watched_file file mtime changes
+    # the entry is marked as expired
+    class ManualEntry < Entry
+
+      # how long (in seconds) to wait before
+      # being allowed to stat the watched_file
+      # again.
+      STAT_TIMEOUT = 1
+
+      def initialize(value, watched_file)
+        super value
+        unless Puppet::FileSystem.exist?(watched_file)
+          raise "Watched environment directory #{watched_file} doesn't exist"
+        end
+        @last_time = Time.now
+        @watched_file = watched_file
+        @watched_file_ctime = watched_file_ctime
+      end
+
+      def expired?
+        ctime = watched_file_ctime
+        result = @watched_file_ctime != ctime
+        @watched_file_ctime = ctime
+        result
+      end
+
+      private
+
+      # return the watched_file ctime, but limit the rate
+      # to 1/STAT_TIMEOUT calls to stat per seconds
+      def watched_file_ctime
+        now = Time.now
+        if @last_time + STAT_TIMEOUT <= now
+          @last_time = now
+          Puppet::FileSystem.stat(@watched_file).ctime
+        else
+          @watched_file_ctime
+        end
+      end
+    end
+
     # Time to Live eviction policy entry
     class TTLEntry < Entry
       def initialize(value, ttl_seconds)

lib/puppet/face/environment.rb

@@ -0,0 +1,101 @@
+require 'puppet/face'
+
+Puppet::Face.define(:environment, '0.0.1') do
+  copyright "Puppet Labs", 2014
+  license   "Apache 2 license; see COPYING"
+
+  summary "Provides interaction with directory environments."
+  description <<-EOT
+  This command helps with management of directory based environments, notably listing them or flushing the puppet master
+  internal cache of directory environments set to 'manual'.
+  EOT
+
+  action :list do
+    summary "List all directory environments."
+    returns "the list of all directory environments"
+    description <<-EOT
+    Lists all directory environments.
+    EOT
+    examples <<-EOT
+      Lists all directory environments:
+
+      $ puppet environment list
+    EOT
+
+    option "--details" do
+      summary "displays more information about the listed environments"
+    end
+
+    when_invoked do |options|
+      setup_environments do
+        Puppet.lookup(:environments).list.each do |env|
+          unless options[:details]
+            puts env.name
+          else
+            unless Puppet.lookup(:environments).get_environment_dir(env.name).nil?
+              conf = Puppet.lookup(:environments).get_conf(env.name)
+              puts "#{env.name} (timeout: #{Puppet::Settings::TTLSetting.unmunge(conf.environment_timeout, 'environment_timeout')}, manifest: #{conf.manifest}, modulepath: #{conf.modulepath})"
+            end
+          end
+        end
+      end
+      nil
+    end
+  end
+
+  action :flush do
+    summary "Flushes the cache of directory environments set to 'manual'."
+    arguments "[<environment> [<environment> ...]]"
+    returns "Nothing."
+    description <<-EOT
+    Flushes the given environment cache. With --all it is possible to flush all directory environments.
+    EOT
+    examples <<-EOT
+      Manually flush the usdatacenter environment:
+
+      $ puppet environment flush usdatacenter
+
+      Manually flush all environments:
+
+      $ puppet environment flush --all
+
+      Manually flush several environments:
+
+      $ puppet environment flush env1 env2 env3
+    EOT
+
+    option "--all" do
+      summary "force all directory environments set to 'manual' to be invalidated"
+    end
+
+    when_invoked do |*args|
+      options = args.pop
+      name = args
+
+      setup_environments do
+        envs = options[:all] ? Puppet.lookup(:environments).list.map(&:name) : [name].flatten
+        envs.each do |envname|
+          if dir = Puppet.lookup(:environments).get_environment_dir(envname)
+            Puppet::FileSystem.touch(dir)
+          end
+        end
+      end
+      nil
+    end
+  end
+
+  def setup_environments
+    # pretend we're the master
+    master_section = Puppet.settings.values(nil, :master)
+
+    loader_settings = {
+      :environmentpath => master_section.interpolate(:environmentpath),
+      :basemodulepath => master_section.interpolate(:basemodulepath),
+    }
+    Puppet.override(Puppet.base_context(loader_settings),
+                   "New environment loaders generated from the requested section.") do
+      yield
+    end
+  end
+
+end

lib/puppet/file_system/memory_file.rb

@@ -15,12 +15,15 @@ def self.an_executable(path)
     new(path, :exist? => true, :executable? => true)
   end
 
-  def self.a_directory(path, children = [])
+  def self.a_directory(path, children = [], options = {})
     new(path,
-        :exist? => true,
-        :excutable? => true,
-        :directory? => true,
-        :children => children)
+        options.merge({
+          :exist? => true,
+          :excutable? => true,
+          :directory? => true,
+          :children => children,
+        })
+       )
   end
 
   def initialize(path, properties)
@@ -34,6 +37,11 @@ def initialize(path, properties)
   def directory?; @properties[:directory?]; end
   def exist?; @properties[:exist?]; end
   def executable?; @properties[:executable?]; end
+  def ctime; @properties[:ctime]; end
+
+  def touch
+    @properties[:ctime] = Time.now
+  end
 
   def each_line(&block)
     handle.each_line(&block)

lib/puppet/file_system/memory_impl.rb

@@ -61,6 +61,23 @@ def assert_path(path)
     end
   end
 
+  # Minimal File::Stat implementation
+  # for some tests
+  class Stat
+    attr_reader :ctime
+    def initialize(ctime)
+      @ctime = ctime
+    end
+  end
+
+  def stat(path)
+    Stat.new(assert_path(path).ctime)
+  end
+
+  def touch(path)
+    assert_path(path).touch
+  end
+
   private
 
   def find(path)

lib/puppet/settings/ttl_setting.rb

@@ -4,6 +4,7 @@
 #
 class Puppet::Settings::TTLSetting < Puppet::Settings::BaseSetting
   INFINITY = 1.0 / 0.0
+  MANUAL = -INFINITY
 
   # How we convert from various units to seconds.
   UNITMAP = {
@@ -30,8 +31,11 @@ def munge(value)
   # Convert the value to Numeric, parsing numeric string with units if necessary.
   def self.munge(value, param_name)
     case
+    when value == 'manual'
+      MANUAL
+
     when value.is_a?(Numeric)
-      if value < 0
+      if value < 0 && value != MANUAL
         raise Puppet::Settings::ValidationError, "Invalid negative 'time to live' #{value.inspect} - did you mean 'unlimited'?"
       end
       value
@@ -45,4 +49,33 @@ def self.munge(value, param_name)
       raise Puppet::Settings::ValidationError, "Invalid 'time to live' format '#{value.inspect}' for parameter: #{param_name}"
     end
   end
+
+  def self.unmunge(ttl, param_name = 'unknown')
+    case
+    when ttl == MANUAL
+      'manual'
+    when ttl == INFINITY
+      'unlimited'
+    when ttl.is_a?(Numeric)
+      multiples = [UNITMAP['y'], UNITMAP['d'], UNITMAP['h'], UNITMAP['m'], UNITMAP['s']]
+      digits = []
+      multiples.inject(ttl.to_f.round) do |total, multiple|
+        # Divide into largest unit
+        digits << total / multiple
+        total % multiple # The remainder will be divided as the next largest
+      end
+
+      # format
+      units = ['y','d','h','m','s']
+      digits.zip(units).map { |v,u|
+        if v > 0
+          "#{v}#{u}"
+        else
+          nil
+        end
+      }.reject(&:nil?).join(" ")
+    else
+      raise Puppet::Settings::ValidationError, "Invalid 'time to live' format '#{ttl}' for parameter: #{param_name}"
+    end
+  end
 end