diff --git a/lib/puppet/application/environment.rb b/lib/puppet/application/environment.rb new file mode 100644 index 00000000000..12825ccb8d9 --- /dev/null +++ b/lib/puppet/application/environment.rb @@ -0,0 +1,4 @@ +require 'puppet/application/face_base' + +class Puppet::Application::Environment < Puppet::Application::FaceBase +end diff --git a/lib/puppet/environments.rb b/lib/puppet/environments.rb index 9f7ac5c3115..ef276b7d4dc 100644 --- a/lib/puppet/environments.rb +++ b/lib/puppet/environments.rb @@ -48,6 +48,15 @@ def for(module_path, manifest) # we are looking up # @return [Puppet::Setting::EnvironmentConf, nil] the configuration for the # requested environment, or nil if not found or no configuration is available + # + # @!macro [new] loader_get_environment_dir + # Attempt to obtain the parent environment dir of a given environment. Only the + # directories environments can provide this value. + # + # @param name [String,Symbol] The name of the environment whose configuration + # we are looking up + # @return [String, nil] the path to the environment directory for the + # requested environment, or nil if not found or no directory is available # A source of pre-defined environments. # @@ -88,6 +97,13 @@ def get_conf(name) nil end end + + # @note There's no environment dir per definition in static environments + # + # @!macro loader_get_environment_dir + def get_environment_dir(name) + nil + end end # A source of unlisted pre-defined environments. @@ -151,6 +167,13 @@ def get(name) def get_conf(name) nil end + + # @note There's no environment dir per definition in legacy environments + # + # @!macro loader_get_environment_dir + def get_environment_dir(name) + nil + end end # Reads environments from a directory on disk. Each environment is @@ -217,6 +240,17 @@ def get_conf(name) nil end + # @!macro loader_get_environment_dir + def get_environment_dir(name) + valid_directories.each do |envdir| + envname = Puppet::FileSystem.basename_string(envdir) + if envname == name.to_s + return envdir + end + end + nil + end + private def valid_directories @@ -269,10 +303,21 @@ def get_conf(name) nil end + # @!macro loader_get_environment_dir + def get_environment_dir(name) + @loaders.each do |loader| + if envdir = loader.get_environment_dir(name) + return envdir + end + end + nil + end + end class Cached < Combined INFINITY = 1.0 / 0.0 + MANUAL = -INFINITY def initialize(*loaders) super @@ -290,13 +335,11 @@ def get(name) end # Clears the cache of the environment with the given name. - # (The intention is that this could be used from a MANUAL cache eviction command (TBD) def clear(name) @cache.delete(name) end # Clears all cached environments. - # (The intention is that this could be used from a MANUAL cache eviction command (TBD) def clear_all() @cache = {} end @@ -319,6 +362,9 @@ def entry(env) NotCachedEntry.new(env) # Entry that is always expired (avoids syscall to get time) when INFINITY Entry.new(env) # Entry that never expires (avoids syscall to get time) + when MANUAL + # Entry that expires on demand (when the environment directory is touched) + ManualEntry.new(env, get_environment_dir(env.name)) else TTLEntry.new(env, ttl) end @@ -352,6 +398,48 @@ def expired? end end + # File based eviction policy entry + # when the watched_file file mtime changes + # the entry is marked as expired + class ManualEntry < Entry + + # how long (in seconds) to wait before + # being allowed to stat the watched_file + # again. + STAT_TIMEOUT = 1 + + def initialize(value, watched_file) + super value + unless Puppet::FileSystem.exist?(watched_file) + raise "Watched environment directory #{watched_file} doesn't exist" + end + @last_time = Time.now + @watched_file = watched_file + @watched_file_ctime = watched_file_ctime + end + + def expired? + ctime = watched_file_ctime + result = @watched_file_ctime != ctime + @watched_file_ctime = ctime + result + end + + private + + # return the watched_file ctime, but limit the rate + # to 1/STAT_TIMEOUT calls to stat per seconds + def watched_file_ctime + now = Time.now + if @last_time + STAT_TIMEOUT <= now + @last_time = now + Puppet::FileSystem.stat(@watched_file).ctime + else + @watched_file_ctime + end + end + end + # Time to Live eviction policy entry class TTLEntry < Entry def initialize(value, ttl_seconds) diff --git a/lib/puppet/face/environment.rb b/lib/puppet/face/environment.rb new file mode 100644 index 00000000000..1a397da3528 --- /dev/null +++ b/lib/puppet/face/environment.rb @@ -0,0 +1,101 @@ +require 'puppet/face' + +Puppet::Face.define(:environment, '0.0.1') do + copyright "Puppet Labs", 2014 + license "Apache 2 license; see COPYING" + + summary "Provides interaction with directory environments." + description <<-EOT + This command helps with management of directory based environments, notably listing them or flushing the puppet master + internal cache of directory environments set to 'manual'. + EOT + + action :list do + summary "List all directory environments." + returns "the list of all directory environments" + description <<-EOT + Lists all directory environments. + EOT + examples <<-EOT + Lists all directory environments: + + $ puppet environment list + EOT + + option "--details" do + summary "displays more information about the listed environments" + end + + when_invoked do |options| + setup_environments do + Puppet.lookup(:environments).list.each do |env| + unless options[:details] + puts env.name + else + unless Puppet.lookup(:environments).get_environment_dir(env.name).nil? + conf = Puppet.lookup(:environments).get_conf(env.name) + puts "#{env.name} (timeout: #{Puppet::Settings::TTLSetting.unmunge(conf.environment_timeout, 'environment_timeout')}, manifest: #{conf.manifest}, modulepath: #{conf.modulepath})" + end + end + end + end + nil + end + end + + action :flush do + summary "Flushes the cache of directory environments set to 'manual'." + arguments "[ [ ...]]" + returns "Nothing." + description <<-EOT + Flushes the given environment cache. With --all it is possible to flush all directory environments. + EOT + examples <<-EOT + Manually flush the usdatacenter environment: + + $ puppet environment flush usdatacenter + + Manually flush all environments: + + $ puppet environment flush --all + + Manually flush several environments: + + $ puppet environment flush env1 env2 env3 + EOT + + option "--all" do + summary "force all directory environments set to 'manual' to be invalidated" + end + + when_invoked do |*args| + options = args.pop + name = args + + setup_environments do + envs = options[:all] ? Puppet.lookup(:environments).list.map(&:name) : [name].flatten + envs.each do |envname| + if dir = Puppet.lookup(:environments).get_environment_dir(envname) + Puppet::FileSystem.touch(dir) + end + end + end + nil + end + end + + def setup_environments + # pretend we're the master + master_section = Puppet.settings.values(nil, :master) + + loader_settings = { + :environmentpath => master_section.interpolate(:environmentpath), + :basemodulepath => master_section.interpolate(:basemodulepath), + } + Puppet.override(Puppet.base_context(loader_settings), + "New environment loaders generated from the requested section.") do + yield + end + end + +end \ No newline at end of file diff --git a/lib/puppet/file_system/memory_file.rb b/lib/puppet/file_system/memory_file.rb index 2f6555cc567..7aebc29357e 100644 --- a/lib/puppet/file_system/memory_file.rb +++ b/lib/puppet/file_system/memory_file.rb @@ -15,12 +15,15 @@ def self.an_executable(path) new(path, :exist? => true, :executable? => true) end - def self.a_directory(path, children = []) + def self.a_directory(path, children = [], options = {}) new(path, - :exist? => true, - :excutable? => true, - :directory? => true, - :children => children) + options.merge({ + :exist? => true, + :excutable? => true, + :directory? => true, + :children => children, + }) + ) end def initialize(path, properties) @@ -34,6 +37,11 @@ def initialize(path, properties) def directory?; @properties[:directory?]; end def exist?; @properties[:exist?]; end def executable?; @properties[:executable?]; end + def ctime; @properties[:ctime]; end + + def touch + @properties[:ctime] = Time.now + end def each_line(&block) handle.each_line(&block) diff --git a/lib/puppet/file_system/memory_impl.rb b/lib/puppet/file_system/memory_impl.rb index 6fa35782398..3a1e11291a9 100644 --- a/lib/puppet/file_system/memory_impl.rb +++ b/lib/puppet/file_system/memory_impl.rb @@ -61,6 +61,23 @@ def assert_path(path) end end + # Minimal File::Stat implementation + # for some tests + class Stat + attr_reader :ctime + def initialize(ctime) + @ctime = ctime + end + end + + def stat(path) + Stat.new(assert_path(path).ctime) + end + + def touch(path) + assert_path(path).touch + end + private def find(path) diff --git a/lib/puppet/settings/ttl_setting.rb b/lib/puppet/settings/ttl_setting.rb index 505066d90fd..5b520eccc13 100644 --- a/lib/puppet/settings/ttl_setting.rb +++ b/lib/puppet/settings/ttl_setting.rb @@ -4,6 +4,7 @@ # class Puppet::Settings::TTLSetting < Puppet::Settings::BaseSetting INFINITY = 1.0 / 0.0 + MANUAL = -INFINITY # How we convert from various units to seconds. UNITMAP = { @@ -30,8 +31,11 @@ def munge(value) # Convert the value to Numeric, parsing numeric string with units if necessary. def self.munge(value, param_name) case + when value == 'manual' + MANUAL + when value.is_a?(Numeric) - if value < 0 + if value < 0 && value != MANUAL raise Puppet::Settings::ValidationError, "Invalid negative 'time to live' #{value.inspect} - did you mean 'unlimited'?" end value @@ -45,4 +49,33 @@ def self.munge(value, param_name) raise Puppet::Settings::ValidationError, "Invalid 'time to live' format '#{value.inspect}' for parameter: #{param_name}" end end + + def self.unmunge(ttl, param_name = 'unknown') + case + when ttl == MANUAL + 'manual' + when ttl == INFINITY + 'unlimited' + when ttl.is_a?(Numeric) + multiples = [UNITMAP['y'], UNITMAP['d'], UNITMAP['h'], UNITMAP['m'], UNITMAP['s']] + digits = [] + multiples.inject(ttl.to_f.round) do |total, multiple| + # Divide into largest unit + digits << total / multiple + total % multiple # The remainder will be divided as the next largest + end + + # format + units = ['y','d','h','m','s'] + digits.zip(units).map { |v,u| + if v > 0 + "#{v}#{u}" + else + nil + end + }.reject(&:nil?).join(" ") + else + raise Puppet::Settings::ValidationError, "Invalid 'time to live' format '#{ttl}' for parameter: #{param_name}" + end + end end diff --git a/spec/unit/environments_spec.rb b/spec/unit/environments_spec.rb index 80a0bb2d59d..103c0160e7d 100644 --- a/spec/unit/environments_spec.rb +++ b/spec/unit/environments_spec.rb @@ -106,6 +106,19 @@ module PuppetEnvironments end end + it "returns a given environment directory" do + directory_tree = FS::MemoryFile.a_directory(File.expand_path("envdir"), [ + FS::MemoryFile.a_directory("env1", [ + FS::MemoryFile.a_missing_file("environment.conf"), + ]) + ]) + + loader_from(:filesystem => [directory_tree], + :directory => directory_tree) do |loader| + expect(loader.get_environment_dir("env1")).to eq(Puppet::FileSystem.children(directory_tree).first) + end + end + context "with an environment.conf" do let(:envdir) do FS::MemoryFile.a_directory(File.expand_path("envdir"), [ @@ -336,6 +349,136 @@ module PuppetEnvironments end end + describe "with a cache" do + + ORIGIN = Time.at(12345678) + + before :each do + Time.stubs(:now).returns(ORIGIN) + end + + describe "which never expires" do + content = <<-EOF + environment_timeout = unlimited + manifest=relative/manifest + modulepath=relative/modules + config_version=relative/script + EOF + + let(:envdir) { + envdir = FS::MemoryFile.a_directory(File.expand_path("envdir"), [ + FS::MemoryFile.a_directory("env1", [ + FS::MemoryFile.a_regular_file_containing("environment.conf", content), + FS::MemoryFile.a_directory("modules"), + FS::MemoryFile.a_directory("manifests"), + ]), + ]) + } + + it "never expires" do + cached_loader_from(:filesystem => envdir, + :directory => envdir) do |loader| + original = loader.get("env1") + Time.stubs(:now).returns(ORIGIN + 10 * 86400 * 365) + expect(loader.get("env1")).to equal(original) + end + end + end + + describe "which expires with a ttl" do + content = <<-EOF + environment_timeout = 5s + manifest=relative/manifest + modulepath=relative/modules + config_version=relative/script + EOF + + let(:envdir) { + envdir = FS::MemoryFile.a_directory(File.expand_path("envdir"), [ + FS::MemoryFile.a_directory("env1", [ + FS::MemoryFile.a_regular_file_containing("environment.conf", content), + FS::MemoryFile.a_directory("modules"), + FS::MemoryFile.a_directory("manifests"), + ]), + ]) + } + + it "has an environment_timeout" do + cached_loader_from(:filesystem => envdir, + :directory => envdir) do |loader| + expect(loader.get_conf("env1").environment_timeout).to eq(5) + end + end + + it "serves the cached environment" do + cached_loader_from(:filesystem => envdir, + :directory => envdir) do |loader| + expect(loader.get("env1")).to equal(loader.get("env1")) + end + end + + it "recreates the environment after the ttl" do + cached_loader_from(:filesystem => envdir, + :directory => envdir) do |loader| + original = loader.get("env1") + Time.stubs(:now).returns(ORIGIN + 20) + expect(loader.get("env1")).not_to equal(original) + end + end + end + + describe "manually flushed" do + content = <<-EOF + environment_timeout = manual + manifest=relative/manifest + modulepath=relative/modules + config_version=relative/script + EOF + + let(:envdir) { + envdir = FS::MemoryFile.a_directory(File.expand_path("envdir"), [ + FS::MemoryFile.a_directory("env1", [ + FS::MemoryFile.a_regular_file_containing("environment.conf", content), + FS::MemoryFile.a_directory("modules"), + FS::MemoryFile.a_directory("manifests"), + ]), + ], :ctime => ORIGIN.to_i) + } + + it "serves the cached environment" do + cached_loader_from(:filesystem => envdir, + :directory => envdir) do |loader| + original = loader.get("env1") + Time.stubs(:now).returns(ORIGIN + 10 * 86400 * 365) + expect(loader.get("env1")).to equal(original) + end + end + + it "recreates the environment when the environment directory is touched" do + cached_loader_from(:filesystem => envdir, + :directory => envdir) do |loader| + original = loader.get("env1") + Time.stubs(:now).returns(ORIGIN + 20) + Puppet::FileSystem.touch(envdir.children.first) + Time.stubs(:now).returns(ORIGIN + 40) + expect(loader.get("env1")).not_to equal(original) + expect(loader.get("env1")).to equal(loader.get("env1")) + end + end + + it "won't invalidate before waiting 1s" do + cached_loader_from(:filesystem => envdir, + :directory => envdir) do |loader| + original = loader.get("env1") + Puppet::FileSystem.touch(envdir.children.first) + expect(loader.get("env1")).to equal(original) + Time.stubs(:now).returns(ORIGIN + 1) + expect(loader.get("env1")).not_to equal(original) + end + end + end + end + RSpec::Matchers.define :environment do |name| match do |env| env.name == name && @@ -379,5 +522,17 @@ def loader_from(options, &block) end end end + + def cached_loader_from(options, &block) + FS.overlay(*options[:filesystem]) do + environments = Puppet::Environments::Cached.new(Puppet::Environments::Directories.new( + options[:directory], + options[:modulepath] || [] + )) + Puppet.override(:environments => environments) do + yield environments + end + end + end end end diff --git a/spec/unit/face/environment_spec.rb b/spec/unit/face/environment_spec.rb new file mode 100644 index 00000000000..0fb2912c429 --- /dev/null +++ b/spec/unit/face/environment_spec.rb @@ -0,0 +1,95 @@ +#! /usr/bin/env ruby +require 'spec_helper' +require 'puppet/face' + +module PuppetFaceSpecs +describe Puppet::Face[:environment, '0.0.1'] do + + FS = Puppet::FileSystem + + before :each do + Puppet[:environmentpath] = '/dev/null/environments' + end + + let(:envdir) { + FS::MemoryFile.a_directory(File.expand_path("/dev/null/environments"), [ + FS::MemoryFile.a_directory("manual", [ + FS::MemoryFile.a_regular_file_containing("environment.conf", <<-CONTENT), + environment_timeout=manual + modulepath=/dev/null/modpath + CONTENT + ], :ctime => 12345678), + FS::MemoryFile.a_directory("unlimited", [ + FS::MemoryFile.a_regular_file_containing("environment.conf", <<-CONTENT), + environment_timeout=unlimited + modulepath=/dev/null/modpath + CONTENT + ], :ctime => 12345678), + FS::MemoryFile.a_directory("timingout", [ + FS::MemoryFile.a_regular_file_containing("environment.conf", <<-CONTENT), + environment_timeout=3m + modulepath=/dev/null/modpath + CONTENT + ], :ctime => 12345678), + ]) + } + + it "prints the list of environments" do + FS.overlay( + *envdir + ) do + expect { subject.list }.to have_printed(<<-OUTPUT) +manual +unlimited +timingout + OUTPUT + end + end + + it "prints detailed list of environments" do + FS.overlay( + *envdir + ) do + expect { subject.list({:details => true}) }.to have_printed(<<-OUTPUT) +manual (timeout: manual, manifest: /dev/null/environments/manual/manifests, modulepath: /dev/null/modpath) +unlimited (timeout: unlimited, manifest: /dev/null/environments/unlimited/manifests, modulepath: /dev/null/modpath) +timingout (timeout: 3m, manifest: /dev/null/environments/timingout/manifests, modulepath: /dev/null/modpath) + OUTPUT + end + end + + it "flushes an environment" do + FS.overlay( + *envdir + ) do + Puppet::FileSystem.stat('/dev/null/environments/manual').ctime.to_i.should_not be > 12345678 + subject.flush('manual') + Puppet::FileSystem.stat('/dev/null/environments/manual').ctime.to_i.should be > 12345678 + end + end + + it "flushes several environments" do + FS.overlay( + *envdir + ) do + Puppet::FileSystem.stat("/dev/null/environments/unlimited").ctime.to_i.should_not be > 12345678 + Puppet::FileSystem.stat("/dev/null/environments/manual").ctime.to_i.should_not be > 12345678 + subject.flush(%w{manual unlimited}) + Puppet::FileSystem.stat('/dev/null/environments/unlimited').ctime.to_i.should be > 12345678 + Puppet::FileSystem.stat('/dev/null/environments/manual').ctime.to_i.should be > 12345678 + end + end + + it "flushes all environments" do + FS.overlay( + *envdir + ) do + subject.flush({:all => true}) + Puppet::FileSystem.stat('/dev/null/environments/manual').ctime.to_i.should be > 12345678 + Puppet::FileSystem.stat('/dev/null/environments/unlimited').ctime.to_i.should be > 12345678 + Puppet::FileSystem.stat('/dev/null/environments/timingout').ctime.to_i.should be > 12345678 + end + end + +end +end diff --git a/spec/unit/settings/ttl_setting_spec.rb b/spec/unit/settings/ttl_setting_spec.rb new file mode 100644 index 00000000000..2926ad1dd1d --- /dev/null +++ b/spec/unit/settings/ttl_setting_spec.rb @@ -0,0 +1,44 @@ +require 'spec_helper' +require 'puppet/settings/ttl_setting.rb' + +describe Puppet::Settings::TTLSetting do + + { '5s' => 5, '10m' => 10 * 60, '12h' => 12 * 60 * 60, '1d' => 86400, '2y' => 2 * 365 * 86400 }.each do |ttl,expected| + it "allows a #{ttl} TTL" do + expect(ttl_setting.munge(ttl)).to eq(expected) + end + end + + it "disallows negative numeric TTL" do + expect do + ttl_setting.munge("-5s") + end.to raise_error(Puppet::Settings::ValidationError) + end + + it "allows an unlimited TTL" do + expect(ttl_setting.munge("unlimited")).to eq(Puppet::Settings::TTLSetting::INFINITY) + end + + it "allows a manual TTL" do + expect(ttl_setting.munge("manual")).to eq(Puppet::Settings::TTLSetting::MANUAL) + end + + it "allows unmunging manual" do + expect(Puppet::Settings::TTLSetting.unmunge(Puppet::Settings::TTLSetting::MANUAL)).to eq('manual') + end + + it "allows unmunging unlimited" do + expect(Puppet::Settings::TTLSetting.unmunge(Puppet::Settings::TTLSetting::INFINITY)).to eq('unlimited') + end + + it "allows unmunging numeric TTL" do + expect(Puppet::Settings::TTLSetting.unmunge(234523)).to eq('2d 17h 8m 43s') + end + + def ttl_setting + Puppet::Settings::TTLSetting.new(:settings => mock('settings'), + :name => "testing", + :desc => "description of testing" + ) + end +end