From 4ccd3bf2156b6319e4160ea93a979e4d07585f37 Mon Sep 17 00:00:00 2001 From: Jeff McCune Date: Mon, 28 Jan 2013 17:17:46 -0800 Subject: [PATCH 1/7] (#7559) Add Facter::Util::EC2.with_metadata_server method Without this patch Facter has no clear way to define the EC2 related facts in a robust and reliable manner. This is a problem because Facter needs to run as robustly on nodes that are inside of EC2 as it does on nodes that are outside of EC2. The root cause of the problem is that we have no way to introspect the system to determine if we're on EC2 or not without making a network call to the metadata server. [1] This means that either non-EC2 nodes will suffer a timeout trying to connect to a non-existent metadata server or EC2 nodes will suffer from facts not being defined. This patch addresses the problem by adding a new utility method with the following behavior. A block is accepted and will be executed only under the following conditions. The block should execute whatever code is necessary to define the EC2 facts themselves. 1: Do not execute if the virtual fact is not "xenu" 2: Do not execute the metadata server does not respond in 100ms after three consecutive attempts. [1] https://forums.aws.amazon.com/thread.jspa?threadID=62617 --- lib/facter/util/ec2.rb | 73 ++++++++++++++++++++++++++++++++++++++ spec/unit/util/ec2_spec.rb | 44 +++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/lib/facter/util/ec2.rb b/lib/facter/util/ec2.rb index c6e5dcadd4..5f1477e204 100644 --- a/lib/facter/util/ec2.rb +++ b/lib/facter/util/ec2.rb @@ -4,6 +4,79 @@ # Provide a set of utility static methods that help with resolving the EC2 # fact. module Facter::Util::EC2 + ## + # with_metadata_server takes a block of code and executes the block only if Facter is + # running on node that can access a metadata server at + # http://169.254.168.254/. This is useful to decide if it's reasonably + # likely that talking to the EC2 metadata server will be successful or not. + # + # @option options [Integer] :timeout (100) the maxiumum number of + # milliseconds Facter will block trying to talk to the metadata server. + # Defaults to 200. + # + # @option options [String] :fact ('virtual') the fact to check. The block will only be + # executed if the fact named here matches the value named in the :value + # option. + # + # @option options [String] :value ('xenu') the value to check. The block will be + # executed if Facter.value(options[:fact]) matches this value. + # + # @option options [String] :api_version ('latest') the Amazon AWS API + # version. The version string is usually a date, e.g. '2008-02-01'. + # + # @option options [Fixnum] :retry_limit (3) the maximum number of times that + # this method will try to contact the metadata server. The maximum run time + # is the timeout times this limit, so please keep the value small. + # + # @return [Object] the return value of the passed block, or {false} if the + # block was not executed because the conditions were not met or a timeout + # occurs. + def self.with_metadata_server(options = {}, &block) + opts = options.dup + opts[:timeout] ||= 100 + opts[:fact] ||= 'virtual' + opts[:value] ||= 'xenu' + opts[:api_version] ||= 'latest' + opts[:retry_limit] ||= 3 + # Conversion to fractional seconds for Timeout + timeout = opts[:timeout] / 1000.0 + raise ArgumentError, "A value is required for :fact" if opts[:fact].nil? + raise ArgumentError, "A value is required for :value" if opts[:value].nil? + return false if Facter.value(opts[:fact]) != opts[:value] + + metadata_base_url = "http://169.254.169.254" + + attempts = 0 + begin + able_to_connect = false + attempts = attempts + 1 + # Read the list of supported API versions + Timeout.timeout(timeout) do + read_uri(metadata_base_url) + end + rescue Timeout::Error => detail + retry if attempts < opts[:retry_limit] + Facter.warn "Timeout exceeded trying to communicate with #{metadata_base_url}, " + + "metadata server facts will be undefined. #{detail.message}" + rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNREFUSED => detail + retry if attempts < opts[:retry_limit] + Facter.warn "No metadata server available at #{metadata_base_url}, " + + "metadata server facts will be undefined. #{detail.message}" + rescue OpenURI::HTTPError => detail + retry if attempts < opts[:retry_limit] + Facter.warn "Metadata server at #{metadata_base_url} responded with an error. " + + "metadata server facts will be undefined. #{detail.message}" + else + able_to_connect = true + end + + if able_to_connect + return block.call + else + return false + end + end + class << self # Test if we can connect to the EC2 api. Return true if able to connect. # On failure this function fails silently and returns false. diff --git a/spec/unit/util/ec2_spec.rb b/spec/unit/util/ec2_spec.rb index f963db6c32..8ba24803f9 100755 --- a/spec/unit/util/ec2_spec.rb +++ b/spec/unit/util/ec2_spec.rb @@ -177,4 +177,48 @@ Facter::Util::EC2.userdata('2008-02-01').should == example_userdata end end + + describe "Facter::Util::EC2.with_metadata_server" do + before :each do + Facter::Util::EC2.stubs(:read_uri).returns("latest") + end + + subject do + Facter::Util::EC2.with_metadata_server do + "HELLO FROM THE CODE BLOCK" + end + end + + it 'returns false when not running on xenu' do + Facter.stubs(:value).with('virtual').returns('vmware') + subject.should be_false + end + + context 'default options and running on a xenu virtual machine' do + before :each do + Facter.stubs(:value).with('virtual').returns('xenu') + end + it 'returns the value of the block when the metadata server responds' do + subject.should == "HELLO FROM THE CODE BLOCK" + end + it 'returns false when the metadata server is unreachable' do + described_class.stubs(:read_uri).raises(Errno::ENETUNREACH) + subject.should be_false + end + it 'does not execute the block if the connection raises an exception' do + described_class.stubs(:read_uri).raises(Timeout::Error) + myvar = "The block didn't get called" + described_class.with_metadata_server do + myvar = "The block was called and should not have been." + end.should be_false + myvar.should == "The block didn't get called" + end + it 'succeeds on the third retry' do + retry_metadata = sequence('metadata') + Timeout.expects(:timeout).twice.in_sequence(retry_metadata).raises(Timeout::Error) + Timeout.expects(:timeout).once.in_sequence(retry_metadata).returns(true) + subject.should == "HELLO FROM THE CODE BLOCK" + end + end + end end From 683a8c962fc575e85ae8cd9676b604a12f4b875d Mon Sep 17 00:00:00 2001 From: Jeff McCune Date: Mon, 28 Jan 2013 17:41:05 -0800 Subject: [PATCH 2/7] (#7559) Try to define EC2 facts if virtual => xenu Without this patch the EC2 facts use a number of methods to determine if they're running on openstack, Amazon EC2, eucalyptus, and others. These methods have proved to be lacking robustness. This is a problem because Facter fails to define facts even when a Metadata server is available. This patch addresses the problem by changing the behavior of deciding when to define the dynamic EC2 facts. The Facter::Util::EC2.with_metadata_server method is used with a 50ms timeout over three consecutive retries. With this patch applied the EC2 metadata and userdata facts will be defined whenever the node is has a virtual fact value of "xenu" and there is a metadata data server available at http://169.254.169.254 This patch also removes the `userdata` and `metadata` methods that were previous defined as instance methods on the base Object class. Backport from master to facter-2 by Adrien Thebo Conflicts: lib/facter/util/ec2.rb spec/unit/ec2_spec.rb spec/unit/util/ec2_spec.rb --- lib/facter/ec2.rb | 36 +------ lib/facter/util/ec2.rb | 140 +++++++++++++++------------ spec/unit/ec2_spec.rb | 193 +++++++++---------------------------- spec/unit/util/ec2_spec.rb | 144 --------------------------- 4 files changed, 127 insertions(+), 386 deletions(-) diff --git a/lib/facter/ec2.rb b/lib/facter/ec2.rb index 09e0109528..fefe88a93d 100644 --- a/lib/facter/ec2.rb +++ b/lib/facter/ec2.rb @@ -1,37 +1,3 @@ require 'facter/util/ec2' -require 'open-uri' -def metadata(id = "") - open("http://169.254.169.254/2008-02-01/meta-data/#{id||=''}").read. - split("\n").each do |o| - key = "#{id}#{o.gsub(/\=.*$/, '/')}" - if key[-1..-1] != '/' - value = open("http://169.254.169.254/2008-02-01/meta-data/#{key}").read. - split("\n") - symbol = "ec2_#{key.gsub(/\-|\//, '_')}".to_sym - Facter.add(symbol) { setcode { value.join(',') } } - else - metadata(key) - end - end -rescue => details - Facter.warn "Could not retrieve ec2 metadata: #{details.message}" -end - -def userdata() - Facter.add(:ec2_userdata) do - setcode do - if userdata = Facter::Util::EC2.userdata - userdata.split - end - end - end -end - -if (Facter::Util::EC2.has_euca_mac? || Facter::Util::EC2.has_openstack_mac? || - Facter::Util::EC2.has_ec2_arp?) && Facter::Util::EC2.can_connect? - metadata - userdata -else - Facter.debug "Not an EC2 host" -end +Facter::Util::EC2.add_ec2_facts diff --git a/lib/facter/util/ec2.rb b/lib/facter/util/ec2.rb index 5f1477e204..30035227d4 100644 --- a/lib/facter/util/ec2.rb +++ b/lib/facter/util/ec2.rb @@ -4,9 +4,60 @@ # Provide a set of utility static methods that help with resolving the EC2 # fact. module Facter::Util::EC2 + CONNECTION_ERRORS = [ + OpenURI::HTTPError, + Errno::EHOSTDOWN, + Errno::EHOSTUNREACH, + Errno::ENETUNREACH, + Errno::ECONNABORTED, + Errno::ECONNREFUSED, + Errno::ECONNRESET, + Errno::ETIMEDOUT, + ] ## - # with_metadata_server takes a block of code and executes the block only if Facter is - # running on node that can access a metadata server at + # metadata is a recursive function that walks over the metadata server + # located at http://169.254.169.254 and defines a fact for each value found. + # This method introduces a high amount of latency to Facter, so care must be + # taken to call it only when reasonably certain the host is running in an + # environment where the metadata server is available. + def self.define_metadata_facts(id = "") + begin + if body = read_uri("http://169.254.169.254/latest/meta-data/#{id}") + body.split("\n").each do |o| + key = "#{id}#{o.gsub(/\=.*$/, '/')}" + if key[-1..-1] != '/' + value = read_uri("http://169.254.169.254/latest/meta-data/#{key}").split("\n") + symbol = "ec2_#{key.gsub(/\-|\//, '_')}".to_sym + Facter.add(symbol) { setcode { value.join(',') } } + else + define_metadata_facts(key) + end + end + end + rescue *CONNECTION_ERRORS => detail + Facter.warn "Could not retrieve ec2 metadata: #{detail.message}" + end + end + + ## + # define_userdata_fact creates a single fact named 'ec2_userdata' which has a + # value of the contents of the EC2 userdata field. This method introduces a + # high amount of latency to Facter, so care must be taken to call it only + # when reasonably certain the host is running in an environment where the + # metadata server is available. + def self.define_userdata_fact + Facter.add(:ec2_userdata) do + setcode do + if userdata = Facter::Util::EC2.userdata + userdata.split + end + end + end + end + + ## + # with_metadata_server takes a block of code and executes the block only if + # Facter is running on node that can access a metadata server at # http://169.254.168.254/. This is useful to decide if it's reasonably # likely that talking to the EC2 metadata server will be successful or not. # @@ -14,12 +65,12 @@ module Facter::Util::EC2 # milliseconds Facter will block trying to talk to the metadata server. # Defaults to 200. # - # @option options [String] :fact ('virtual') the fact to check. The block will only be - # executed if the fact named here matches the value named in the :value - # option. + # @option options [String] :fact ('virtual') the fact to check. The block + # will only be executed if the fact named here matches the value named in the + # :value option. # - # @option options [String] :value ('xenu') the value to check. The block will be - # executed if Facter.value(options[:fact]) matches this value. + # @option options [String] :value ('xenu') the value to check. The block + # will be executed if Facter.value(options[:fact]) matches this value. # # @option options [String] :api_version ('latest') the Amazon AWS API # version. The version string is usually a date, e.g. '2008-02-01'. @@ -77,61 +128,6 @@ def self.with_metadata_server(options = {}, &block) end end - class << self - # Test if we can connect to the EC2 api. Return true if able to connect. - # On failure this function fails silently and returns false. - # - # The +wait_sec+ parameter provides you with an adjustable timeout. - # - def can_connect?(wait_sec=2) - url = "http://169.254.169.254:80/" - Timeout::timeout(wait_sec) {open(url)} - return true - rescue Timeout::Error - return false - rescue - return false - end - - # Test if this host has a mac address used by Eucalyptus clouds, which - # normally is +d0:0d+. - def has_euca_mac? - !!(Facter.value(:macaddress) =~ %r{^[dD]0:0[dD]:}) - end - - # Test if this host has a mac address used by OpenStack, which - # normally starts with FA:16:3E (older versions of OpenStack - # may generate mac addresses starting with 02:16:3E) - def has_openstack_mac? - !!(Facter.value(:macaddress) =~ %r{^(02|[fF][aA]):16:3[eE]}) - end - - # Test if the host has an arp entry in its cache that matches the EC2 arp, - # which is normally +fe:ff:ff:ff:ff:ff+. - def has_ec2_arp? - kernel = Facter.value(:kernel) - - mac_address_re = case kernel - when /Windows/i - /fe-ff-ff-ff-ff-ff/i - else - /fe:ff:ff:ff:ff:ff/i - end - - arp_command = case kernel - when /Windows/i, /SunOS/i - "arp -a" - else - "arp -an" - end - - if arp_table = Facter::Core::Execution.exec(arp_command) - return true if arp_table.match(mac_address_re) - end - return false - end - end - ## # userdata returns a single string containing the body of the response of the # GET request for the URI http://169.254.169.254/latest/user-data/ If the @@ -171,4 +167,24 @@ def self.read_uri(uri) open(uri).read end private_class_method :read_uri + + ## + # add_ec2_facts defines EC2 related facts when running on an EC2 compatible + # node. This method will only ever do work once for the life of a process in + # order to limit the amount of network I/O. + # + # @option options [Boolean] :force (false) whether or not to force + # re-definition of the facts. + def self.add_ec2_facts(options = {}) + opts = options.dup + opts[:force] ||= false + unless opts[:force] + return nil if @add_ec2_facts_has_run + end + @add_ec2_facts_has_run = true + with_metadata_server :timeout => 50 do + define_userdata_fact + define_metadata_facts + end + end end diff --git a/spec/unit/ec2_spec.rb b/spec/unit/ec2_spec.rb index f26a61316f..4a672017a4 100755 --- a/spec/unit/ec2_spec.rb +++ b/spec/unit/ec2_spec.rb @@ -17,171 +17,74 @@ # Assume we can connect Facter::Util::EC2.stubs(:can_connect?).returns(true) - end - - it "should create flat meta-data facts" do - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/"). - at_least_once.returns(StringIO.new("foo")) - - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/foo"). - at_least_once.returns(StringIO.new("bar")) - - Facter.collection.internal_loader.load(:ec2) - Facter.fact(:ec2_foo).value.should == "bar" + # The stubs above this line may not be necessary any + # longer. + Facter::Util::EC2.stubs(:read_uri). + with('http://169.254.169.254').returns('OK') + Facter.stubs(:value). + with('virtual').returns('xenu') end - it "should create flat meta-data facts with comma seperation" do - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/"). - at_least_once.returns(StringIO.new("foo")) - - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/foo"). - at_least_once.returns(StringIO.new("bar\nbaz")) - - Facter.collection.internal_loader.load(:ec2) - - Facter.fact(:ec2_foo).value.should == "bar,baz" + let :util do + Facter::Util::EC2 end - it "should create structured meta-data facts" do - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/"). - at_least_once.returns(StringIO.new("foo/")) - - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/foo/"). - at_least_once.returns(StringIO.new("bar")) + it "defines facts dynamically from meta-data/" do + util.stubs(:read_uri). + with("#{api_prefix}/latest/meta-data/"). + returns("some_key_name") + util.stubs(:read_uri). + with("#{api_prefix}/latest/meta-data/some_key_name"). + at_least_once.returns("some_key_value") - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/foo/bar"). - at_least_once.returns(StringIO.new("baz")) + Facter::Util::EC2.add_ec2_facts(:force => true) - Facter.collection.internal_loader.load(:ec2) - - Facter.fact(:ec2_foo_bar).value.should == "baz" + Facter.fact(:ec2_some_key_name). + value.should == "some_key_value" end - it "should create ec2_user_data fact" do - # No meta-data - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/"). - at_least_once.returns(StringIO.new("")) + it "defines fact values with comma separation" do + util.stubs(:read_uri). + with("#{api_prefix}/latest/meta-data/"). + returns("some_key_name") + util.stubs(:read_uri). + with("#{api_prefix}/latest/meta-data/some_key_name"). + at_least_once.returns("bar\nbaz") - Facter::Util::EC2.stubs(:read_uri). - with("#{api_prefix}/latest/user-data/"). - returns("test") + Facter::Util::EC2.add_ec2_facts(:force => true) - Facter.collection.internal_loader.load(:ec2) - Facter.fact(:ec2_userdata).value.should == ["test"] + Facter.fact(:ec2_some_key_name). + value.should == "bar,baz" end - end - describe "when running on eucalyptus" do - before :each do - # Return false for ec2, true for eucalyptus - Facter::Util::EC2.stubs(:has_euca_mac?).returns(true) - Facter::Util::EC2.stubs(:has_openstack_mac?).returns(false) - Facter::Util::EC2.stubs(:has_ec2_arp?).returns(false) - - # Assume we can connect - Facter::Util::EC2.stubs(:can_connect?).returns(true) - end - - it "should create ec2_user_data fact" do - # No meta-data - Object.any_instance.expects(:open).\ - with("#{api_prefix}/2008-02-01/meta-data/").\ - at_least_once.returns(StringIO.new("")) - - Facter::Util::EC2.stubs(:read_uri). - with("#{api_prefix}/latest/user-data/"). - returns("test") - - # Force a fact load - Facter.collection.internal_loader.load(:ec2) - - Facter.fact(:ec2_userdata).value.should == ["test"] - end - end - - describe "when running on openstack" do - before :each do - # Return false for ec2, true for eucalyptus - Facter::Util::EC2.stubs(:has_openstack_mac?).returns(true) - Facter::Util::EC2.stubs(:has_euca_mac?).returns(false) - Facter::Util::EC2.stubs(:has_ec2_arp?).returns(false) + it "should create structured meta-data facts" do + util.stubs(:read_uri). + with("#{api_prefix}/latest/meta-data/"). + returns("foo/") + util.stubs(:read_uri). + with("#{api_prefix}/latest/meta-data/foo/"). + at_least_once.returns("bar") + util.stubs(:read_uri). + with("#{api_prefix}/latest/meta-data/foo/bar"). + at_least_once.returns("baz") + + Facter::Util::EC2.add_ec2_facts(:force => true) - # Assume we can connect - Facter::Util::EC2.stubs(:can_connect?).returns(true) + Facter.fact(:ec2_foo_bar).value.should == "baz" end - it "should create ec2_user_data fact" do - # No meta-data - Object.any_instance.expects(:open).\ - with("#{api_prefix}/2008-02-01/meta-data/").\ - at_least_once.returns(StringIO.new("")) - - Facter::Util::EC2.stubs(:read_uri). + it "should create ec2_userdata fact" do + util.stubs(:read_uri). + with("#{api_prefix}/latest/meta-data/"). + returns("") + util.stubs(:read_uri). with("#{api_prefix}/latest/user-data/"). - returns("test") + at_least_once.returns("test") - # Force a fact load - Facter.collection.internal_loader.load(:ec2) + Facter::Util::EC2.add_ec2_facts(:force => true) Facter.fact(:ec2_userdata).value.should == ["test"] end - - it "should return nil if open fails" do - Facter.stubs(:warn) # do not pollute test output - Facter.expects(:warn).with('Could not retrieve ec2 metadata: host unreachable') - - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/"). - at_least_once.raises(RuntimeError, 'host unreachable') - - Facter::Util::EC2.stubs(:read_uri). - with("#{api_prefix}/latest/user-data/"). - raises(RuntimeError, 'host unreachable') - - # Force a fact load - Facter.collection.internal_loader.load(:ec2) - - Facter.fact(:ec2_userdata).value.should be_nil - end - - end - - describe "when api connect test fails" do - before :each do - Facter.stubs(:warnonce) - end - - it "should not populate ec2_userdata" do - # Emulate ec2 for now as it matters little to this test - Facter::Util::EC2.stubs(:has_euca_mac?).returns(true) - Facter::Util::EC2.stubs(:has_ec2_arp?).never - Facter::Util::EC2.expects(:can_connect?).at_least_once.returns(false) - - # The API should never be called at this point - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/meta-data/").never - Object.any_instance.expects(:open). - with("#{api_prefix}/2008-02-01/user-data/").never - - # Force a fact load - Facter.collection.internal_loader.load(:ec2) - - Facter.fact(:ec2_userdata).should == nil - end - - it "should rescue the exception" do - Facter::Util::EC2.expects(:open).with("#{api_prefix}:80/").raises(Timeout::Error) - - Facter::Util::EC2.should_not be_can_connect - end end end diff --git a/spec/unit/util/ec2_spec.rb b/spec/unit/util/ec2_spec.rb index 8ba24803f9..2472c8712f 100755 --- a/spec/unit/util/ec2_spec.rb +++ b/spec/unit/util/ec2_spec.rb @@ -8,150 +8,6 @@ # environments. let(:api_prefix) { "http://169.254.169.254" } - describe "is_ec2_arp? method" do - describe "on linux" do - before :each do - # Return fake kernel - Facter.stubs(:value).with(:kernel).returns("linux") - end - - it "should succeed if arp table contains fe:ff:ff:ff:ff:ff" do - ec2arp = my_fixture_read("linux-arp-ec2.out") - Facter::Core::Execution.expects(:exec).with("arp -an").\ - at_least_once.returns(ec2arp) - Facter::Util::EC2.has_ec2_arp?.should == true - end - - it "should succeed if arp table contains FE:FF:FF:FF:FF:FF" do - ec2arp = my_fixture_read("centos-arp-ec2.out") - Facter::Core::Execution.expects(:exec).with("arp -an").\ - at_least_once.returns(ec2arp) - Facter::Util::EC2.has_ec2_arp?.should == true - end - - it "should fail if arp table does not contain fe:ff:ff:ff:ff:ff" do - ec2arp = my_fixture_read("linux-arp-not-ec2.out") - Facter::Core::Execution.expects(:exec).with("arp -an"). - at_least_once.returns(ec2arp) - Facter::Util::EC2.has_ec2_arp?.should == false - end - end - - describe "on windows" do - before :each do - # Return fake kernel - Facter.stubs(:value).with(:kernel).returns("windows") - end - - it "should succeed if arp table contains fe-ff-ff-ff-ff-ff" do - ec2arp = my_fixture_read("windows-2008-arp-a.out") - Facter::Core::Execution.expects(:exec).with("arp -a").\ - at_least_once.returns(ec2arp) - Facter::Util::EC2.has_ec2_arp?.should == true - end - - it "should fail if arp table does not contain fe-ff-ff-ff-ff-ff" do - ec2arp = my_fixture_read("windows-2008-arp-a-not-ec2.out") - Facter::Core::Execution.expects(:exec).with("arp -a"). - at_least_once.returns(ec2arp) - Facter::Util::EC2.has_ec2_arp?.should == false - end - end - - describe "on solaris" do - before :each do - Facter.stubs(:value).with(:kernel).returns("SunOS") - end - - it "should fail if arp table does not contain fe:ff:ff:ff:ff:ff" do - ec2arp = my_fixture_read("solaris8_arp_a_not_ec2.out") - - Facter::Core::Execution.expects(:exec).with("arp -a"). - at_least_once.returns(ec2arp) - - Facter::Util::EC2.has_ec2_arp?.should == false - end - end - end - - describe "is_euca_mac? method" do - it "should return true when the mac is a eucalyptus one" do - Facter.expects(:value).with(:macaddress).\ - at_least_once.returns("d0:0d:1a:b0:a1:00") - - Facter::Util::EC2.has_euca_mac?.should == true - end - - it "should return false when the mac is not a eucalyptus one" do - Facter.expects(:value).with(:macaddress).\ - at_least_once.returns("0c:1d:a0:bc:aa:02") - - Facter::Util::EC2.has_euca_mac?.should == false - end - end - - describe "is_openstack_mac? method" do - it "should return true when the mac is an openstack one" do - Facter.expects(:value).with(:macaddress).\ - at_least_once.returns("02:16:3e:54:89:fd") - - Facter::Util::EC2.has_openstack_mac?.should == true - end - - it "should return true when the mac is a newer openstack mac" do - # https://github.com/openstack/nova/commit/b684d651f540fc512ced58acd5ae2ef4d55a885c#nova/utils.py - Facter.expects(:value).with(:macaddress).\ - at_least_once.returns("fa:16:3e:54:89:fd") - - Facter::Util::EC2.has_openstack_mac?.should == true - end - - it "should return true when the mac is a newer openstack mac and returned in upper case" do - # https://github.com/openstack/nova/commit/b684d651f540fc512ced58acd5ae2ef4d55a885c#nova/utils.py - Facter.expects(:value).with(:macaddress).\ - at_least_once.returns("FA:16:3E:54:89:FD") - - Facter::Util::EC2.has_openstack_mac?.should == true - end - - it "should return false when the mac is not a openstack one" do - Facter.expects(:value).with(:macaddress).\ - at_least_once.returns("0c:1d:a0:bc:aa:02") - - Facter::Util::EC2.has_openstack_mac?.should == false - end - end - - describe "can_connect? method" do - it "returns true if api responds" do - # Return something upon connecting to the root - Module.any_instance.expects(:open).with("#{api_prefix}:80/"). - at_least_once.returns("2008-02-01\nlatest") - - Facter::Util::EC2.can_connect?.should be_true - end - - describe "when connection times out" do - it "should return false" do - # Emulate a timeout when connecting by throwing an exception - Module.any_instance.expects(:open).with("#{api_prefix}:80/"). - at_least_once.raises(RuntimeError) - - Facter::Util::EC2.can_connect?.should be_false - end - end - - describe "when connection is refused" do - it "should return false" do - # Emulate a connection refused - Module.any_instance.expects(:open).with("#{api_prefix}:80/"). - at_least_once.raises(Errno::ECONNREFUSED) - - Facter::Util::EC2.can_connect?.should be_false - end - end - end - describe "Facter::Util::EC2.userdata" do let :not_found_error do OpenURI::HTTPError.new("404 Not Found", StringIO.new) From 6098a1467eb6b390de1ad452c2f53031b2e041ac Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Mon, 24 Mar 2014 14:23:56 -0700 Subject: [PATCH 3/7] (FACT-185) Return ec2 metadata as a structured fact The previous implementation of the EC2 metadata facts could only produce flattened version of the EC2 metadata. Since we can now return structured facts it makes more sense to return the structure data in the original form. --- lib/facter/ec2.rb | 17 +- lib/facter/util/ec2.rb | 217 ++++++--------------- spec/fixtures/unit/util/ec2/meta-data/root | 20 ++ spec/unit/util/ec2_spec.rb | 134 +++++++------ 4 files changed, 179 insertions(+), 209 deletions(-) create mode 100644 spec/fixtures/unit/util/ec2/meta-data/root diff --git a/lib/facter/ec2.rb b/lib/facter/ec2.rb index fefe88a93d..4c5236c649 100644 --- a/lib/facter/ec2.rb +++ b/lib/facter/ec2.rb @@ -1,3 +1,18 @@ require 'facter/util/ec2' -Facter::Util::EC2.add_ec2_facts +Facter.define_fact(:ec2_metadata) do + define_resolution(:rest) do + confine do + Facter.value(:virtual).match /^xen/ + end + + confine do + Facter::Util::EC2.uri_reachable?("http://169.254.169.254/latest/meta-data/") + end + + setcode do + metadata_uri = "http://169.254.169.254/latest/meta-data/" + Facter::Util::EC2.recursive_fetch(metadata_uri) + end + end +end diff --git a/lib/facter/util/ec2.rb b/lib/facter/util/ec2.rb index 30035227d4..36c32f8488 100644 --- a/lib/facter/util/ec2.rb +++ b/lib/facter/util/ec2.rb @@ -3,9 +3,10 @@ # Provide a set of utility static methods that help with resolving the EC2 # fact. +# +# @see http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html module Facter::Util::EC2 CONNECTION_ERRORS = [ - OpenURI::HTTPError, Errno::EHOSTDOWN, Errno::EHOSTUNREACH, Errno::ENETUNREACH, @@ -14,177 +15,87 @@ module Facter::Util::EC2 Errno::ECONNRESET, Errno::ETIMEDOUT, ] - ## - # metadata is a recursive function that walks over the metadata server - # located at http://169.254.169.254 and defines a fact for each value found. - # This method introduces a high amount of latency to Facter, so care must be - # taken to call it only when reasonably certain the host is running in an - # environment where the metadata server is available. - def self.define_metadata_facts(id = "") - begin - if body = read_uri("http://169.254.169.254/latest/meta-data/#{id}") - body.split("\n").each do |o| - key = "#{id}#{o.gsub(/\=.*$/, '/')}" - if key[-1..-1] != '/' - value = read_uri("http://169.254.169.254/latest/meta-data/#{key}").split("\n") - symbol = "ec2_#{key.gsub(/\-|\//, '_')}".to_sym - Facter.add(symbol) { setcode { value.join(',') } } - else - define_metadata_facts(key) - end - end + + # Query a specific AWS metadata URI. + # + # @api private + def self.fetch(uri) + body = open(uri).read + + lines = body.split("\n").map do |line| + if (match = line.match(/^(\d+)=.*$/)) + # Metadata arrays are formatted like '=/', so + # we need to extract the index from that output. + "#{match[1]}/" + else + line end - rescue *CONNECTION_ERRORS => detail - Facter.warn "Could not retrieve ec2 metadata: #{detail.message}" end - end - ## - # define_userdata_fact creates a single fact named 'ec2_userdata' which has a - # value of the contents of the EC2 userdata field. This method introduces a - # high amount of latency to Facter, so care must be taken to call it only - # when reasonably certain the host is running in an environment where the - # metadata server is available. - def self.define_userdata_fact - Facter.add(:ec2_userdata) do - setcode do - if userdata = Facter::Util::EC2.userdata - userdata.split - end - end + lines + rescue OpenURI::HTTPError => e + if e.message.match /404 Not Found/i + return nil + else + Facter.log_exception(e, "Failed to fetch ec2 uri #{uri}: #{e.message}") + return nil end + rescue *CONNECTION_ERRORS => e + Facter.log_exception(e, "Failed to fetch ec2 uri #{uri}: #{e.message}") end - ## - # with_metadata_server takes a block of code and executes the block only if - # Facter is running on node that can access a metadata server at - # http://169.254.168.254/. This is useful to decide if it's reasonably - # likely that talking to the EC2 metadata server will be successful or not. - # - # @option options [Integer] :timeout (100) the maxiumum number of - # milliseconds Facter will block trying to talk to the metadata server. - # Defaults to 200. - # - # @option options [String] :fact ('virtual') the fact to check. The block - # will only be executed if the fact named here matches the value named in the - # :value option. - # - # @option options [String] :value ('xenu') the value to check. The block - # will be executed if Facter.value(options[:fact]) matches this value. - # - # @option options [String] :api_version ('latest') the Amazon AWS API - # version. The version string is usually a date, e.g. '2008-02-01'. - # - # @option options [Fixnum] :retry_limit (3) the maximum number of times that - # this method will try to contact the metadata server. The maximum run time - # is the timeout times this limit, so please keep the value small. - # - # @return [Object] the return value of the passed block, or {false} if the - # block was not executed because the conditions were not met or a timeout - # occurs. - def self.with_metadata_server(options = {}, &block) - opts = options.dup - opts[:timeout] ||= 100 - opts[:fact] ||= 'virtual' - opts[:value] ||= 'xenu' - opts[:api_version] ||= 'latest' - opts[:retry_limit] ||= 3 - # Conversion to fractional seconds for Timeout - timeout = opts[:timeout] / 1000.0 - raise ArgumentError, "A value is required for :fact" if opts[:fact].nil? - raise ArgumentError, "A value is required for :value" if opts[:value].nil? - return false if Facter.value(opts[:fact]) != opts[:value] + def self.recursive_fetch(uri) + results = {} - metadata_base_url = "http://169.254.169.254" + keys = fetch(uri) - attempts = 0 - begin - able_to_connect = false - attempts = attempts + 1 - # Read the list of supported API versions - Timeout.timeout(timeout) do - read_uri(metadata_base_url) + keys.each do |key| + if key.match(%r[/$]) + # If a metadata key is suffixed with '/' then it's a general metadata + # resource, so we have to recursively look up all the keys in the given + # collection. + name = key[0..-2] + results[name] = recursive_fetch("#{uri}#{key}") + else + # This is a simple key/value pair, we can just query the given endpoint + # and store the results. + ret = fetch("#{uri}#{key}") + results[key] = ret.size > 1 ? ret : ret.first end - rescue Timeout::Error => detail - retry if attempts < opts[:retry_limit] - Facter.warn "Timeout exceeded trying to communicate with #{metadata_base_url}, " + - "metadata server facts will be undefined. #{detail.message}" - rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNREFUSED => detail - retry if attempts < opts[:retry_limit] - Facter.warn "No metadata server available at #{metadata_base_url}, " + - "metadata server facts will be undefined. #{detail.message}" - rescue OpenURI::HTTPError => detail - retry if attempts < opts[:retry_limit] - Facter.warn "Metadata server at #{metadata_base_url} responded with an error. " + - "metadata server facts will be undefined. #{detail.message}" - else - able_to_connect = true end - if able_to_connect - return block.call - else - return false - end + results end - ## - # userdata returns a single string containing the body of the response of the - # GET request for the URI http://169.254.169.254/latest/user-data/ If the - # metadata server responds with a 404 Not Found error code then this method - # retuns `nil`. - # - # @param version [String] containing the API version for the request. - # Defaults to "latest" and other examples are documented at - # http://aws.amazon.com/archives/Amazon%20EC2 + # Is the given URI reachable? # - # @api public + # @param uri [String] The HTTP URI to attempt to reach # - # @return [String] containing the response body or `nil` - def self.userdata(version="latest") - uri = "http://169.254.169.254/#{version}/user-data/" + # @return [true, false] If the given URI could be fetched after retry_limit attempts + def self.uri_reachable?(uri, retry_limit = 3) + timeout = 0.2 + able_to_connect = false + attempts = 0 + begin - read_uri(uri) - rescue OpenURI::HTTPError => detail - case detail.message - when /404 Not Found/i - Facter.debug "No user-data present at #{uri}: server responded with #{detail.message}" - return nil + Timeout.timeout(timeout) do + open(uri).read + end + able_to_connect = true + rescue OpenURI::HTTPError => e + if e.message.match /404 Not Found/i + able_to_connect = false else - raise detail + retry if attempts < retry_limit end + rescue Timeout::Error + retry if attempts < retry_limit + rescue *CONNECTION_ERRORS + retry if attempts < retry_limit + ensure + attempts = attempts + 1 end - end - - ## - # read_uri provides a seam method to easily test the HTTP client - # functionality of a HTTP based metadata server. - # - # @api private - # - # @return [String] containing the body of the response - def self.read_uri(uri) - open(uri).read - end - private_class_method :read_uri - ## - # add_ec2_facts defines EC2 related facts when running on an EC2 compatible - # node. This method will only ever do work once for the life of a process in - # order to limit the amount of network I/O. - # - # @option options [Boolean] :force (false) whether or not to force - # re-definition of the facts. - def self.add_ec2_facts(options = {}) - opts = options.dup - opts[:force] ||= false - unless opts[:force] - return nil if @add_ec2_facts_has_run - end - @add_ec2_facts_has_run = true - with_metadata_server :timeout => 50 do - define_userdata_fact - define_metadata_facts - end + able_to_connect end end diff --git a/spec/fixtures/unit/util/ec2/meta-data/root b/spec/fixtures/unit/util/ec2/meta-data/root new file mode 100644 index 0000000000..9ec3bbedad --- /dev/null +++ b/spec/fixtures/unit/util/ec2/meta-data/root @@ -0,0 +1,20 @@ +ami-id +ami-launch-index +ami-manifest-path +block-device-mapping/ +hostname +instance-action +instance-id +instance-type +kernel-id +local-hostname +local-ipv4 +mac +metrics/ +network/ +placement/ +profile +public-hostname +public-ipv4 +public-keys/ +reservation-id diff --git a/spec/unit/util/ec2_spec.rb b/spec/unit/util/ec2_spec.rb index 2472c8712f..201b2beb61 100755 --- a/spec/unit/util/ec2_spec.rb +++ b/spec/unit/util/ec2_spec.rb @@ -1,80 +1,104 @@ -#! /usr/bin/env ruby - require 'spec_helper' require 'facter/util/ec2' describe Facter::Util::EC2 do - # This is the standard prefix for making an API call in EC2 (or fake) - # environments. - let(:api_prefix) { "http://169.254.169.254" } + let(:response) { StringIO.new } + + describe "fetching a uri" do + it "splits the body into an array" do + response.string = my_fixture_read("meta-data/root") + described_class.stubs(:open).with("http://169.254.169.254/latest/meta-data/").returns response + output = described_class.fetch("http://169.254.169.254/latest/meta-data/") - describe "Facter::Util::EC2.userdata" do - let :not_found_error do - OpenURI::HTTPError.new("404 Not Found", StringIO.new) + expect(output).to eq %w[ + ami-id ami-launch-index ami-manifest-path block-device-mapping/ hostname + instance-action instance-id instance-type kernel-id local-hostname + local-ipv4 mac metrics/ network/ placement/ profile public-hostname + public-ipv4 public-keys/ reservation-id + ] end - let :example_userdata do - "owner=jeff@puppetlabs.com\ngroup=platform_team" + it "reformats keys that are array indices" do + response.string = "0=adrien@grey/" + described_class.stubs(:open).with("http://169.254.169.254/latest/meta-data/public-keys/").returns response + output = described_class.fetch("http://169.254.169.254/latest/meta-data/public-keys/") + + expect(output).to eq %w[0/] end - it 'returns nil when no userdata is present' do - Facter::Util::EC2.stubs(:read_uri).raises(not_found_error) - Facter::Util::EC2.userdata.should be_nil + it "returns nil if the endpoint returns a 404" do + described_class.stubs(:open).with("http://169.254.169.254/latest/meta-data/public-keys/1/").raises OpenURI::HTTPError.new("404 Not Found", response) + output = described_class.fetch("http://169.254.169.254/latest/meta-data/public-keys/1/") + + expect(output).to be_nil end - it "returns the string containing the body" do - Facter::Util::EC2.stubs(:read_uri).returns(example_userdata) - Facter::Util::EC2.userdata.should == example_userdata + it "logs an error if the endpoint raises a non-404 HTTPError" do + Facter.expects(:log_exception).with(instance_of(OpenURI::HTTPError), anything) + + described_class.stubs(:open).with("http://169.254.169.254/latest/meta-data/").raises OpenURI::HTTPError.new("418 I'm a Teapot", response) + output = described_class.fetch("http://169.254.169.254/latest/meta-data/") + + expect(output).to be_nil end - it "uses the specified API version" do - expected_uri = "http://169.254.169.254/2008-02-01/user-data/" - Facter::Util::EC2.expects(:read_uri).with(expected_uri).returns(example_userdata) - Facter::Util::EC2.userdata('2008-02-01').should == example_userdata + it "logs an error if the endpoint raises a connection error" do + Facter.expects(:log_exception).with(instance_of(Errno::ECONNREFUSED), anything) + + described_class.stubs(:open).with("http://169.254.169.254/latest/meta-data/").raises Errno::ECONNREFUSED + output = described_class.fetch("http://169.254.169.254/latest/meta-data/") + + expect(output).to be_nil end end - describe "Facter::Util::EC2.with_metadata_server" do - before :each do - Facter::Util::EC2.stubs(:read_uri).returns("latest") + describe "recursively fetching the EC2 metadata API" do + it "queries the given endpoint for metadata keys" do + described_class.expects(:fetch).with("http://169.254.169.254/latest/meta-data/").returns([]) + described_class.recursive_fetch("http://169.254.169.254/latest/meta-data/") + end + + it "fetches the value for a simple metadata key" do + described_class.expects(:fetch).with("http://169.254.169.254/latest/meta-data/").returns(['indexthing']) + described_class.expects(:fetch).with("http://169.254.169.254/latest/meta-data/indexthing").returns(['first', 'second']) + + output = described_class.recursive_fetch("http://169.254.169.254/latest/meta-data/") + expect(output).to eq({'indexthing' => ['first', 'second']}) + end + + it "unwraps metadata values that are in single element arrays" do + described_class.expects(:fetch).with("http://169.254.169.254/latest/meta-data/").returns(['ami-id']) + described_class.expects(:fetch).with("http://169.254.169.254/latest/meta-data/ami-id").returns(['i-12x']) + + output = described_class.recursive_fetch("http://169.254.169.254/latest/meta-data/") + expect(output).to eq({'ami-id' => 'i-12x'}) end - subject do - Facter::Util::EC2.with_metadata_server do - "HELLO FROM THE CODE BLOCK" - end + it "recursively queries an endpoint if the key ends with '/'" do + described_class.expects(:fetch).with("http://169.254.169.254/latest/meta-data/").returns(['metrics/']) + described_class.expects(:fetch).with("http://169.254.169.254/latest/meta-data/metrics/").returns(['vhostmd']) + described_class.expects(:fetch).with("http://169.254.169.254/latest/meta-data/metrics/vhostmd").returns(['woo']) + + output = described_class.recursive_fetch("http://169.254.169.254/latest/meta-data/") + expect(output).to eq({'metrics' => {'vhostmd' => 'woo'}}) + end + end + + describe "determining if a uri is reachable" do + it "retries if the connection times out" do + described_class.stubs(:fetch) + Timeout.expects(:timeout).with(0.2).twice.raises(Timeout::Error).returns(true) + expect(described_class.uri_reachable?("http://169.254.169.254/latest/meta-data/")).to be_true end - it 'returns false when not running on xenu' do - Facter.stubs(:value).with('virtual').returns('vmware') - subject.should be_false + it "retries if the connection is reset" do + described_class.expects(:open).with(anything).twice.raises(Errno::ECONNREFUSED).returns(StringIO.new("woo")) + expect(described_class.uri_reachable?("http://169.254.169.254/latest/meta-data/")).to be_true end - context 'default options and running on a xenu virtual machine' do - before :each do - Facter.stubs(:value).with('virtual').returns('xenu') - end - it 'returns the value of the block when the metadata server responds' do - subject.should == "HELLO FROM THE CODE BLOCK" - end - it 'returns false when the metadata server is unreachable' do - described_class.stubs(:read_uri).raises(Errno::ENETUNREACH) - subject.should be_false - end - it 'does not execute the block if the connection raises an exception' do - described_class.stubs(:read_uri).raises(Timeout::Error) - myvar = "The block didn't get called" - described_class.with_metadata_server do - myvar = "The block was called and should not have been." - end.should be_false - myvar.should == "The block didn't get called" - end - it 'succeeds on the third retry' do - retry_metadata = sequence('metadata') - Timeout.expects(:timeout).twice.in_sequence(retry_metadata).raises(Timeout::Error) - Timeout.expects(:timeout).once.in_sequence(retry_metadata).returns(true) - subject.should == "HELLO FROM THE CODE BLOCK" - end + it "is false if the given uri returns a 404" do + described_class.expects(:open).with(anything).once.raises(OpenURI::HTTPError.new("404 Not Found", StringIO.new("woo"))) + expect(described_class.uri_reachable?("http://169.254.169.254/latest/meta-data/")).to be_false end end end From 83e3dd0e1534ca82dfd811373a17063975289920 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Fri, 28 Mar 2014 13:58:33 -0700 Subject: [PATCH 4/7] (FACT-185) Provide flattened version of EC2 facts To maintain compatibility with the existing EC2 facts, we need to return both a structured version of the EC2 facts as well as flattened versions. This has the unfortunate side effects of forcing fact evaluation at fact load time, but between the options of managing a singleton object, duplicating the API queries, and forcing the evaluation, the last one seems the least bad. --- lib/facter/ec2.rb | 9 +++ lib/facter/util/values.rb | 29 ++++++++ spec/unit/ec2_spec.rb | 124 +++++++++++++++------------------- spec/unit/util/values_spec.rb | 40 +++++++++++ 4 files changed, 131 insertions(+), 71 deletions(-) diff --git a/lib/facter/ec2.rb b/lib/facter/ec2.rb index 4c5236c649..e0e22583d9 100644 --- a/lib/facter/ec2.rb +++ b/lib/facter/ec2.rb @@ -16,3 +16,12 @@ end end end + +# The flattened version of the EC2 facts are deprecated and will be removed in +# a future release of Facter. +if (ec2_metadata = Facter.value(:ec2_metadata)) + ec2_facts = Facter::Util::Values.flatten_structure("ec2", ec2_metadata) + ec2_facts.each_pair do |factname, factvalue| + Facter.add(factname, :value => factvalue) + end +end diff --git a/lib/facter/util/values.rb b/lib/facter/util/values.rb index a7048d54da..1fbb1b686e 100644 --- a/lib/facter/util/values.rb +++ b/lib/facter/util/values.rb @@ -1,3 +1,4 @@ + module Facter module Util # A util module for facter containing helper methods @@ -75,6 +76,34 @@ def convert(value) value = value.downcase if value.is_a?(String) value end + + # Flatten the given data structure to something that's suitable to return + # as flat facts. + # + # @param path [String] The fact path to be prefixed to the given value. + # @param structure [Object] The data structure to flatten. Nested hashes + # will be recursively flattened, everything else will be returned as-is. + # + # @return [Hash] The given data structure prefixed with the given path + def flatten_structure(path, structure) + results = {} + + if structure.is_a? Hash + structure.each_pair do |name, value| + new_path = "#{path}_#{name}".gsub(/\-|\//, '_') + results.merge! flatten_structure(new_path, value) + end + elsif structure.is_a? Array + structure.each_with_index do |value, index| + new_path = "#{path}_#{index}" + results.merge! flatten_structure(new_path, value) + end + else + results[path] = structure + end + + results + end end end end diff --git a/spec/unit/ec2_spec.rb b/spec/unit/ec2_spec.rb index 4a672017a4..948a02ebeb 100755 --- a/spec/unit/ec2_spec.rb +++ b/spec/unit/ec2_spec.rb @@ -1,90 +1,72 @@ -#! /usr/bin/env ruby - require 'spec_helper' require 'facter/util/ec2' -describe "ec2 facts" do - # This is the standard prefix for making an API call in EC2 (or fake) - # environments. - let(:api_prefix) { "http://169.254.169.254" } - - describe "when running on ec2" do - before :each do - # This is an ec2 instance, not a eucalyptus instance - Facter::Util::EC2.stubs(:has_euca_mac?).returns(false) - Facter::Util::EC2.stubs(:has_openstack_mac?).returns(false) - Facter::Util::EC2.stubs(:has_ec2_arp?).returns(true) - - # Assume we can connect - Facter::Util::EC2.stubs(:can_connect?).returns(true) - - # The stubs above this line may not be necessary any - # longer. - Facter::Util::EC2.stubs(:read_uri). - with('http://169.254.169.254').returns('OK') - Facter.stubs(:value). - with('virtual').returns('xenu') - end +describe "ec2_metadata" do + before do + Facter.collection.internal_loader.load(:ec2) + end - let :util do - Facter::Util::EC2 - end + subject { Facter.fact(:ec2_metadata).resolution(:rest) } - it "defines facts dynamically from meta-data/" do - util.stubs(:read_uri). - with("#{api_prefix}/latest/meta-data/"). - returns("some_key_name") - util.stubs(:read_uri). - with("#{api_prefix}/latest/meta-data/some_key_name"). - at_least_once.returns("some_key_value") + it "is unsuitable if the virtual fact is not xen" do + Facter.fact(:virtual).stubs(:value).returns "kvm" + Facter::Util::EC2.stubs(:uri_reachable?).returns true + expect(subject).to_not be_suitable + end - Facter::Util::EC2.add_ec2_facts(:force => true) + it "is unsuitable if ec2 endpoint is not reachable" do + Facter.fact(:virtual).stubs(:value).returns "xen" + Facter::Util::EC2.stubs(:uri_reachable?).returns false + expect(subject).to_not be_suitable + end - Facter.fact(:ec2_some_key_name). - value.should == "some_key_value" + describe "when the ec2 endpoint is reachable" do + before do + Facter::Util::EC2.stubs(:uri_reachable?).returns true end - it "defines fact values with comma separation" do - util.stubs(:read_uri). - with("#{api_prefix}/latest/meta-data/"). - returns("some_key_name") - util.stubs(:read_uri). - with("#{api_prefix}/latest/meta-data/some_key_name"). - at_least_once.returns("bar\nbaz") - - Facter::Util::EC2.add_ec2_facts(:force => true) + it "is suitable if the virtual fact is xen" do + Facter.fact(:virtual).stubs(:value).returns "xen" + expect(subject).to be_suitable + end - Facter.fact(:ec2_some_key_name). - value.should == "bar,baz" + it "is suitable if the virtual fact is xenu" do + Facter.fact(:virtual).stubs(:value).returns "xenu" + expect(subject).to be_suitable end + end - it "should create structured meta-data facts" do - util.stubs(:read_uri). - with("#{api_prefix}/latest/meta-data/"). - returns("foo/") - util.stubs(:read_uri). - with("#{api_prefix}/latest/meta-data/foo/"). - at_least_once.returns("bar") - util.stubs(:read_uri). - with("#{api_prefix}/latest/meta-data/foo/bar"). - at_least_once.returns("baz") + let(:metadata_root) { 'http://169.254.169.254/latest/meta-data/' } - Facter::Util::EC2.add_ec2_facts(:force => true) + it "resolves the value by recursively querying the rest endpoint" do + Facter::Util::EC2.expects(:recursive_fetch).with(metadata_root).returns({"hello" => "world"}) + expect(subject.value).to eq({"hello" => "world"}) + end +end - Facter.fact(:ec2_foo_bar).value.should == "baz" - end +describe "flattened versions of ec2 facts" do + # These facts are tricky to test because they are dynamic facts, and they are + # generated from a fact that is defined in the same file. In order to pull + # this off we need to define the ec2_metadata fact ahead of time so that we + # can stub the value, and then manually load the correct files. - it "should create ec2_userdata fact" do - util.stubs(:read_uri). - with("#{api_prefix}/latest/meta-data/"). - returns("") - util.stubs(:read_uri). - with("#{api_prefix}/latest/user-data/"). - at_least_once.returns("test") + it "unpacks the ec2_metadata fact" do + Facter.define_fact(:ec2_metadata).stubs(:value).returns({"hello" => "world"}) + Facter.collection.internal_loader.load(:ec2) - Facter::Util::EC2.add_ec2_facts(:force => true) + expect(Facter.value("ec2_hello")).to eq "world" + end - Facter.fact(:ec2_userdata).value.should == ["test"] - end + it "does not set any flat ec2 facts if the ec2_metadata fact is nil" do + Facter.define_fact(:ec2_metadata).stubs(:value) + Facter.define_fact(:ec2_userdata).stubs(:value).returns(nil) + + Facter.collection.internal_loader.load(:ec2) + + all_facts = Facter.collection.to_hash + + ec2_facts = all_facts.keys.select { |k| k =~ /^ec2_/ } + expect(ec2_facts).to be_empty end + end diff --git a/spec/unit/util/values_spec.rb b/spec/unit/util/values_spec.rb index 557eb195e6..bc347c49a6 100644 --- a/spec/unit/util/values_spec.rb +++ b/spec/unit/util/values_spec.rb @@ -128,4 +128,44 @@ end end end + + describe "flatten_structure" do + it "converts a string to a hash containing that string" do + input = "foo" + output = described_class.flatten_structure("path", input) + expect(output).to eq({"path" => "foo"}) + end + + it "converts an array to a hash with the array elements with indexes" do + input = ["foo"] + output = described_class.flatten_structure("path", input) + expect(output).to eq({"path_0" => "foo"}) + end + + it "prefixes a non-nested hash with the given path" do + input = {"foo" => "bar"} + output = described_class.flatten_structure("path", input) + expect(output).to eq({"path_foo" => "bar"}) + end + + it "flattens elements till it reaches the first non-flattenable structure" do + input = { + "first" => "second", + "arr" => ["zero", "one"], + "nested_array" => [ + "hash" => "string", + ], + "top" => {"middle" => ['bottom']}, + } + output = described_class.flatten_structure("path", input) + + expect(output).to eq({ + "path_first" => "second", + "path_arr_0" => "zero", + "path_arr_1" => "one", + "path_nested_array_0_hash" => "string", + "path_top_middle_0" => "bottom" + }) + end + end end From 69462827ced771b7dd49a8447692fc78210243a3 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Mon, 31 Mar 2014 10:39:33 -0700 Subject: [PATCH 5/7] (maint) Don't cache suitability information When testing a resolution, the confines may be tested to confirm suitability on a given platform and so may be changed without destroying and recreating the resolution. Because of this we need to recalculate suitability every time #suitable? method is invoked. In normal operation fact suitability is only determined once so this should have no performance impact. --- lib/facter/core/suitable.rb | 6 +----- spec/unit/core/suitable_spec.rb | 10 ++++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/facter/core/suitable.rb b/lib/facter/core/suitable.rb index 1b3c04fceb..0262470919 100644 --- a/lib/facter/core/suitable.rb +++ b/lib/facter/core/suitable.rb @@ -108,10 +108,6 @@ def weight # # @api private def suitable? - unless defined? @suitable - @suitable = ! @confines.detect { |confine| ! confine.true? } - end - - return @suitable + @confines.all? { |confine| confine.true? } end end diff --git a/spec/unit/core/suitable_spec.rb b/spec/unit/core/suitable_spec.rb index 4c0b1fde06..8277408799 100644 --- a/spec/unit/core/suitable_spec.rb +++ b/spec/unit/core/suitable_spec.rb @@ -92,5 +92,15 @@ def initialize expect(subject).to_not be_suitable end + + it "recalculates suitability on every invocation" do + subject.confine :kernel => 'Linux' + + subject.confines.first.stubs(:true?).returns false + expect(subject).to_not be_suitable + subject.confines.first.unstub(:true?) + subject.confines.first.stubs(:true?).returns true + expect(subject).to be_suitable + end end end From 9468e6cf024f70172ada7f56d8497a6da8c18f22 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Mon, 31 Mar 2014 11:40:53 -0700 Subject: [PATCH 6/7] (FACT-185) Reimplement EC2 userdata fact --- lib/facter/ec2.rb | 18 +++++++++++++++ spec/unit/ec2_spec.rb | 54 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/lib/facter/ec2.rb b/lib/facter/ec2.rb index e0e22583d9..7afd97d86f 100644 --- a/lib/facter/ec2.rb +++ b/lib/facter/ec2.rb @@ -17,6 +17,24 @@ end end +Facter.define_fact(:ec2_userdata) do + define_resolution(:rest) do + confine do + Facter.value(:virtual).match /^xen/ + end + + confine do + Facter::Util::EC2.uri_reachable?("http://169.254.169.254/latest/user-data/") + end + + setcode do + userdata_uri = "http://169.254.169.254/latest/user-data/" + output = Facter::Util::EC2.fetch(userdata_uri) + output.join("\n") + end + end +end + # The flattened version of the EC2 facts are deprecated and will be removed in # a future release of Facter. if (ec2_metadata = Facter.value(:ec2_metadata)) diff --git a/spec/unit/ec2_spec.rb b/spec/unit/ec2_spec.rb index 948a02ebeb..a206e898af 100755 --- a/spec/unit/ec2_spec.rb +++ b/spec/unit/ec2_spec.rb @@ -44,6 +44,60 @@ end end +describe "ec2_userdata" do + before do + Facter.collection.internal_loader.load(:ec2) + end + + subject { Facter.fact(:ec2_userdata).resolution(:rest) } + + it "is unsuitable if the virtual fact is not xen" do + Facter.fact(:virtual).stubs(:value).returns "kvm" + Facter::Util::EC2.stubs(:uri_reachable?).returns true + expect(subject).to_not be_suitable + end + + it "is unsuitable if ec2 endpoint is not reachable" do + Facter.fact(:virtual).stubs(:value).returns "xen" + Facter::Util::EC2.stubs(:uri_reachable?).returns false + expect(subject).to_not be_suitable + end + + describe "when the ec2 endpoint is reachable" do + before do + Facter::Util::EC2.stubs(:uri_reachable?).returns true + end + + it "is suitable if the virtual fact is xen" do + Facter.fact(:virtual).stubs(:value).returns "xen" + expect(subject).to be_suitable + end + + it "is suitable if the virtual fact is xenu" do + Facter.fact(:virtual).stubs(:value).returns "xenu" + expect(subject).to be_suitable + end + end + + let(:userdata_uri) { 'http://169.254.169.254/latest/user-data/' } + + it "resolves the value by fetching the rest endpoint" do + Facter::Util::EC2.expects(:fetch).with(userdata_uri).returns(["my ec2 userdata"]) + expect(subject.value).to eq "my ec2 userdata" + end + + it "returns the user data as a single string" do + userdata = [ + "my multiline", + "user data", + "split by fetch" + ] + + Facter::Util::EC2.expects(:fetch).with(userdata_uri).returns(userdata) + expect(subject.value).to eq "my multiline\nuser data\nsplit by fetch" + end +end + describe "flattened versions of ec2 facts" do # These facts are tricky to test because they are dynamic facts, and they are # generated from a fact that is defined in the same file. In order to pull From 6d33636ff6cdc5e3fc6db26cbc7ae485c11a3da2 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Mon, 31 Mar 2014 11:47:18 -0700 Subject: [PATCH 7/7] (maint) Define Util::Resolution#resolution_type The resolution_type method is needed when reopening a fact definition with the Facter 2.0 API: Facter.define_fact(:myfact) do define_resolution(:myres) do # [...] The aggregate resolution type implemented this but Util::Resolution did not, which could cause errors using the new syntax. This corrects the omission. --- lib/facter/util/resolution.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/facter/util/resolution.rb b/lib/facter/util/resolution.rb index 0182a9ae63..9b3efa097f 100644 --- a/lib/facter/util/resolution.rb +++ b/lib/facter/util/resolution.rb @@ -62,6 +62,10 @@ def initialize(name, fact) @weight = nil end + def resolution_type + :simple + end + # Evaluate the given block in the context of this resolution. If a block has # already been evaluated emit a warning to that effect. #