Add support for a hiera variable syntax which interpolates data by

performing a hiera lookup:

[Redmine Ticket](https://projects.puppetlabs.com/issues/21367)

This commit adds support for interpolating hiera data in a similar fashion to
how scope interpolation functions in hiera. This commit adds the syntax,
`%{hiera('foo')}`, for hiera lookups and the syntax, `%{scope('foo')}`, for
scope lookups, in order to clearly differentiate the two types. It also retains
backward compatibility with the previous syntax for scope lookups, `%{foo}`.

Example:

  ips.yaml

      potto01_ip: 10.10.1.52

  potto01.yaml

      firewall_rules:
        - "0.0.0.0:22:%{hiera('potto01_ip')}"

When evaluating the string, `"0.0.0.0:22:%{hiera('potto01_ip')}"` hiera would
lookup the value for potto01_ip in hiera and interpolate that into the string,
the result being, `"0.0.0.0:22:10.10.1.52"`. This avoids having to repeatedly
perform this type of lookup logic in puppet and allows you to use your hiera
data from within hiera itself.

lib/hiera/backend.rb

@@ -1,5 +1,5 @@
 require 'hiera/util'
-require 'hiera/recursive_lookup'
+require 'hiera/recursive_guard'
 
 begin
   require 'deep_merge'
@@ -9,6 +9,9 @@
 class Hiera
   module Backend
     INTERPOLATION = /%\{([^\}]*)\}/
+    SCOPE_INTERPOLATION = /%\{scope\(['"]([^\}]*)["']\)\}/
+    HIERA_INTERPOLATION = /%\{hiera\(['"]([^\}]*)["']\)\}/
+    INTERPOLATION_TYPE = /^([^\(]+)\(/
 
     class << self
       # Data lives in /var/lib/hiera by default.  If a backend
@@ -85,23 +88,64 @@ def datasources(scope, override=nil, hierarchy=nil)
       #
       # @api public
       def parse_string(data, scope, extra_data={})
-        interpolate(data, Hiera::RecursiveLookup.new(scope, extra_data))
+        interpolate(data, Hiera::RecursiveGuard.new, scope, extra_data)
       end
 
-      def interpolate(data, values)
-        if data.is_a?(String)
-          data.gsub(INTERPOLATION) do
-            name = $1
-            values.lookup(name) do |value|
-              interpolate(value, values)
-            end
+      def interpolate(data, recurse_guard, scope, extra_data)
+        if data =~ INTERPOLATION
+          interpolation_variable = $1
+          recurse_guard.check(interpolation_variable) do
+            interpolate_method = get_interpolation_method(interpolation_variable)
+            interpolated_data = send(interpolate_method, data, scope, extra_data)
+            interpolate(interpolated_data, recurse_guard, scope, extra_data)
           end
         else
           data
         end
       end
       private :interpolate
 
+      def get_interpolation_method(interpolation_variable)
+        interpolation_type = interpolation_variable.match(INTERPOLATION_TYPE)
+        if interpolation_type
+          case interpolation_type[1]
+          when 'hiera' then :hiera_interpolate
+          when 'scope' then :scope_interpolate
+          end
+        else
+          :scope_interpolate
+        end
+      end
+
+      def get_scope_value(data)
+        if data =~ SCOPE_INTERPOLATION
+          data.match(SCOPE_INTERPOLATION)[1]
+        elsif data =~ INTERPOLATION
+          data.match(INTERPOLATION)[1]
+        end
+      end
+      private :get_scope_value
+
+      def scope_interpolate(data, scope, extra_data)
+        data.sub(INTERPOLATION) do
+          value = get_scope_value(data)
+          scope_val = scope[value]
+          if scope_val.nil? || scope_val == :undefined
+            scope_val = extra_data[value]
+          end
+          scope_val
+        end
+      end
+      private :scope_interpolate
+
+      def hiera_interpolate(data, scope, extra_data)
+        data.sub(HIERA_INTERPOLATION) do
+          value = $1
+          lookup(value, nil, scope, nil, :priority)
+        end
+      end
+      private :hiera_interpolate
+
       # Parses a answer received from data files
       #
       # Ultimately it just pass the data through parse_string but

lib/hiera/recursive_guard.rb

@@ -0,0 +1,18 @@
+# Allow for safe recursive lookup of values during variable interpolation.
+#
+# @api private
+class Hiera::RecursiveGuard
+  def initialize
+    @seen = []
+  end
+
+  def check(value, &block)
+    if @seen.include?(value)
+      raise Exception, "Interpolation loop detected in [#{@seen.join(', ')}]"
+    end
+    @seen.push(value)
+    ret = yield
+    @seen.pop
+    ret
+  end
+end

spec/unit/backend_spec.rb

@@ -105,11 +105,19 @@ class Hiera
         Backend.parse_string(input, {}).should == input
       end
 
-      it "replaces interpolations with data looked up in the scope" do
-        input = "replace %{part1} and %{part2}"
-        scope = {"part1" => "value of part1", "part2" => "value of part2"}
-
-        Backend.parse_string(input, scope).should == "replace value of part1 and value of part2"
+      @scope_interpolation_tests = {
+        "replace %{part1} and %{part2}" =>
+          "replace value of part1 and value of part2",
+        "replace %{scope('part1')} and %{scope('part2')}" =>
+          "replace value of part1 and value of part2"
+      }
+
+      @scope_interpolation_tests.each do |input, expected|
+        it "replaces interpolations with data looked up in the scope" do
+          scope = {"part1" => "value of part1", "part2" => "value of part2"}
+
+          Backend.parse_string(input, scope).should == expected
+        end
       end
 
       it "replaces interpolations with data looked up in extra_data when scope does not contain the value" do
@@ -122,14 +130,27 @@ class Hiera
         Backend.parse_string(input, {"rspec" => "test"}, {"rspec" => "fail"}).should == "test_test_test"
       end
 
-      it "interprets nil in scope as a non-value" do
-        input = "test_%{rspec}_test"
-        Backend.parse_string(input, {"rspec" => nil}).should == "test__test"
+      @interprets_nil_in_scope_tests = {
+        "test_%{rspec}_test" => "test__test",
+        "test_%{scope('rspec')}_test" => "test__test"
+      }
+
+      @interprets_nil_in_scope_tests.each do |input, expected|
+        it "interprets nil in scope as a non-value" do
+          Backend.parse_string(input, {"rspec" => nil}).should == expected
+        end
       end
 
-      it "interprets false in scope as a real value" do
-        input = "test_%{rspec}_test"
-        Backend.parse_string(input, {"rspec" => false}).should == "test_false_test"
+      @intreprets_false_in_scope_tests = {
+        "test_%{rspec}_test" => "test_false_test",
+        "test_%{scope('rspec')}_test" => "test_false_test"
+      }
+
+      @intreprets_false_in_scope_tests.each do |input, expected|
+        it "interprets false in scope as a real value" do
+          input = "test_%{scope('rspec')}_test"
+          Backend.parse_string(input, {"rspec" => false}).should == expected
+        end
       end
 
       it "interprets false in extra_data as a real value" do
@@ -142,34 +163,62 @@ class Hiera
         Backend.parse_string(input, {}, {"rspec" => nil}).should == "test__test"
       end
 
-      it "interprets :undefined in scope as a non-value" do
-        input = "test_%{rspec}_test"
-        Backend.parse_string(input, {"rspec" => :undefined}).should == "test__test"
+      @interprets_undefined_in_scope_tests = {
+        "test_%{rspec}_test" => "test__test",
+        "test_%{scope('rspec')}_test" => "test__test"
+      }
+
+      @interprets_undefined_in_scope_tests.each do |input, expected|
+        it "interprets :undefined in scope as a non-value" do
+          Backend.parse_string(input, {"rspec" => :undefined}).should == expected
+        end
       end
 
       it "uses the value from extra_data when scope is :undefined" do
         input = "test_%{rspec}_test"
         Backend.parse_string(input, {"rspec" => :undefined}, { "rspec" => "extra" }).should == "test_extra_test"
       end
 
-      it "looks up the interpolated value exactly as it appears in the input" do
-        input = "test_%{::rspec::data}_test"
-        Backend.parse_string(input, {"::rspec::data" => "value"}).should == "test_value_test"
+      @exact_lookup_tests = {
+        "test_%{::rspec::data}_test" => "test_value_test",
+        "test_%{scope('::rspec::data')}_test" => "test_value_test"
+      }
+
+      @exact_lookup_tests.each do |input, expected|
+        it "looks up the interpolated value exactly as it appears in the input" do
+          Backend.parse_string(input, {"::rspec::data" => "value"}).should == expected
+        end
       end
 
-      it "does not remove any surrounding whitespace when parsing the key to lookup" do
-        input = "test_%{\trspec::data }_test"
-        Backend.parse_string(input, {"\trspec::data " => "value"}).should == "test_value_test"
+      @surrounding_whitespace_tests = {
+        "test_%{\trspec::data }_test" => "test_value_test",
+        "test_%{scope('\trspec::data ')}_test" => "test_value_test"
+      }
+      @surrounding_whitespace_tests.each do |input, expected|
+        it "does not remove any surrounding whitespace when parsing the key to lookup" do
+          Backend.parse_string(input, {"\trspec::data " => "value"}).should == expected
+        end
       end
 
-      it "does not try removing leading :: when a full lookup fails (#17434)" do
-        input = "test_%{::rspec::data}_test"
-        Backend.parse_string(input, {"rspec::data" => "value"}).should == "test__test"
+      @leading_double_colon_tests = {
+        "test_%{::rspec::data}_test" => "test__test",
+        "test_%{scope('::rspec::data')}_test" => "test__test"
+      }
+
+      @leading_double_colon_tests.each do |input, expected|
+        it "does not try removing leading :: when a full lookup fails (#17434)" do
+          Backend.parse_string(input, {"rspec::data" => "value"}).should == expected
+        end
       end
 
-      it "does not try removing leading sections separated by :: when a full lookup fails (#17434)" do
-        input = "test_%{::rspec::data}_test"
-        Backend.parse_string(input, {"data" => "value"}).should == "test__test"
+      @double_colon_key_tests = {
+        "test_%{::rspec::data}_test" => "test__test",
+        "test_%{scope('::rspec::data')}_test" => "test__test"
+      }
+      @double_colon_key_tests.each do |input, expected|
+        it "does not try removing leading sections separated by :: when a full lookup fails (#17434)" do
+          Backend.parse_string(input, {"data" => "value"}).should == expected
+        end
       end
 
       it "looks up recursively" do
@@ -185,6 +234,16 @@ class Hiera
           Backend.parse_string(input, scope)
         end.to raise_error Exception, "Interpolation loop detected in [first, second]"
       end
+
+      it "replaces hiera interpolations with data looked up in hiera" do
+        input = "%{hiera('key1')}"
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("key1", scope, nil, :priority).returns("answer")
+
+        Backend.parse_string(input, scope).should == "answer"
+      end
     end
 
     describe "#parse_answer" do
@@ -218,6 +277,78 @@ class Hiera
         Backend.parse_answer(input, {"rspec" => "test"}).should == {"foo"=>"test_test_test", "bar"=>["test_test_test", "test_test_test"]}
       end
 
+      it "interpolates hiera lookups values in strings" do
+        input = "test_%{hiera('rspec')}_test"
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("test")
+        Backend.parse_answer(input, scope).should == "test_test_test"
+      end
+
+      it "interpolates hiera lookups in each string in an array" do
+        input = ["test_%{hiera('rspec')}_test", "test_%{hiera('rspec')}_test", ["test_%{hiera('rspec')}_test"]]
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("test")
+        Backend.parse_answer(input, scope).should == ["test_test_test", "test_test_test", ["test_test_test"]]
+      end
+
+      it "interpolates hiera lookups in each string in a hash" do
+        input = {"foo" => "test_%{hiera('rspec')}_test", "bar" => "test_%{hiera('rspec')}_test"}
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("test")
+        Backend.parse_answer(input, scope).should == {"foo"=>"test_test_test", "bar"=>"test_test_test"}
+      end
+
+      it "interpolates hiera lookups in string in hash keys" do
+        input = {"%{hiera('rspec')}" => "test"}
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("foo")
+        Backend.parse_answer(input, scope).should == {"foo"=>"test"}
+      end
+
+      it "interpolates hiera lookups in strings in nested hash keys" do
+        input = {"topkey" => {"%{hiera('rspec')}" => "test"}}
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("foo")
+        Backend.parse_answer(input, scope).should == {"topkey"=>{"foo" => "test"}}
+      end
+
+      it "interpolates hiera lookups in strings in a mixed structure of arrays and hashes" do
+        input = {"foo" => "test_%{hiera('rspec')}_test", "bar" => ["test_%{hiera('rspec')}_test", "test_%{hiera('rspec')}_test"]}
+        scope = {}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("test")
+        Backend.parse_answer(input, scope).should == {"foo"=>"test_test_test", "bar"=>["test_test_test", "test_test_test"]}
+      end
+
+      it "interpolates hiera lookups and scope lookups in the same string" do
+        input = {"foo" => "test_%{hiera('rspec')}_test", "bar" => "test_%{rspec2}_test"}
+        scope = {"rspec2" => "scope_rspec"}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("hiera_rspec")
+        Backend.parse_answer(input, scope).should == {"foo"=>"test_hiera_rspec_test", "bar"=>"test_scope_rspec_test"}
+      end
+
+      it "interpolates hiera and scope lookups with the same lookup query in a single string" do
+        input =  "test_%{hiera('rspec')}_test_%{rspec}"
+        scope = {"rspec" => "scope_rspec"}
+        Config.load({:yaml => {:datadir => "/tmp"}})
+        Config.load_backends
+        Backend::Yaml_backend.any_instance.stubs(:lookup).with("rspec", scope, nil, :priority).returns("hiera_rspec")
+        Backend.parse_answer(input, scope).should == "test_hiera_rspec_test_scope_rspec"
+      end
+
       it "passes integers unchanged" do
         input = 1
         Backend.parse_answer(input, {"rspec" => "test"}).should == 1
@@ -237,6 +368,12 @@ class Hiera
         input = false
         Backend.parse_answer(input, {"rspec" => "test"}).should == false
       end
+
+      it "interpolates lookups using single or double quotes" do
+        input =  "test_%{scope(\"rspec\")}_test_%{scope('rspec')}"
+        scope = {"rspec" => "scope_rspec"}
+        Backend.parse_answer(input, scope).should == "test_scope_rspec_test_scope_rspec"
+      end
     end
 
     describe "#resolve_answer" do