substitute variables also on datadir path

By providing the possibility that datadir can also contain
variables we enable true per enviornment data lookup.

Given that you use environments for example to test your manifests
in different versions, up to now you could still have only one
datadir path.
However, if you use extlookup in combination with that kind of
environments you might want to have your configuration data aside
your modules in the same repository. While having your code (and
data) being checked out in different versions as multiple
environments, extlookup would still lookup your data from the same
source represented by only one version.

For sure you could precedence your data by the current environment
and thus simulating some kind of data versioning. But this method
is quite cumbersome and not treating data the same way as your code.

You can now use variables in datadir the same way you can use them
in the precedences:

  :datadir: /etc/puppet/environments/%{environment}/extdata

This enables also other possibilities, such as having different data
sources by customers to which they could have write access.

README.md

@@ -67,6 +67,16 @@ data it wont touch those.
 This configuration matches the above CSV configuration, all you
 need to do is create yaml files instead of CSV files.
 
+Both (YAML and CSV) support variables also in the datadir option,
+so that for example you are able to point puppet to different
+datadirs depending on the current environment:
+
+<pre>
+[...]
+:yaml:
+    :datadir: /etc/puppet/environments/%{environment}/extdata
+</pre>
+
 Puppet
 ------
 

lib/puppet/util/extlookup.rb

@@ -2,15 +2,14 @@ module Puppet
     module Util
         module Extlookup
             class << self
-                def datadir(config, key, dirkey)
+                def datadir(config, key, dirkey, store)
                     config = config[key] || {}
 
-                    if config[dirkey]
-                        datadir = config[dirkey]
-                    else
-                        datadir = File.join(File.dirname(Puppet.settings[:config]), "extdata")
-                        Puppet.notice("extlookup/#{key}: Using #{datadir} for extlookup data as no datadir is configured")
+                    unless (datadirpath = config[dirkey])
+                        datadirpath = File.join(File.dirname(Puppet.settings[:config]), "extdata")
+                        Puppet.notice("extlookup/#{key}: Using #{datadirpath} for extlookup data as no datadir is configured")
                     end
+                    datadir = parse_data_contents(datadirpath,store)
 
                     raise(Puppet::ParseError, "Extlookup datadir (#{datadir}) not found") unless File.directory?(datadir)
 

lib/puppet/util/extlookup/csv_parser.rb

@@ -14,12 +14,11 @@ def initialize(default, override, config, scope)
                 def backward_compat_datadir
                     config = @config[:csv] || {}
 
-                    if config[:datadir]
-                        datadir = config[:datadir]
-                    else
-                        datadir = File.join(File.dirname(Puppet.settings[:config]), "extdata")
-                        Puppet.notice("extlookup/csv: Using #{datadir} for extlookup CSV data as no datadir is configured")
+                    unless (datadirpath = config[:datadir])
+                        datadirpath = File.join(File.dirname(Puppet.settings[:config]), "extdata")
+                        Puppet.notice("extlookup/csv: Using #{datadirpath} for extlookup CSV data as no datadir is configured")
                     end
+                    datadir = Extlookup.parse_data_contents(datadirpath, @scope)
 
                     if @scope.respond_to?(:lookupvar)
                         scope_dir = @scope.lookupvar("extlookup_datadir")

lib/puppet/util/extlookup/yaml_parser.rb

@@ -17,7 +17,7 @@ def lookup(key)
 
                     raise(Puppet::ParseError, "Extlookup YAML backend is unconfigured") unless @config.include?(:yaml)
 
-                    datadir = Extlookup.datadir(@config, :yaml, :datadir)
+                    datadir = Extlookup.datadir(@config, :yaml, :datadir,@scope)
 
                     Extlookup.datasources(@config, @override, precedence) do |source|
                         source = Extlookup.parse_data_contents(source, @scope)

spec/unit/util/extlookup/csv_parser_spec.rb

@@ -50,6 +50,19 @@ module Puppet::Util::Extlookup
                     csv.backward_compat_datadir
                 }.to raise_error("Extlookup CSV datadir (/nonexisting) not found")
             end
+
+            it "should substitute variables in datadir" do
+                config = mock
+                config.expects("[]").with(:csv).returns({:datadir => "/some/path/with/variables/%{test}/in/path"})
+                scope = mock
+                scope.expects(:lookupvar).with("test").returns("data")
+                scope.stubs(:respond_to?).with(:lookupvar).returns(true)
+                scope.expects(:lookupvar).with('extlookup_datadir').returns("")
+                File.expects(:directory?).with("/some/path/with/variables/data/in/path").returns(true)
+
+                csv = CSV_Parser.new(nil, nil, config, scope)
+                csv.backward_compat_datadir.should == "/some/path/with/variables/data/in/path"
+            end
         end
 
         describe "#parse_csv" do

spec/unit/util/extlookup_spec.rb

@@ -8,28 +8,45 @@ module Puppet::Util
             it "should default to configured data dir" do
                 config = mock
                 config.expects("[]").with(:csv).returns({:datadir => "/tmp"})
+                store = mock
+                store.expects(:lookupvar).never
 
-                Puppet::Util::Extlookup.datadir(config, :csv, :datadir).should == "/tmp"
+                Puppet::Util::Extlookup.datadir(config, :csv, :datadir,store).should == "/tmp"
             end
 
             it "should default to Puppet configdir + extdata for data if unconfigured" do
                 config = mock
                 config.expects("[]").with(:csv).returns({})
+                store = mock
+                store.expects(:lookupvar).never
 
                 Puppet.expects(:settings).returns({:config => "/tmp/puppet.conf"})
                 File.expects(:directory?).with("/tmp/extdata").returns(true)
 
-                Puppet::Util::Extlookup.datadir(config, :csv, :datadir).should == "/tmp/extdata"
+                Puppet::Util::Extlookup.datadir(config, :csv, :datadir,store).should == "/tmp/extdata"
             end
 
             it "should fail if the datadir does not exist" do
                 config = mock
                 config.expects("[]").with(:csv).returns({:datadir => "/nonexisting"})
+                store = mock
+                store.expects(:lookupvar).never
 
                 expect {
-                    Puppet::Util::Extlookup.datadir(config, :csv, :datadir)
+                    Puppet::Util::Extlookup.datadir(config, :csv, :datadir, store)
                 }.to raise_error("Extlookup datadir (/nonexisting) not found")
             end
+
+            it "should substitute variables in datadir" do
+                config = mock
+                config.expects("[]").with(:csv).returns({:datadir => "/some/path/with/variables/%{test}/in/path"})
+                store = mock
+                store.expects(:lookupvar).with("test").returns("data")
+                store.expects(:respond_to?).with(:lookupvar).returns(true)
+                File.expects(:directory?).with("/some/path/with/variables/data/in/path").returns(true)
+
+                Puppet::Util::Extlookup.datadir(config, :csv, :datadir,store).should == "/some/path/with/variables/data/in/path"
+            end
         end
 
         describe "#parse_data_contents" do