Merge pull request #234 from lsst/tickets/DM-23711

DM-23711: Support shell variables in includeConfigs directive

doc/lsst.daf.butler/configuring.rst

@@ -19,6 +19,7 @@ To construct a Butler configuration object (`~lsst.daf.butler.ButlerConfig`) fro
 * The supplied config is read in.
 * If any leaf nodes in the configuration end in ``includeConfigs`` the values (either a scalar or list) will be treated as the names of other config files.
   These files will be located either as an absolute path or relative to the current working directory, or the directory in which the original configuration file was found.
+  Shell variables will be expanded.
   The contents of these files will then be inserted into the configuration at the same hierarchy as the ``includeConfigs`` directive, with priority given to the values defined explicitly in the parent configuration (for lists of include files later files overwrite content from earlier ones).
 * Each sub configuration class is constructed by supplying the relevant subset of the global config to the component Config constructor.
 * A search path is constructed by concatenating the supplied search path, the environment variable path (``$DAF_BUTLER_CONFIG_PATH``), and the daf_butler config directory (``$DAF_BUTLER_DIR/config``).

python/lsst/daf/butler/core/config.py

@@ -343,6 +343,8 @@ def _processExplicitIncludes(self):
                 # assume resolve explicitly
                 subConfigs = []
                 for fileName in includes:
+                    # Expand any shell variables
+                    fileName = os.path.expandvars(fileName)
                     found = None
                     if os.path.isabs(fileName):
                         found = fileName

tests/config/testConfigs/configIncludesEnv.yaml

@@ -0,0 +1,11 @@
+includeConfigs:
+  - viacls.yaml
+  - abspath.yaml
+comp:
+  item2: "hello"
+  item50: 5000
+unrelated: 1
+addon:
+  includeConfigs: ${SPECIAL_BUTLER_DIR}/viacls2.yaml
+  comp:
+    item11: -1

tests/config/testConfigs/test3/viacls2.yaml

@@ -0,0 +1,5 @@
+# Intended to be loaded via a "cls" entry in another config
+comp:
+  item1: "envvar"
+  item11: "class"
+  item50: 501

tests/test_config.py

@@ -361,6 +361,7 @@ def setUp(self):
         self.testDir = os.path.abspath(os.path.dirname(__file__))
         self.configDir = os.path.join(self.testDir, "config", "testConfigs")
         self.configDir2 = os.path.join(self.testDir, "config", "testConfigs", "test2")
+        self.configDir3 = os.path.join(self.testDir, "config", "testConfigs", "test3")
 
     def testEmpty(self):
         """Ensure that we can read an empty file."""
@@ -523,6 +524,24 @@ def testIncludeConfigs(self):
         self.assertEqual(c["addon", "comp", "item11"], -1)
         self.assertEqual(c["addon", "comp", "item50"], 500)
 
+        # Now test with an environment variable in includeConfigs
+        with modified_environment(SPECIAL_BUTLER_DIR=self.configDir3):
+            c = Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
+            self.assertEqual(c["comp", "item2"], "hello")
+            self.assertEqual(c["comp", "item50"], 5000)
+            self.assertEqual(c["comp", "item1"], "first")
+            self.assertEqual(c["comp", "item10"], "tenth")
+            self.assertEqual(c["comp", "item11"], "eleventh")
+            self.assertEqual(c["unrelated"], 1)
+            self.assertEqual(c["addon", "comp", "item1"], "envvar")
+            self.assertEqual(c["addon", "comp", "item11"], -1)
+            self.assertEqual(c["addon", "comp", "item50"], 501)
+
+        # This will fail
+        with modified_environment(SPECIAL_BUTLER_DIR=self.configDir2):
+            with self.assertRaises(FileNotFoundError):
+                Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
+
 
 if __name__ == "__main__":
     unittest.main()