DM-27256: Add JSON config support

python/lsst/daf/butler/core/config.py

@@ -27,6 +27,7 @@
 
 import collections
 import copy
+import json
 import logging
 import pprint
 import os
@@ -168,7 +169,7 @@ class Config(collections.abc.MutableMapping):
     Storage formats supported:
 
     - yaml: read and write is supported.
-
+    - json: read and write is supported but no ``!include`` directive.
 
     Parameters
     ----------
@@ -244,6 +245,31 @@ def __iter__(self):
     def copy(self):
         return type(self)(self)
 
+    @classmethod
+    def fromString(cls, string: str, format: str = "yaml") -> Config:
+        """Create a new Config instance from a serialized string.
+
+        Parameters
+        ----------
+        string : `str`
+            String containing content in specified format
+        format : `str`, optional
+            Format of the supplied string. Can be ``json`` or ``yaml``.
+
+        Returns
+        -------
+        c : `Config`
+            Newly-constructed Config.
+        """
+        if format == "yaml":
+            new_config = cls().__initFromYaml(string)
+        elif format == "json":
+            new_config = cls().__initFromJson(string)
+        else:
+            raise ValueError(f"Unexpected format of string: {format}")
+        new_config._processExplicitIncludes()
+        return new_config
+
     @classmethod
     def fromYaml(cls, string: str) -> Config:
         """Create a new Config instance from a YAML string.
@@ -258,9 +284,7 @@ def fromYaml(cls, string: str) -> Config:
         c : `Config`
             Newly-constructed Config.
         """
-        new_config = cls().__initFromYaml(string)
-        new_config._processExplicitIncludes()
-        return new_config
+        return cls.fromString(string, format="yaml")
 
     def __initFromUri(self, path: str) -> None:
         """Load a file from a path or an URI.
@@ -271,13 +295,18 @@ def __initFromUri(self, path: str) -> None:
             Path or a URI to a persisted config file.
         """
         uri = ButlerURI(path)
-        if uri.getExtension() == ".yaml":
+        ext = uri.getExtension()
+        if ext == ".yaml":
             log.debug("Opening YAML config file: %s", uri.geturl())
             content = uri.read()
             # Use a stream so we can name it
             stream = io.BytesIO(content)
             stream.name = uri.geturl()
             self.__initFromYaml(stream)
+        elif ext == ".json":
+            log.debug("Opening JSON config file: %s", uri.geturl())
+            content = uri.read()
+            self.__initFromJson(content)
         else:
             raise RuntimeError(f"Unhandled config file type: {uri}")
         self.configFile = uri
@@ -303,6 +332,29 @@ def __initFromYaml(self, stream):
         self._data = content
         return self
 
+    def __initFromJson(self, stream):
+        """Loads a JSON config from any readable stream that contains one.
+
+        Parameters
+        ----------
+        stream: `IO` or `str`
+            Stream to pass to the JSON loader. This can include a string as
+            well as an IO stream.
+
+        Raises
+        ------
+        TypeError:
+            Raised if there is an error loading the content.
+        """
+        if isinstance(stream, (bytes, str)):
+            content = json.loads(stream)
+        else:
+            content = json.load(stream)
+        if content is None:
+            content = {}
+        self._data = content
+        return self
+
     def _processExplicitIncludes(self):
         """Scan through the configuration searching for the special
         includeConfigs directive and process the includes."""
@@ -754,23 +806,33 @@ def __ne__(self, other):
     #######
     # i/o #
 
-    def dump(self, output: Optional[IO] = None) -> Optional[str]:
-        """Writes the config to a yaml stream.
+    def dump(self, output: Optional[IO] = None, format: str = "yaml") -> Optional[str]:
+        """Writes the config to an output stream.
 
         Parameters
         ----------
         output : `IO`, optional
-            The YAML stream to use for output. If `None` the YAML content
+            The stream to use for output. If `None` the serialized content
             will be returned.
+        format : `str`, optional
+            The format to use for the output. Can be "yaml" or "json".
 
         Returns
         -------
-        yaml : `str` or `None`
+        serialized : `str` or `None`
             If a stream was given the stream will be used and the return
-            value will be `None`. If the stream was `None` the YAML
+            value will be `None`. If the stream was `None` the
             serialization will be returned as a string.
         """
-        return yaml.safe_dump(self._data, output, default_flow_style=False)
+        if format == "yaml":
+            return yaml.safe_dump(self._data, output, default_flow_style=False)
+        elif format == "json":
+            if output is not None:
+                json.dump(self._data, output, ensure_ascii=False)
+                return
+            else:
+                return json.dumps(self._data, ensure_ascii=False)
+        raise ValueError(f"Unsupported format for Config serialization: {format}")
 
     def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True,
                   defaultFileName: str = "butler.yaml",
@@ -799,7 +861,11 @@ def dumpToUri(self, uri: Union[ButlerURI, str], updateFile: bool = True,
         if updateFile and not uri.getExtension():
             uri.updateFile(defaultFileName)
 
-        uri.write(self.dump().encode(), overwrite=overwrite)
+        # Try to work out the format from the extension
+        ext = uri.getExtension()
+        format = ext[1:].lower()
+
+        uri.write(self.dump(format=format).encode(), overwrite=overwrite)
         self.configFile = uri
 
     @staticmethod

tests/config/testConfigs/configIncludes.json

@@ -0,0 +1 @@
+{"includeConfigs": ["viacls.yaml", "abspath.yaml"], "comp": {"item2": "hello", "item50": 5000}, "unrelated": 1, "addon": {"includeConfigs": "viacls2.json", "comp": {"item11": -1}}}

tests/config/testConfigs/viacls2.json

@@ -0,0 +1 @@
+{"comp": {"item1": "posix", "item11": "class", "item50": 500}}

tests/test_config.py

@@ -366,15 +366,29 @@ def testHierarchy(self):
         self.assertEqual(c._D, c2._D)  # Check that the child inherits
         self.assertNotEqual(c2._D, Config._D)
 
-    def testStringYaml(self):
+    def testSerializedString(self):
         """Test that we can create configs from strings"""
 
-        c = Config.fromYaml("""
+        serialized = {
+            "yaml": """
 testing: hello
 formatters:
-  calexp: 3""")
-        self.assertEqual(c["formatters", "calexp"], 3)
-        self.assertEqual(c["testing"], "hello")
+  calexp: 3""",
+            "json": '{"testing": "hello", "formatters": {"calexp": 3}}'
+        }
+
+        for format, string in serialized.items():
+            c = Config.fromString(string, format=format)
+            self.assertEqual(c["formatters", "calexp"], 3)
+            self.assertEqual(c["testing"], "hello")
+
+        # Round trip JSON -> Config -> YAML -> Config -> JSON -> Config
+        c1 = Config.fromString(serialized["json"], format="json")
+        yaml = c1.dump(format="yaml")
+        c2 = Config.fromString(yaml, format="yaml")
+        json = c2.dump(format="json")
+        c3 = Config.fromString(json, format="json")
+        self.assertEqual(c3.toDict(), c1.toDict())
 
 
 class ConfigSubsetTestCase(unittest.TestCase):
@@ -546,6 +560,17 @@ def testIncludeConfigs(self):
         self.assertEqual(c["addon", "comp", "item11"], -1)
         self.assertEqual(c["addon", "comp", "item50"], 500)
 
+        c = Config(os.path.join(self.configDir, "configIncludes.json"))
+        self.assertEqual(c["comp", "item2"], "hello")
+        self.assertEqual(c["comp", "item50"], 5000)
+        self.assertEqual(c["comp", "item1"], "first")
+        self.assertEqual(c["comp", "item10"], "tenth")
+        self.assertEqual(c["comp", "item11"], "eleventh")
+        self.assertEqual(c["unrelated"], 1)
+        self.assertEqual(c["addon", "comp", "item1"], "posix")
+        self.assertEqual(c["addon", "comp", "item11"], -1)
+        self.assertEqual(c["addon", "comp", "item50"], 500)
+
         # Now test with an environment variable in includeConfigs
         with modified_environment(SPECIAL_BUTLER_DIR=self.configDir3):
             c = Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
@@ -600,15 +625,16 @@ def testDump(self):
 
         c = Config({"1": 2, "3": 4, "key3": 6, "dict": {"a": 1, "b": 2}})
 
-        outpath = os.path.join(self.tmpdir, "test.yaml")
-        c.dumpToUri(outpath)
+        for format in ("yaml", "json"):
+            outpath = os.path.join(self.tmpdir, f"test.{format}")
+            c.dumpToUri(outpath)
 
-        c2 = Config(outpath)
-        self.assertEqual(c2, c)
+            c2 = Config(outpath)
+            self.assertEqual(c2, c)
 
-        c.dumpToUri(outpath, overwrite=True)
-        with self.assertRaises(FileExistsError):
-            c.dumpToUri(outpath, overwrite=False)
+            c.dumpToUri(outpath, overwrite=True)
+            with self.assertRaises(FileExistsError):
+                c.dumpToUri(outpath, overwrite=False)
 
 
 if __name__ == "__main__":

tests/test_testRepo.py

@@ -71,7 +71,7 @@ def testButlerKwargs(self):
         # outfile has the most obvious effects of any Butler.makeRepo keyword
         temp = tempfile.mkdtemp(dir=TESTDIR)
         try:
-            path = os.path.join(temp, 'oddConfig.py')
+            path = os.path.join(temp, 'oddConfig.json')
             makeTestRepo(self.root, {}, outfile=path, createRegistry=False)
             self.assertTrue(os.path.isfile(path))
         finally: