DM-14771: Add support for views to Schema

config/schema.yaml

@@ -694,7 +694,7 @@ schema:
         type: string
         doc: A string value composed only of letters, numbers, and underscores.
 
-  dataunit_joins:
+  dataUnitJoins:
     ExposureRangeJoin:
       doc: >
         A calculated join between Datasets identified with an Exposure
@@ -844,10 +844,9 @@ schema:
             tgt:
               - Visit.camera
               - Visit.visit
-      sql: >
-        CREATE VIEW VisitSkyPixJoin
-          SELECT DISTINCT camera, visit, skypix
-          FROM VisitSensorSkyPixJoin;
+          sql: >
+            SELECT DISTINCT camera, visit, skypix
+            FROM VisitSensorSkyPixJoin;
 
     PatchSkyPixJoin:
       doc: >
@@ -931,10 +930,9 @@ schema:
             tgt:
               - Tract.skymap
               - Tract.tract
-      sql: >
-        CREATE VIEW TractPixJoin
-          SELECT DISTINCT skymap, tract, skypix
-          FROM PatchSkyPixJoin;
+          sql: >
+            SELECT DISTINCT skymap, tract, skypix
+            FROM PatchSkyPixJoin;
 
     VisitSensorPatchJoin:
       doc: >
@@ -1004,19 +1002,18 @@ schema:
               - Patch.skymap
               - Patch.tract
               - Patch.patch
-      sql: >
-        CREATE VIEW VisitSensorPatchJoin AS
-          SELECT DISTINCT
-            VisitSensorSkyPixJoin.camera,
-            VisitSensorSkyPixJoin.visit,
-            VisitSensorSkyPixJoin.sensor,
-            PatchSkyPixJoin.skymap,
-            PatchSkyPixJoin.tract,
-            PatchSkyPixJoin.patch
-          FROM
-            VisitSensorSkyPixJoin INNER JOIN PatchSkyPixJoin ON (
-              VisitSensorSkyPixJoin.skypix = PatchSkyPixJoin.skypix
-            );
+          sql: >
+            SELECT DISTINCT
+              VisitSensorSkyPixJoin.camera,
+              VisitSensorSkyPixJoin.visit,
+              VisitSensorSkyPixJoin.sensor,
+              PatchSkyPixJoin.skymap,
+              PatchSkyPixJoin.tract,
+              PatchSkyPixJoin.patch
+            FROM
+              VisitSensorSkyPixJoin INNER JOIN PatchSkyPixJoin ON (
+                VisitSensorSkyPixJoin.skypix = PatchSkyPixJoin.skypix
+              );
 
     VisitPatchJoin:
       doc: >
@@ -1080,18 +1077,17 @@ schema:
               - Patch.skymap
               - Patch.tract
               - Patch.patch
-      sql: >
-        CREATE VIEW VisitPatchJoin AS
-          SELECT DISTINCT
-            VisitSkyPixJoin.camera,
-            VisitSkyPixJoin.visit,
-            PatchSkyPixJoin.skymap,
-            PatchSkyPixJoin.tract,
-            PatchSkyPixJoin.patch
-          FROM
-            VisitSkyPixJoin INNER JOIN PatchSkyPixJoin ON (
-              VisitSkyPixJoin.skypix = PatchSkyPixJoin.skypix
-            );
+          sql: >
+            SELECT DISTINCT
+              VisitSkyPixJoin.camera,
+              VisitSkyPixJoin.visit,
+              PatchSkyPixJoin.skymap,
+              PatchSkyPixJoin.tract,
+              PatchSkyPixJoin.patch
+            FROM
+              VisitSkyPixJoin INNER JOIN PatchSkyPixJoin ON (
+                VisitSkyPixJoin.skypix = PatchSkyPixJoin.skypix
+              );
 
     VisitSensorTractJoin:
       doc: >
@@ -1156,18 +1152,17 @@ schema:
             tgt:
               - Tract.skymap
               - Tract.tract
-      sql: >
-        CREATE VIEW VisitSensorTractJoin AS
-          SELECT DISTINCT
-            VisitSensorSkyPixJoin.camera,
-            VisitSensorSkyPixJoin.visit,
-            VisitSensorSkyPixJoin.sensor,
-            TractSkyPixJoin.skymap,
-            TractSkyPixJoin.tract
-          FROM
-            VisitSensorSkyPixJoin INNER JOIN TractSkyPixJoin ON (
-              VisitSensorSkyPixJoin.skypix = TractSkyPixJoin.skypix
-            );
+          sql: >
+            SELECT DISTINCT
+              VisitSensorSkyPixJoin.camera,
+              VisitSensorSkyPixJoin.visit,
+              VisitSensorSkyPixJoin.sensor,
+              TractSkyPixJoin.skymap,
+              TractSkyPixJoin.tract
+            FROM
+              VisitSensorSkyPixJoin INNER JOIN TractSkyPixJoin ON (
+                VisitSensorSkyPixJoin.skypix = TractSkyPixJoin.skypix
+              );
 
     VisitTractJoin:
       doc: >
@@ -1226,17 +1221,16 @@ schema:
             tgt:
               - Tract.skymap
               - Tract.tract
-      sql: >
-        CREATE VIEW VisitTractJoin AS
-          SELECT DISTINCT
-            VisitSkyPixJoin.camera,
-            VisitSkyPixJoin.visit,
-            TractSkyPixJoin.skymap,
-            TractSkyPixJoin.tract
-          FROM
-            VisitSkyPixJoin INNER JOIN TractSkyPixJoin ON (
-              VisitSkyPixJoin.skypix = TractSkyPixJoin.skypix
-            );
+          sql: >
+            SELECT DISTINCT
+              VisitSkyPixJoin.camera,
+              VisitSkyPixJoin.visit,
+              TractSkyPixJoin.skymap,
+              TractSkyPixJoin.tract
+            FROM
+              VisitSkyPixJoin INNER JOIN TractSkyPixJoin ON (
+                VisitSkyPixJoin.skypix = TractSkyPixJoin.skypix
+              );
 
   tables:
     Dataset:

python/lsst/daf/butler/butler.py

@@ -74,7 +74,7 @@ class Butler:
     """
 
     @staticmethod
-    def makeRepo(root, config=None, standalone=False):
+    def makeRepo(root, config=None, standalone=False, createRegistry=True):
         """Create an empty data repository by adding a butler.yaml config
         to a repository root directory.
 
@@ -95,6 +95,8 @@ def makeRepo(root, config=None, standalone=False):
             may be good or bad, depending on the nature of the changes).
             Future *additions* to the defaults will still be picked up when
             initializing `Butlers` to repos created with ``standalone=True``.
+        createRegistry : `bool`
+            If `True` create a new Registry.
 
         Note that when ``standalone=False`` (the default), the configuration
         search path (see `ConfigSubset.defaultSearchPaths`) that was used to
@@ -130,6 +132,8 @@ def makeRepo(root, config=None, standalone=False):
         if standalone:
             config.merge(full)
         config.dumpToFile(os.path.join(root, "butler.yaml"))
+        # Create Registry and populate tables
+        registryClass.fromConfig(config, create=createRegistry)
         return config
 
     def __init__(self, config=None, collection=None, run=None):

python/lsst/daf/butler/core/dataUnit.py

@@ -150,6 +150,60 @@ def validateId(self, dataId):
                 missing, dataId, self.name))
 
 
+class DataUnitJoin:
+    """Represents a join between one or more `DataUnit`s.
+
+    Parameters
+    ----------
+    name : `str`
+        Name of this `DataUnit` (`str`, read-only).
+        Also assumed to be the name of the primary table (if present).
+    lhs : `tuple`
+        Left-hand-side of the join.
+    rhs : `tuple`
+        Right-hand-side of the join.
+    summarizes : `DataUnitJoin`
+        Summarizes this other `DataUnitJoin`.
+    table : `sqlalchemy.TableClause` or `sqlalchemy.Table`
+        The table to be used for queries.  Note that this is not
+        an actual `Table` in many cases because joins are often
+        materialized as views (and thus are also not present
+        in `Registry._schema._metadata`).
+    """
+    def __init__(self, name, lhs=None, rhs=None, summarizes=None, table=None):
+        self._name = name
+        self._lhs = lhs
+        self._rhs = rhs
+        self._summarizes = summarizes
+        self._table = table
+
+    @property
+    def name(self):
+        """Name of this `DataUnit` (`str`, read-only).
+
+        Also assumed to be the name of the primary table (if present)."""
+        return self._name
+
+    @property
+    def lhs(self):
+        return self._lhs
+
+    @property
+    def rhs(self):
+        return self._rhs
+
+    @property
+    def summarizes(self):
+        return self._summarizes
+
+    @property
+    def table(self):
+        """When not ``None`` the primary table entry corresponding to this
+        `DataUnitJoin` (`sqlalchemy.core.TableClause`, optional).
+        """
+        return getattr(self, '_table', None)
+
+
 class DataUnitRegistry:
     """Instances of this class keep track of `DataUnit` relations.
 
@@ -161,6 +215,7 @@ def __init__(self):
         self._dataUnits = {}
         self.links = {}
         self.constraints = []
+        self.joins = {}
 
     @classmethod
     def fromConfig(cls, config, builder=None):
@@ -177,8 +232,9 @@ def fromConfig(cls, config, builder=None):
             When given, create `sqlalchemy.core.Table` entries for every `DataUnit` table.
         """
         dataUnitRegistry = cls()
-        dataUnitRegistry._initDataUnitNames(config)
-        dataUnitRegistry._initDataUnits(config, builder)
+        dataUnitRegistry._initDataUnitNames(config['dataUnits'])
+        dataUnitRegistry._initDataUnits(config['dataUnits'], builder)
+        dataUnitRegistry._initDataUnitJoins(config['dataUnitJoins'], builder)
         return dataUnitRegistry
 
     def __len__(self):
@@ -275,6 +331,31 @@ def _initDataUnits(self, config, builder):
                                 link=link)
             self[dataUnitName] = dataUnit
 
+    def _initDataUnitJoins(self, config, builder):
+        """Initialize `DataUnit` join entries.
+
+        Parameters
+        ----------
+        config : `SchemaConfig`
+            Schema configuration describing `DataUnit` join relations.
+        builder : `SchemaBuilder`, optional
+            When given, create `sqlalchemy.core.Table` entries for every `DataUnit` table.
+        """
+        for dataUnitJoinName, dataUnitJoinDescription in config.items():
+            table = None
+            if 'tables' in dataUnitJoinDescription and builder is not None:
+                for tableName, tableDescription in dataUnitJoinDescription['tables'].items():
+                    table = builder.addTable(tableName, tableDescription)
+            lhs = frozenset((dataUnitJoinDescription.get('lhs', None)))
+            rhs = frozenset((dataUnitJoinDescription.get('rhs', None)))
+            summarizes = dataUnitJoinDescription.get('summarizes', None)
+            dataUnitJoin = DataUnitJoin(name=dataUnitJoinName,
+                                        lhs=lhs,
+                                        rhs=rhs,
+                                        summarizes=summarizes,
+                                        table=table)
+            self.joins[(lhs, rhs)] = dataUnitJoin
+
     def getPrimaryKeyNames(self, dataUnitNames):
         """Get all primary-key column names for the given ``dataUnitNames``.
 

python/lsst/daf/butler/core/registry.py

@@ -72,7 +72,7 @@ def setConfigRoot(cls, root, config, full):
         raise NotImplementedError()
 
     @staticmethod
-    def fromConfig(registryConfig, schemaConfig=None):
+    def fromConfig(registryConfig, schemaConfig=None, create=False):
         """Create `Registry` subclass instance from `config`.
 
         Uses ``registry.cls`` from `config` to determine which subclass to instantiate.
@@ -85,6 +85,8 @@ def fromConfig(registryConfig, schemaConfig=None):
             Schema configuration. Can be read from supplied registryConfig
             if the relevant component is defined and ``schemaConfig`` is
             `None`.
+        create : `bool`
+            Assume empty Registry and create a new one.
 
         Returns
         -------
@@ -108,9 +110,9 @@ def fromConfig(registryConfig, schemaConfig=None):
                 raise ValueError("Incompatible Registry configuration: {}".format(registryConfig))
 
         cls = doImport(registryConfig['cls'])
-        return cls(registryConfig, schemaConfig)
+        return cls(registryConfig, schemaConfig, create=create)
 
-    def __init__(self, registryConfig, schemaConfig=None):
+    def __init__(self, registryConfig, schemaConfig=None, create=False):
         assert isinstance(registryConfig, RegistryConfig)
         self.config = registryConfig