Merge pull request #398 from lsst/tickets/DM-27035

DM-27035: miscellaneous fixes and integration after DM-27034

python/lsst/daf/butler/core/__init__.py

@@ -27,4 +27,5 @@
 from .dimensions import *
 from .fileDataset import *
 from . import time_utils
+from ._topology import *
 from .timespan import *

python/lsst/daf/butler/core/dimensions/_topology.py → python/lsst/daf/butler/core/_topology.py

@@ -34,13 +34,14 @@
     Any,
     Mapping,
     Optional,
+    Type,
     TypeVar,
 )
 
 import sqlalchemy
 
-from ..named import NamedValueAbstractSet
-from ..utils import immutable
+from .named import NamedValueAbstractSet
+from .utils import immutable
 
 
 @enum.unique
@@ -182,8 +183,7 @@ class TopologicalExtentDatabaseRepresentation(ABC):
 
     @classmethod
     @abstractmethod
-    def fromSelectable(cls, selectable: sqlalchemy.sql.FromClause
-                       ) -> TopologicalExtentDatabaseRepresentation:
+    def fromSelectable(cls: Type[_S], selectable: sqlalchemy.sql.FromClause) -> _S:
         """Construct an instance that represents a logical column (which may
         actually be backed by multiple columns) in the given table or subquery.
 

python/lsst/daf/butler/core/ddl.py

@@ -51,7 +51,7 @@
 from .named import NamedValueSet
 
 if TYPE_CHECKING:
-    from .timespan import DatabaseTimespanRepresentation
+    from .timespan import TimespanDatabaseRepresentation
 
 
 _LOG = logging.getLogger(__name__)
@@ -306,6 +306,16 @@ def getSizedColumnType(self) -> sqlalchemy.types.TypeEngine:
             return self.dtype(nbytes=self.nbytes)
         return self.dtype
 
+    def getPythonType(self) -> type:
+        """Return the Python type associated with this field's (SQL) dtype.
+
+        Returns
+        -------
+        type : `type`
+            Python type associated with this field's (SQL) `dtype`.
+        """
+        return self.dtype().python_type
+
 
 @dataclass
 class ForeignKeySpec:
@@ -380,7 +390,7 @@ class TableSpec:
         Special constraints that prohibit overlaps between timespans over rows
         where other columns are equal.  These take the same form as unique
         constraints, but each tuple may contain a single
-        `DatabaseTimespanRepresentation` subclass representing a timespan
+        `TimespanDatabaseRepresentation` subclass representing a timespan
         column.
     recycleIds : bool, optional
         If `True`, allow databases that might normally recycle autoincrement
@@ -394,7 +404,7 @@ def __init__(
         unique: Iterable[Tuple[str, ...]] = (),
         indexes: Iterable[Tuple[str, ...]] = (),
         foreignKeys: Iterable[ForeignKeySpec] = (),
-        exclusion: Iterable[Tuple[Union[str, Type[DatabaseTimespanRepresentation]], ...]] = (),
+        exclusion: Iterable[Tuple[Union[str, Type[TimespanDatabaseRepresentation]], ...]] = (),
         recycleIds: bool = True,
         doc: Optional[str] = None,
     ):
@@ -418,7 +428,7 @@ def __init__(
     foreignKeys: List[ForeignKeySpec]
     """Foreign key constraints for the table."""
 
-    exclusion: Set[Tuple[Union[str, Type[DatabaseTimespanRepresentation]], ...]]
+    exclusion: Set[Tuple[Union[str, Type[TimespanDatabaseRepresentation]], ...]]
     """Exclusion constraints for the table.
 
     Exclusion constraints behave mostly like unique constraints, but may

python/lsst/daf/butler/core/dimensions/__init__.py

@@ -26,7 +26,6 @@
 # function-scope imports.  The order below is one that is consistent with the
 # unguarded dependencies.
 
-from ._topology import *
 from ._elements import *
 from ._schema import *
 from ._graph import *

python/lsst/daf/butler/core/dimensions/_config.py

@@ -28,10 +28,10 @@
 from ..config import Config, ConfigSubset
 from .. import ddl
 from .._butlerUri import ButlerURI
+from .._topology import TopologicalSpace
 from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
 from ._packer import DimensionPackerConstructionVisitor
 from ._skypix import SkyPixConstructionVisitor
-from ._topology import TopologicalSpace
 from .standard import (
     StandardDimensionElementConstructionVisitor,
     StandardTopologicalFamilyConstructionVisitor,

python/lsst/daf/butler/core/dimensions/_coordinate.py

@@ -529,10 +529,10 @@ def timespan(self) -> Optional[Timespan]:
             record = self._record(element.name)
             # DimensionRecord subclasses for temporal elements always have
             # .timespan, but they're dynamic so this can't be type-checked.
-            if record is None or record.timespan is None:
+            if record is None or record.timespan is None:  # type: ignore
                 return None
             else:
-                timespans.append(record.timespan)
+                timespans.append(record.timespan)  # type: ignore
         return Timespan.intersection(*timespans)
 
     def pack(self, name: str, *, returnMaxBits: bool = False) -> Union[Tuple[int, int], int]:

python/lsst/daf/butler/core/dimensions/_elements.py

@@ -23,6 +23,7 @@
 
 __all__ = (
     "Dimension",
+    "DimensionCombination",
     "DimensionElement",
 )
 
@@ -38,8 +39,8 @@
 from ..named import NamedValueAbstractSet, NamedValueSet
 from ..utils import cached_getter
 from .. import ddl
+from .._topology import TopologicalRelationshipEndpoint
 
-from ._topology import TopologicalRelationshipEndpoint
 
 if TYPE_CHECKING:  # Imports needed only for type annotations; may be circular.
     from ._universe import DimensionUniverse
@@ -286,8 +287,12 @@ def primaryKey(self) -> ddl.FieldSpec:
     @property  # type: ignore
     @cached_getter
     def alternateKeys(self) -> NamedValueAbstractSet[ddl.FieldSpec]:
-        """Additional unique key fields for this dimension that are not the the
+        """Additional unique key fields for this dimension that are not the
         primary key (`NamedValueAbstractSet` of `FieldSpec`).
+
+        If this dimension has required dependencies, the keys of those
+        dimensions are also included in the unique constraints defined for
+        these alternate keys.
         """
         _, *alternateKeys = self.uniqueKeys
         return NamedValueSet(alternateKeys).freeze()

python/lsst/daf/butler/core/dimensions/_graph.py

@@ -41,8 +41,8 @@
 
 from ..named import NamedValueAbstractSet, NamedValueSet
 from ..utils import immutable
+from .._topology import TopologicalSpace, TopologicalFamily
 
-from ._topology import TopologicalSpace, TopologicalFamily
 
 if TYPE_CHECKING:  # Imports needed only for type annotations; may be circular.
     from ._universe import DimensionUniverse

python/lsst/daf/butler/core/dimensions/_records.py

@@ -31,7 +31,8 @@
     Type,
 )
 
-from ..timespan import Timespan, DatabaseTimespanRepresentation
+from ..timespan import Timespan, TimespanDatabaseRepresentation
+from ..utils import immutable
 from ._elements import Dimension, DimensionElement
 
 if TYPE_CHECKING:  # Imports needed only for type annotations; may be circular.
@@ -59,7 +60,7 @@ def _subclassDimensionRecord(definition: DimensionElement) -> Type[DimensionReco
     if definition.spatial:
         slots.append(REGION_FIELD_SPEC.name)
     if definition.temporal:
-        slots.append(DatabaseTimespanRepresentation.NAME)
+        slots.append(TimespanDatabaseRepresentation.NAME)
     d = {
         "definition": definition,
         "__slots__": tuple(slots),
@@ -68,6 +69,7 @@ def _subclassDimensionRecord(definition: DimensionElement) -> Type[DimensionReco
     return type(definition.name + ".RecordClass", (DimensionRecord,), d)
 
 
+@immutable
 class DimensionRecord:
     """Base class for the Python representation of database records for
     a `DimensionElement`.
@@ -130,9 +132,13 @@ def __init__(self, **kwargs: Any):
             object.__setattr__(self, name, kwargs.get(name))
         if self.definition.temporal is not None:
             if self.timespan is None:  # type: ignore
-                self.timespan = Timespan(
-                    kwargs.get("datetime_begin"),
-                    kwargs.get("datetime_end"),
+                object.__setattr__(
+                    self,
+                    "timespan",
+                    Timespan(
+                        kwargs.get("datetime_begin"),
+                        kwargs.get("datetime_end"),
+                    )
                 )
 
         from ._coordinate import DataCoordinate

python/lsst/daf/butler/core/dimensions/_schema.py

@@ -31,7 +31,7 @@
 
 from .. import ddl
 from ..named import NamedValueSet
-from ..timespan import DatabaseTimespanRepresentation
+from ..timespan import TimespanDatabaseRepresentation
 
 if TYPE_CHECKING:  # Imports needed only for type annotations; may be circular.
     from ._elements import DimensionElement, Dimension
@@ -186,16 +186,16 @@ def __init__(self, element: DimensionElement):
             self._tableSpec.fields.add(REGION_FIELD_SPEC)
             names.append(REGION_FIELD_SPEC.name)
         if element.temporal is not None:
-            names.append(DatabaseTimespanRepresentation.NAME)
+            names.append(TimespanDatabaseRepresentation.NAME)
         self.names = tuple(names)
 
-    def makeTableSpec(self, tsRepr: Type[DatabaseTimespanRepresentation]) -> ddl.TableSpec:
+    def makeTableSpec(self, tsRepr: Type[TimespanDatabaseRepresentation]) -> ddl.TableSpec:
         """Construct a complete specification for a table that could hold the
         records of this element.
 
         Parameters
         ----------
-        tsRepr : `type` (`DatabaseTimespanRepresentation` subclass)
+        tsRepr : `type` (`TimespanDatabaseRepresentation` subclass)
             Class object that specifies how timespans are represented in the
             database.
 
@@ -217,6 +217,15 @@ def makeTableSpec(self, tsRepr: Type[DatabaseTimespanRepresentation]) -> ddl.Tab
             spec = self._tableSpec
         return spec
 
+    def __str__(self) -> str:
+        lines = [f"{self.element.name}: "]
+        lines.extend(f"  {field.name}: {field.getPythonType().__name__}" for field in self.standard)
+        if self.element.spatial is not None:
+            lines.append("  region: lsst.sphgeom.ConvexPolygon")
+        if self.element.temporal is not None:
+            lines.append("  timespan: lsst.daf.butler.Timespan")
+        return "\n".join(lines)
+
     element: DimensionElement
     """The dimension element these fields correspond to (`DimensionElement`).
     """

python/lsst/daf/butler/core/dimensions/_skypix.py

@@ -39,9 +39,9 @@
 from lsst.sphgeom import Pixelization
 from lsst.utils import doImport
 from .. import ddl
+from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace
 
 from ..named import NamedValueAbstractSet, NamedValueSet
-from ._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace
 from ._elements import Dimension
 from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor
 

python/lsst/daf/butler/core/dimensions/_universe.py

@@ -41,10 +41,10 @@
 
 from ..config import Config
 from ..named import NamedValueAbstractSet
+from .._topology import TopologicalSpace, TopologicalFamily
 from ..utils import immutable
 from ._config import DimensionConfig
 from ._elements import Dimension, DimensionElement
-from ._topology import TopologicalSpace, TopologicalFamily
 from ._graph import DimensionGraph
 
 if TYPE_CHECKING:  # Imports needed only for type annotations; may be circular.

python/lsst/daf/butler/core/dimensions/construction.py

@@ -30,7 +30,7 @@
 )
 
 from ..named import NamedValueSet
-from ._topology import TopologicalFamily, TopologicalSpace
+from .._topology import TopologicalFamily, TopologicalSpace
 
 if TYPE_CHECKING:
     from ._elements import Dimension, DimensionElement

python/lsst/daf/butler/core/dimensions/standard.py

@@ -36,9 +36,9 @@
 
 from .. import ddl
 from ..named import NamedValueAbstractSet, NamedValueSet
+from .._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace
 
 from ._elements import Dimension, DimensionCombination, DimensionElement
-from ._topology import TopologicalFamily, TopologicalRelationshipEndpoint, TopologicalSpace
 from .construction import DimensionConstructionBuilder, DimensionConstructionVisitor