Skip to content

Commit

Permalink
Replace "tsRepr" with "TimespanReprClass"
Browse files Browse the repository at this point in the history
...and similarly for regRepr->RegionReprClass.

These symbols invariably refer to a type object that is a subclass of
TimespanDatabaseRepresentation (or
SpatialRegionDatabaseRepresentation).
  • Loading branch information
TallJimbo committed Dec 16, 2020
1 parent a26ee5c commit 15afa4a
Show file tree
Hide file tree
Showing 16 changed files with 149 additions and 122 deletions.
13 changes: 6 additions & 7 deletions python/lsst/daf/butler/core/dimensions/_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,19 +188,18 @@ def __init__(self, element: DimensionElement):

def makeTableSpec(
self,
regRepr: Type[SpatialRegionDatabaseRepresentation],
tsRepr: Type[TimespanDatabaseRepresentation],
RegionReprClass: Type[SpatialRegionDatabaseRepresentation],
TimespanReprClass: Type[TimespanDatabaseRepresentation],
) -> ddl.TableSpec:
"""Construct a complete specification for a table that could hold the
records of this element.
Parameters
----------
regRepr : `type` (`SpatialRegionDatabaseRepresentation` subclass)
RegionReprClass : `type` [ `SpatialRegionDatabaseRepresentation` ]
Class object that specifies how spatial regions are represented in
the database.
tsRepr : `type` (`TimespanDatabaseRepresentation` subclass)
TimespanReprClass : `type` [ `TimespanDatabaseRepresentation` ]
Class object that specifies how timespans are represented in the
database.
Expand All @@ -217,9 +216,9 @@ def makeTableSpec(
foreignKeys=self._tableSpec.foreignKeys,
)
if self.element.spatial is not None:
spec.fields.update(regRepr.makeFieldSpecs(nullable=True))
spec.fields.update(RegionReprClass.makeFieldSpecs(nullable=True))
if self.element.temporal is not None:
spec.fields.update(tsRepr.makeFieldSpecs(nullable=True))
spec.fields.update(TimespanReprClass.makeFieldSpecs(nullable=True))
else:
spec = self._tableSpec
return spec
Expand Down
6 changes: 3 additions & 3 deletions python/lsst/daf/butler/registry/_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -1330,7 +1330,7 @@ def makeQueryBuilder(self, summary: queries.QuerySummary) -> queries.QueryBuilde
collections=self._collections,
dimensions=self._dimensions,
datasets=self._datasets,
tsRepr=self._db.getTimespanRepresentation(),
TimespanReprClass=self._db.getTimespanRepresentation(),
),
)

Expand Down Expand Up @@ -1724,7 +1724,7 @@ def queryDatasetAssociations(
a single collection.
"""
collections = CollectionQuery.fromExpression(collections)
tsRepr = self._db.getTimespanRepresentation()
TimespanReprClass = self._db.getTimespanRepresentation()
if isinstance(datasetType, str):
storage = self._datasets[datasetType]
else:
Expand All @@ -1744,7 +1744,7 @@ def queryDatasetAssociations(
ref = DatasetRef(storage.datasetType, dataId, id=row["id"], run=runRecord.name,
conform=False)
if collectionRecord.type is CollectionType.CALIBRATION:
timespan = tsRepr.extract(row)
timespan = TimespanReprClass.extract(row)
else:
timespan = None
yield DatasetAssociation(ref=ref, collection=collectionRecord.name, timespan=timespan)
Expand Down
16 changes: 8 additions & 8 deletions python/lsst/daf/butler/registry/collections/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def _makeCollectionForeignKey(sourceColumnName: str, collectionIdName: str,


def makeRunTableSpec(collectionIdName: str, collectionIdType: type,
tsRepr: Type[TimespanDatabaseRepresentation]) -> ddl.TableSpec:
TimespanReprClass: Type[TimespanDatabaseRepresentation]) -> ddl.TableSpec:
"""Define specification for "run" table.
Parameters
Expand All @@ -96,7 +96,7 @@ def makeRunTableSpec(collectionIdName: str, collectionIdType: type,
collectionIdType
Type of the PK column in the collections table, one of the
`sqlalchemy` types.
tsRepr : `type` [ `TimespanDatabaseRepresentation` ]
TimespanReprClass : `type` [ `TimespanDatabaseRepresentation` ]
Subclass of `TimespanDatabaseRepresentation` that encapsulates how
timespans are stored in this database.
Expand All @@ -121,7 +121,7 @@ def makeRunTableSpec(collectionIdName: str, collectionIdType: type,
_makeCollectionForeignKey(collectionIdName, collectionIdName, onDelete="CASCADE"),
],
)
for fieldSpec in tsRepr.makeFieldSpecs(nullable=True):
for fieldSpec in TimespanReprClass.makeFieldSpecs(nullable=True):
result.fields.add(fieldSpec)
return result

Expand Down Expand Up @@ -330,7 +330,7 @@ def refresh(self) -> None:
# in place, for exception safety.
records = []
chains = []
tsRepr = self._db.getTimespanRepresentation()
TimespanReprClass = self._db.getTimespanRepresentation()
for row in self._db.query(sql).fetchall():
collection_id = row[self._tables.collection.columns[self._collectionIdName]]
name = row[self._tables.collection.columns.name]
Expand All @@ -344,7 +344,7 @@ def refresh(self) -> None:
table=self._tables.run,
idColumnName=self._collectionIdName,
host=row[self._tables.run.columns.host],
timespan=tsRepr.extract(row),
timespan=TimespanReprClass.extract(row),
)
elif type is CollectionType.CHAINED:
record = DefaultChainedCollectionRecord(db=self._db,
Expand Down Expand Up @@ -374,11 +374,11 @@ def register(self, name: str, type: CollectionType, doc: Optional[str] = None) -
assert row is not None
collection_id = row[self._collectionIdName]
if type is CollectionType.RUN:
tsRepr = self._db.getTimespanRepresentation()
TimespanReprClass = self._db.getTimespanRepresentation()
row, _ = self._db.sync(
self._tables.run,
keys={self._collectionIdName: collection_id},
returning=("host",) + tsRepr.getFieldNames(),
returning=("host",) + TimespanReprClass.getFieldNames(),
)
assert row is not None
record = DefaultRunRecord(
Expand All @@ -388,7 +388,7 @@ def register(self, name: str, type: CollectionType, doc: Optional[str] = None) -
table=self._tables.run,
idColumnName=self._collectionIdName,
host=row["host"],
timespan=tsRepr.extract(row),
timespan=TimespanReprClass.extract(row),
)
elif type is CollectionType.CHAINED:
record = DefaultChainedCollectionRecord(db=self._db, key=collection_id, name=name,
Expand Down
4 changes: 2 additions & 2 deletions python/lsst/daf/butler/registry/collections/nameKey.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
_VERSION = VersionTuple(2, 0, 0)


def _makeTableSpecs(tsRepr: Type[TimespanDatabaseRepresentation]) -> CollectionTablesTuple:
def _makeTableSpecs(TimespanReprClass: Type[TimespanDatabaseRepresentation]) -> CollectionTablesTuple:
return CollectionTablesTuple(
collection=ddl.TableSpec(
fields=[
Expand All @@ -61,7 +61,7 @@ def _makeTableSpecs(tsRepr: Type[TimespanDatabaseRepresentation]) -> CollectionT
ddl.FieldSpec("doc", dtype=sqlalchemy.Text, nullable=True),
],
),
run=makeRunTableSpec("name", sqlalchemy.String, tsRepr),
run=makeRunTableSpec("name", sqlalchemy.String, TimespanReprClass),
collection_chain=makeCollectionChainTableSpec("name", sqlalchemy.String),
)

Expand Down
4 changes: 2 additions & 2 deletions python/lsst/daf/butler/registry/collections/synthIntKey.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
_VERSION = VersionTuple(2, 0, 0)


def _makeTableSpecs(tsRepr: Type[TimespanDatabaseRepresentation]) -> CollectionTablesTuple:
def _makeTableSpecs(TimespanReprClass: Type[TimespanDatabaseRepresentation]) -> CollectionTablesTuple:
return CollectionTablesTuple(
collection=ddl.TableSpec(
fields=[
Expand All @@ -65,7 +65,7 @@ def _makeTableSpecs(tsRepr: Type[TimespanDatabaseRepresentation]) -> CollectionT
],
unique=[("name",)],
),
run=makeRunTableSpec("collection_id", sqlalchemy.BigInteger, tsRepr),
run=makeRunTableSpec("collection_id", sqlalchemy.BigInteger, TimespanReprClass),
collection_chain=makeCollectionChainTableSpec("collection_id", sqlalchemy.BigInteger),
)

Expand Down
30 changes: 18 additions & 12 deletions python/lsst/daf/butler/registry/datasets/byDimensions/_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,8 +241,10 @@ def _buildCalibOverlapQuery(self, collection: CollectionRecord,
lambda name: self._calibs.columns[name], # type: ignore
)
# Add WHERE clause for timespan overlaps.
tsRepr = self._db.getTimespanRepresentation()
query.where.append(tsRepr.fromSelectable(self._calibs).overlaps(tsRepr.fromLiteral(timespan)))
TimespanReprClass = self._db.getTimespanRepresentation()
query.where.append(
TimespanReprClass.fromSelectable(self._calibs).overlaps(TimespanReprClass.fromLiteral(timespan))
)
return query

def certify(self, collection: CollectionRecord, datasets: Iterable[DatasetRef],
Expand All @@ -254,7 +256,7 @@ def certify(self, collection: CollectionRecord, datasets: Iterable[DatasetRef],
if collection.type is not CollectionType.CALIBRATION:
raise TypeError(f"Cannot certify into collection '{collection.name}' "
f"of type {collection.type.name}; must be CALIBRATION.")
tsRepr = self._db.getTimespanRepresentation()
TimespanReprClass = self._db.getTimespanRepresentation()
protoRow = {
self._collections.getCollectionForeignKeyName(): collection.key,
"dataset_type_id": self._dataset_type_id,
Expand All @@ -263,12 +265,14 @@ def certify(self, collection: CollectionRecord, datasets: Iterable[DatasetRef],
governorValues: Dict[str, Set[str]] = {
name: set() for name in self.datasetType.dimensions.governors.names
}
dataIds: Optional[Set[DataCoordinate]] = set() if not tsRepr.hasExclusionConstraint() else None
dataIds: Optional[Set[DataCoordinate]] = (
set() if not TimespanReprClass.hasExclusionConstraint() else None
)
for dataset in datasets:
row = dict(protoRow, dataset_id=dataset.getCheckedId())
for dimension, value in dataset.dataId.items():
row[dimension.name] = value
tsRepr.update(timespan, result=row)
TimespanReprClass.update(timespan, result=row)
for governorName, values in governorValues.items():
values.add(dataset.dataId[governorName]) # type: ignore
rows.append(row)
Expand All @@ -279,7 +283,7 @@ def certify(self, collection: CollectionRecord, datasets: Iterable[DatasetRef],
# inserted there.
self._ensureSummaries(collection, governorValues)
# Update the association table itself.
if tsRepr.hasExclusionConstraint():
if TimespanReprClass.hasExclusionConstraint():
# Rely on database constraint to enforce invariants; we just
# reraise the exception for consistency across DB engines.
try:
Expand Down Expand Up @@ -325,7 +329,7 @@ def decertify(self, collection: CollectionRecord, timespan: Timespan, *,
if collection.type is not CollectionType.CALIBRATION:
raise TypeError(f"Cannot decertify from collection '{collection.name}' "
f"of type {collection.type.name}; must be CALIBRATION.")
tsRepr = self._db.getTimespanRepresentation()
TimespanReprClass = self._db.getTimespanRepresentation()
# Construct a SELECT query to find all rows that overlap our inputs.
dataIdSet: Optional[DataCoordinateSet]
if dataIds is not None:
Expand Down Expand Up @@ -357,10 +361,10 @@ def decertify(self, collection: CollectionRecord, timespan: Timespan, *,
newInsertRow["dataset_id"] = row["dataset_id"]
for name in self.datasetType.dimensions.required.names:
newInsertRow[name] = row[name]
rowTimespan = tsRepr.extract(row)
rowTimespan = TimespanReprClass.extract(row)
assert rowTimespan is not None, "Field should have a NOT NULL constraint."
for diffTimespan in rowTimespan.difference(timespan):
rowsToInsert.append(tsRepr.update(diffTimespan, result=newInsertRow.copy()))
rowsToInsert.append(TimespanReprClass.update(diffTimespan, result=newInsertRow.copy()))
# Run the DELETE and INSERT queries.
self._db.delete(self._calibs, ["id"], *rowsToDelete)
self._db.insert(self._calibs, *rowsToInsert)
Expand Down Expand Up @@ -420,14 +424,16 @@ def select(self, collection: CollectionRecord,
if collection.type is CollectionType.CALIBRATION:
assert self._calibs is not None, \
"DatasetTypes with isCalibration() == False can never be found in a CALIBRATION collection."
tsRepr = self._db.getTimespanRepresentation()
TimespanReprClass = self._db.getTimespanRepresentation()
# Add the timespan column(s) to the result columns, or constrain
# the timespan via an overlap condition.
if timespan is SimpleQuery.Select:
kwargs.update({k: SimpleQuery.Select for k in tsRepr.getFieldNames()})
kwargs.update({k: SimpleQuery.Select for k in TimespanReprClass.getFieldNames()})
elif timespan is not None:
query.where.append(
tsRepr.fromSelectable(self._calibs).overlaps(tsRepr.fromLiteral(timespan))
TimespanReprClass.fromSelectable(self._calibs).overlaps(
TimespanReprClass.fromLiteral(timespan)
)
)
query.join(
self._calibs,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ def makeTagTableSpec(datasetType: DatasetType, collections: Type[CollectionManag


def makeCalibTableSpec(datasetType: DatasetType, collections: Type[CollectionManager],
tsRepr: Type[TimespanDatabaseRepresentation]) -> ddl.TableSpec:
TimespanReprClass: Type[TimespanDatabaseRepresentation]) -> ddl.TableSpec:
"""Construct the specification for a dynamic (DatasetType-dependent) tag +
validity range table used by the classes in this package.
Expand Down Expand Up @@ -525,15 +525,15 @@ def makeCalibTableSpec(datasetType: DatasetType, collections: Type[CollectionMan
)
# Add validity-range field(s) (part of the temporal lookup
# index/constraint).
tsFieldSpecs = tsRepr.makeFieldSpecs(nullable=False)
tsFieldSpecs = TimespanReprClass.makeFieldSpecs(nullable=False)
for fieldSpec in tsFieldSpecs:
tableSpec.fields.add(fieldSpec)
if tsRepr.hasExclusionConstraint():
if TimespanReprClass.hasExclusionConstraint():
# This database's timespan representation can define a database-level
# constraint that prevents overlapping validity ranges for entries with
# the same DatasetType, collection, and data ID.
# This also creates an index.
index.append(tsRepr)
index.append(TimespanReprClass)
tableSpec.exclusion.add(tuple(index))
else:
# No database-level constraint possible. We'll have to simulate that
Expand Down
4 changes: 2 additions & 2 deletions python/lsst/daf/butler/registry/dimensions/governor.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ def initialize(cls, db: Database, element: GovernorDimension, *,
config: Mapping[str, Any]) -> GovernorDimensionRecordStorage:
# Docstring inherited from GovernorDimensionRecordStorage.
spec = element.RecordClass.fields.makeTableSpec(
regRepr=db.getSpatialRegionRepresentation(),
tsRepr=db.getTimespanRepresentation(),
RegionReprClass=db.getSpatialRegionRepresentation(),
TimespanReprClass=db.getTimespanRepresentation(),
)
if context is not None:
table = context.addTable(element.name, spec)
Expand Down
4 changes: 2 additions & 2 deletions python/lsst/daf/butler/registry/dimensions/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ def __init__(self, db: Database, element: DatabaseDimensionElement, viewOf: str)
self._element = element
self._target = element.universe[viewOf]
self._targetSpec = self._target.RecordClass.fields.makeTableSpec(
regRepr=self._db.getSpatialRegionRepresentation(),
tsRepr=self._db.getTimespanRepresentation(),
RegionReprClass=self._db.getSpatialRegionRepresentation(),
TimespanReprClass=self._db.getTimespanRepresentation(),
)
self._viewOf = viewOf
self._query = None # Constructed on first use.
Expand Down
18 changes: 9 additions & 9 deletions python/lsst/daf/butler/registry/dimensions/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,8 @@ def initialize(
) -> DatabaseDimensionRecordStorage:
# Docstring inherited from DatabaseDimensionRecordStorage.
spec = element.RecordClass.fields.makeTableSpec(
regRepr=db.getSpatialRegionRepresentation(),
tsRepr=db.getTimespanRepresentation(),
RegionReprClass=db.getSpatialRegionRepresentation(),
TimespanReprClass=db.getTimespanRepresentation(),
)
if context is not None:
table = context.addTable(element.name, spec)
Expand Down Expand Up @@ -200,24 +200,24 @@ def fetch(self, dataIds: DataCoordinateIterable) -> Iterable[DimensionRecord]:
if self.element.spatial is not None:
query.columns.append(self._table.columns["region"])
if self.element.temporal is not None:
tsRepr = self._db.getTimespanRepresentation()
query.columns.extend(self._table.columns[name] for name in tsRepr.getFieldNames())
TimespanReprClass = self._db.getTimespanRepresentation()
query.columns.extend(self._table.columns[name] for name in TimespanReprClass.getFieldNames())
query.join(self._table)
dataIds.constrain(query, lambda name: self._fetchColumns[name])
for row in self._db.query(query.combine()):
values = dict(row)
if self.element.temporal is not None:
values[TimespanDatabaseRepresentation.NAME] = tsRepr.extract(values)
values[TimespanDatabaseRepresentation.NAME] = TimespanReprClass.extract(values)
yield RecordClass(**values)

def insert(self, *records: DimensionRecord) -> None:
# Docstring inherited from DimensionRecordStorage.insert.
elementRows = [record.toDict() for record in records]
if self.element.temporal is not None:
tsRepr = self._db.getTimespanRepresentation()
TimespanReprClass = self._db.getTimespanRepresentation()
for row in elementRows:
timespan = row.pop(TimespanDatabaseRepresentation.NAME)
tsRepr.update(timespan, result=row)
TimespanReprClass.update(timespan, result=row)
with self._db.transaction():
self._db.insert(self._table, *elementRows)
if self._skyPixOverlap is not None:
Expand All @@ -230,9 +230,9 @@ def sync(self, record: DimensionRecord) -> bool:
for name in record.fields.required.names:
keys[name] = compared.pop(name)
if self.element.temporal is not None:
tsRepr = self._db.getTimespanRepresentation()
TimespanReprClass = self._db.getTimespanRepresentation()
timespan = compared.pop(TimespanDatabaseRepresentation.NAME)
tsRepr.update(timespan, result=compared)
TimespanReprClass.update(timespan, result=compared)
with self._db.transaction():
_, inserted = self._db.sync(
self._table,
Expand Down
4 changes: 2 additions & 2 deletions python/lsst/daf/butler/registry/interfaces/_database.py
Original file line number Diff line number Diff line change
Expand Up @@ -970,7 +970,7 @@ def getTimespanRepresentation(cls) -> Type[TimespanDatabaseRepresentation]:
Returns
-------
tsRepr : `type` (`TimespanDatabaseRepresention` subclass)
TimespanReprClass : `type` (`TimespanDatabaseRepresention` subclass)
A type that encapsulates the way `Timespan` objects should be
stored in this database.
Expand Down Expand Up @@ -1006,7 +1006,7 @@ def getSpatialRegionRepresentation(cls) -> Type[SpatialRegionDatabaseRepresentat
Returns
-------
regRepr : `type` (`SpatialRegionDatabaseRepresention` subclass)
RegionReprClass : `type` (`SpatialRegionDatabaseRepresention` subclass)
A type that encapsulates the way `lsst.sphgeom.Region` objects
should be stored in this database.
Expand Down

0 comments on commit 15afa4a

Please sign in to comment.