From 048e90fbd6e9b4d3ab89bc62387cd3536c7aafca Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Fri, 5 Jun 2026 16:48:43 -0400 Subject: [PATCH 01/15] [no ci] feat: example update to use astropy.Time --- mapcat/database/atomic_coadd.py | 45 ++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index 73653e9..b0355d6 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -2,8 +2,12 @@ Atomic map coadds """ +import uuid +from datetime import datetime from typing import TYPE_CHECKING +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel if TYPE_CHECKING: @@ -12,18 +16,31 @@ from .links import AtomicMapToCoaddTable, CoaddMapToCoaddTable # pragma: no cover +class AtomicMapCoadd(SQLModel): + coadd_id: uuid.UUID + coadd_name: str + prefix_path: str + platform: str + interval: str + start_time: AstroPydanticTime + stop_time: AstroPydanticTime + freq_channel: str + geom_file_path: str + split_label: str + + class AtomicMapCoaddTable(SQLModel, table=True): __tablename__ = "atomic_map_coadds" - coadd_id: int = Field(primary_key=True) + coadd_id: uuid.UUID = Field(default_factory=uuid.uuid7, primary_key=True) coadd_name: str = Field() prefix_path: str = Field() platform: str = Field() interval: str = Field() - start_time: float = Field() - stop_time: float = Field() + start_time: datetime = Field() + stop_time: datetime = Field() freq_channel: str = Field() geom_file_path: str = Field() split_label: str = Field() @@ -50,3 +67,25 @@ class AtomicMapCoaddTable(SQLModel, table=True): "secondaryjoin": "AtomicMapCoaddTable.coadd_id == CoaddMapToCoaddTable.parent_coadd_id", }, ) + + def to_model(self) -> AtomicMapCoadd: + """ + Return an AtomicMapCoadd model from this table entry. + + Returns + ------- + AtomicMapCoadd : AtomicMapCoadd + The AtomicMapCoadd model corresponding to this table entry. + """ + return AtomicMapCoadd( + coadd_id=self.coadd_id, + coadd_name=self.coadd_name, + prefix_path=self.prefix_path, + platform=self.platform, + interval=self.interval, + start_time=Time(self.start_time), + stop_time=Time(self.stop_time), + freq_channel=self.freq_channel, + geom_file_path=self.geom_file_path, + split_label=self.split_label, + ) From 7c42bb43d4d1fe4d3c9c54ec1c47b5efe57428a7 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Fri, 12 Jun 2026 15:59:25 -0400 Subject: [PATCH 02/15] [no ci] feat: refactoring more files to uuids and datetime --- mapcat/database/atomic_coadd.py | 2 + mapcat/database/atomic_map.py | 97 +++++++++++++++++++++++++++++- mapcat/database/depth_one_coadd.py | 58 ++++++++++++++++-- mapcat/database/depth_one_map.py | 70 ++++++++++++++++++--- 4 files changed, 213 insertions(+), 14 deletions(-) diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index b0355d6..025a00d 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -18,8 +18,10 @@ class AtomicMapCoadd(SQLModel): coadd_id: uuid.UUID + coadd_name: str prefix_path: str + platform: str interval: str start_time: AstroPydanticTime diff --git a/mapcat/database/atomic_map.py b/mapcat/database/atomic_map.py index 08753a3..a888320 100644 --- a/mapcat/database/atomic_map.py +++ b/mapcat/database/atomic_map.py @@ -2,8 +2,12 @@ Table for atomic maps. """ +import uuid +from datetime import datetime from typing import TYPE_CHECKING +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel from .links import AtomicMapToCoaddTable @@ -12,16 +16,58 @@ from .atomic_coadd import AtomicMapCoaddTable # pragma: no cover +class AtomicMap(SQLModel): + atomic_map_id: uuid.UUID + + obs_id: str + telescope: str + freq_channel: str + wafer: str + ctime: AstroPydanticTime + split_label: str + + map_path: str | None + ivar_path: str | None + + valid: bool | None + split_detail: str | None + prefix_path: str | None + azimuth: float | None + pwv: float | None + dpwv: float | None + total_weight_qu: float | None + mean_weight_qu: float | None + median_weight_qu: float | None + leakage_avg: float | None + noise_avg: float | None + ampl_2f_avg: float | None + gain_avg: float | None + f_hwp: float | None + roll_angle: float | None + scan_speed: float | None + scan_acc: float | None + sun_distance: float | None + ambient_temperature: float | None + uv: float | None + ra_center: float | None + dec_center: float | None + number_dets: int | None + moon_distance: float | None + wind_speed: float | None + wind_direction: float | None + rqu_avg: float | None + + class AtomicMapTable(SQLModel, table=True): __tablename__ = "atomic_maps" - atomic_map_id: int = Field(primary_key=True) + atomic_map_id: uuid.UUID = Field(default_factory=uuid.uuid7, primary_key=True) obs_id: str = Field() telescope: str = Field() freq_channel: str = Field() wafer: str = Field() - ctime: int = Field() + ctime: datetime = Field() split_label: str = Field() map_path: str | None = Field() @@ -61,3 +107,50 @@ class AtomicMapTable(SQLModel, table=True): back_populates="atomic_maps", link_model=AtomicMapToCoaddTable, ) + + def to_model(self) -> AtomicMap: + """ + Return an AtomicMap model from this table entry. + + Returns + ------- + AtomicMap : AtomicMap + The AtmoicMap model corresponding to this table entry. + """ + return AtomicMap( + atomic_map_id=self.atomic_map_id, + obs_id=self.obs_id, + telescope=self.telescope, + freq_channel=self.freq_channel, + wafer=self.wafer, + ctime=Time(self.ctime), + split_label=self.split_label, + map_path=self.map_path, + ivar_path=self.ivar_path, + valid=self.valid, + split_detail=self.split_detail, + prefix_path=self.prefix_path, + azimuth=self.azimuth, + pwv=self.pwv, + dpwv=self.dpwv, + total_weight_qu=self.total_weight_qu, + mean_weight_qu=self.mean_weight_qu, + leakage_avg=self.leakage_avg, + noise_avg=self.noise_avg, + ampl_2f_avg=self.ampl_2f_ave, + gain_avg=self.gain_avg, + f_hwp=self.f_hwp, + roll_angle=self.roll_angle, + scan_speed=self.scan_speed, + scan_acc=self.scan_acc, + sun_distance=self.sun_distance, + ambient_temperature=self.ambient_temperature, + uv=self.uv, + ra_center=self.ra_center, + dec_center=self.dec_center, + number_dets=self.number_dets, + moon_distance=self.moon_distance, + wind_speed=self.wind_speed, + wind_direction=self.wind_direction, + rqu_avg=self.rqu_avg, + ) diff --git a/mapcat/database/depth_one_coadd.py b/mapcat/database/depth_one_coadd.py index 9660be9..f87dfb6 100644 --- a/mapcat/database/depth_one_coadd.py +++ b/mapcat/database/depth_one_coadd.py @@ -2,12 +2,36 @@ Table containing information about Depth-1 map coadds. """ +import uuid +from datetime import datetime + +from astropy.time import Time from sqlmodel import Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable from .links import DepthOneToCoaddTable +class DepthOneCoadd(SQLModel): + coadd_id: uuid.UUID + coadd_name: str + coadd_type: str + + map_path: str + ivar_path: str | None + rho_path: str | None + kappa_path: str | None + + start_time_path: str | None + mean_time_path: str | None + end_time_path: str | None + + frequency: str + ctime: datetime + start_time: datetime + stop_time: datetime + + class DepthOneCoaddTable(SQLModel, table=True): """ A co-add of multiple depth-1 maps. This is the table model, @@ -16,7 +40,7 @@ class DepthOneCoaddTable(SQLModel, table=True): __tablename__ = "depth_one_coadds" - coadd_id: int = Field(primary_key=True) + coadd_id: int = Field(default_factory=uuid.uuid7, primary_key=True) coadd_name: str = Field(nullable=False) coadd_type: str = Field(nullable=False) @@ -30,11 +54,37 @@ class DepthOneCoaddTable(SQLModel, table=True): end_time_path: str | None = None frequency: str = Field(nullable=False) - ctime: float = Field(nullable=False) - start_time: float = Field(nullable=False) - stop_time: float = Field(nullable=False) + ctime: datetime = Field(nullable=False) + start_time: datetime = Field(nullable=False) + stop_time: datetime = Field(nullable=False) maps: list["DepthOneMapTable"] = Relationship( back_populates="coadds", link_model=DepthOneToCoaddTable, ) + + def to_model(self) -> DepthOneCoadd: + """ + Return an DepthOneCoadd model from this table entry. + + Returns + ------- + DepthOneCoadd : DepthOneCoadd + The DepthOneCoadd model corresponding to this table entry. + """ + return DepthOneCoadd( + coadd_id=self.coadd_id, + coadd_name=self.coadd_name, + coadd_type=self.coadd_type, + map_path=self.map_path, + ivar_path=self.ivar_path, + rho_path=self.rho_path, + kappa_path=self.kappa_path, + start_time_path=self.start_time_path, + mean_time_path=self.mean_time_path, + end_time_path=self.end_time_path, + frequency=self.frequency, + ctime=Time(self.ctime), + start_time=Time(self.start_time), + stop_time=Time(self.end_time), + ) diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py index 6760f6a..8c63f98 100644 --- a/mapcat/database/depth_one_map.py +++ b/mapcat/database/depth_one_map.py @@ -2,8 +2,12 @@ Depth one map table. """ +import uuid +from datetime import datetime from typing import TYPE_CHECKING, Any +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import JSON, Field, Relationship, SQLModel if TYPE_CHECKING: # pragma: no cover @@ -17,13 +21,35 @@ from .links import DepthOneToCoaddTable, TODToMapTable +class DepthOneMap(SQLModel): + map_id: uuid.UUID + map_name: str + + map_path: str | None + ivar_path: str | None + rho_path: str | None + kappa_path: str | None + flux_path: str | None + snr_path: str | None + + start_time_path: str | None + mean_time_path: str | None + end_time_path: str | None + + tube_slot: str + frequency: str + ctime: AstroPydanticTime + start_time: AstroPydanticTime + stop_time: AstroPydanticTime + + class DepthOneMapTable(SQLModel, table=True): """ A depth-1 map. Attributes ---------- - id : int + id : uuid.UUID Unique map identifiers. Internal to SO map_name : str Name of depth 1 map @@ -54,11 +80,11 @@ class DepthOneMapTable(SQLModel, table=True): Standardized names of wafers used in this map frequency : str Frequency channel of map - ctime : float + ctime : datetime Mean unix time of map - start_time : float + start_time : datetime Start unix time of map - stop_time : float + stop_time : datetime Stop unix time of map processing_status : list[TimeDomainProcessingTable] List of processing status tables associated with d1 map @@ -77,7 +103,7 @@ class DepthOneMapTable(SQLModel, table=True): __tablename__ = "depth_one_maps" - map_id: int = Field(primary_key=True) + map_id: uuid.UUID = Field(default_factory=uuid.uuid7, primary_key=True) map_name: str = Field(index=True, unique=True, nullable=False) map_path: str | None = None @@ -93,9 +119,9 @@ class DepthOneMapTable(SQLModel, table=True): tube_slot: str = Field(index=True, nullable=False) frequency: str = Field(index=True, nullable=False) - ctime: float = Field(index=True, nullable=False) - start_time: float = Field(index=True, nullable=False) - stop_time: float = Field(index=True, nullable=False) + ctime: datetime = Field(index=True, nullable=False) + start_time: datetime = Field(index=True, nullable=False) + stop_time: datetime = Field(index=True, nullable=False) processing_status: list["TimeDomainProcessingTable"] = Relationship( back_populates="map", @@ -146,3 +172,31 @@ def coverage_path(self) -> str: raise ValueError( f"No coverage map available for map {self.map_name} (id {self.map_id})" ) + + def to_model(self) -> DepthOneMap: + """ + Return an DepthOneMap model from this table entry. + + Returns + ------- + DepthOneMap : DepthOneMap + The DepthOneMap model corresponding to this table entry. + """ + return DepthOneMap( + map_id=self.map_id, + map_name=self.map_name, + map_path=self.map_path, + ivar_path=self.ivar_path, + rho_path=self.rho_path, + kappa_path=self.kappa_path, + flux_path=self.flux_path, + snr_path=self.snr_path, + start_time_path=self.start_time_path, + mean_time_path=self.mean_time_path, + end_time_path=self.end_time_path, + tube_slot=self.tube_slot, + frequency=self.frequency, + ctime=Time(self.ctime), + start_time=Time(self.start_time), + stop_time=Time(self.stop_time), + ) From 2e0cd5d1221819d35a518373763567f8ca36e00d Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Tue, 23 Jun 2026 16:59:53 -0400 Subject: [PATCH 03/15] (no ci) fix: remaining DB entries to uuid/astropy.Time and starting to update tools and tests --- mapcat/database/links.py | 18 ++--- mapcat/database/pipeline_information.py | 9 ++- mapcat/database/pointing_residual.py | 10 ++- mapcat/database/sky_coverage.py | 6 +- mapcat/database/time_domain_processing.py | 50 +++++++++++--- mapcat/database/tod.py | 84 ++++++++++++++++++++--- mapcat/toolkit/act.py | 9 +-- mapcat/toolkit/reset.py | 8 ++- tests/test_act.py | 4 +- 9 files changed, 158 insertions(+), 40 deletions(-) diff --git a/mapcat/database/links.py b/mapcat/database/links.py index 1db33b2..e32c81b 100644 --- a/mapcat/database/links.py +++ b/mapcat/database/links.py @@ -2,6 +2,8 @@ Link tables. """ +import uuid + from sqlmodel import Field, SQLModel @@ -12,14 +14,14 @@ class DepthOneToCoaddTable(SQLModel, table=True): __tablename__ = "link_depth_one_map_to_coadd" - map_id: int = Field( + map_id: uuid.UUID = Field( foreign_key="depth_one_maps.map_id", primary_key=True, nullable=False, index=True, ondelete="CASCADE", ) - coadd_id: int = Field( + coadd_id: uuid.UUID = Field( foreign_key="depth_one_coadds.coadd_id", primary_key=True, nullable=False, @@ -35,14 +37,14 @@ class TODToMapTable(SQLModel, table=True): __tablename__ = "link_tod_to_depth_one_map" - tod_id: int = Field( + tod_id: uuid.UUID = Field( foreign_key="tod_depth_one.tod_id", primary_key=True, nullable=False, index=True, ondelete="CASCADE", ) - map_id: int = Field( + map_id: uuid.UUID = Field( foreign_key="depth_one_maps.map_id", primary_key=True, nullable=False, @@ -58,7 +60,7 @@ class AtomicMapToCoaddTable(SQLModel, table=True): __tablename__ = "link_atomic_map_to_coadd" - atomic_map_id: int = Field( + atomic_map_id: uuid.UUID = Field( foreign_key="atomic_maps.atomic_map_id", primary_key=True, nullable=False, @@ -66,7 +68,7 @@ class AtomicMapToCoaddTable(SQLModel, table=True): ondelete="CASCADE", ) - coadd_id: int = Field( + coadd_id: uuid.UUID = Field( foreign_key="atomic_map_coadds.coadd_id", primary_key=True, nullable=False, @@ -82,7 +84,7 @@ class CoaddMapToCoaddTable(SQLModel, table=True): __tablename__ = "link_coadd_map_to_coadd" - parent_coadd_id: int = Field( + parent_coadd_id: uuid.UUID = Field( foreign_key="atomic_map_coadds.coadd_id", primary_key=True, nullable=False, @@ -90,7 +92,7 @@ class CoaddMapToCoaddTable(SQLModel, table=True): ondelete="CASCADE", ) - child_coadd_id: int = Field( + child_coadd_id: uuid.UUID = Field( foreign_key="atomic_map_coadds.coadd_id", primary_key=True, nullable=False, diff --git a/mapcat/database/pipeline_information.py b/mapcat/database/pipeline_information.py index 3f47ee0..4cdf988 100644 --- a/mapcat/database/pipeline_information.py +++ b/mapcat/database/pipeline_information.py @@ -2,6 +2,7 @@ Information on the map making pipeline run. """ +import uuid from typing import Any from sqlmodel import JSON, Field, Relationship, SQLModel @@ -15,7 +16,7 @@ class PipelineInformationTable(SQLModel, table=True): Attributes ---------- - id : str + id : uuid.UUID Internal ID of the pipeline info map_name : str Name of depth 1 map being tracked. Foreign into DepthOneMap @@ -29,8 +30,10 @@ class PipelineInformationTable(SQLModel, table=True): __tablename__ = "pipeline_information" - pipeline_information_id: int = Field(primary_key=True) - map_id: int = Field(foreign_key="depth_one_maps.map_id", nullable=False) + pipeline_information_id: uuid.UUID = Field( + default_factory=uuid.uuid7, primary_key=True + ) + map_id: uuid.UUID = Field(foreign_key="depth_one_maps.map_id", nullable=False) map: DepthOneMapTable = Relationship(back_populates="pipeline_information") sotodlib_version: str diff --git a/mapcat/database/pointing_residual.py b/mapcat/database/pointing_residual.py index 7f9a14e..2d81e9a 100644 --- a/mapcat/database/pointing_residual.py +++ b/mapcat/database/pointing_residual.py @@ -2,6 +2,8 @@ Table containing pointing residuals. """ +import uuid + from sqlmodel import Field, Relationship, SQLModel from mapcat.pointing.base import PointingModelStats @@ -22,7 +24,7 @@ class PointingResidualTable(SQLModel, table=True): Attributes ---------- - map_id : int + map_id : uuid.UUID Internal ID of the depth one map residual_model: ConstantPointingModel | PolynomialPointingModel The pointing model to actually store in the database. @@ -31,9 +33,11 @@ class PointingResidualTable(SQLModel, table=True): """ __tablename__ = "depth_one_pointing_residuals" - pointing_residual_id: int = Field(primary_key=True) + pointing_residual_id: uuid.UUID = Field( + default_factory=uuid.uuid7, primary_key=True + ) - map_id: int = Field( + map_id: uuid.UUID = Field( index=True, nullable=False, foreign_key="depth_one_maps.map_id", diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index fa5021f..3f21d1e 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -2,6 +2,8 @@ Sky coverage table. """ +import uuid + from sqlalchemy import PrimaryKeyConstraint from sqlmodel import Field, Relationship, SQLModel @@ -19,7 +21,7 @@ class SkyCoverageTable(SQLModel, table=True): Composite ID from map_id, x, and y map : DepthOneMapTable Depth 1 map being tracked. Foreign into DepthOneMap - map_id : int + map_id : uuid.UUID ID of depth 1 map being tracked x : int x-index of coverage patch. x=0 runs from RA 0 to 10,, etc. @@ -32,7 +34,7 @@ class SkyCoverageTable(SQLModel, table=True): x: int = Field(index=True, primary_key=True) y: int = Field(index=True, primary_key=True) - map_id: int = Field( + map_id: uuid.UUID = Field( foreign_key="depth_one_maps.map_id", nullable=False, ondelete="CASCADE", diff --git a/mapcat/database/time_domain_processing.py b/mapcat/database/time_domain_processing.py index f271437..1d37eaa 100644 --- a/mapcat/database/time_domain_processing.py +++ b/mapcat/database/time_domain_processing.py @@ -2,11 +2,26 @@ Table containing information about processing status of the Depth-1 maps. """ +import uuid +from datetime import datetime + +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable +class TimeDomainProcessing(SQLModel): + processing_status_id: uuid.UUID + + map_id: uuid.UUID + + processing_start: AstroPydanticTime | None + processing_end: AstroPydanticTime | None + processing_status: str + + class TimeDomainProcessingTable(SQLModel, table=True): """ Table for tracking processing status of depth-1 maps @@ -16,13 +31,13 @@ class TimeDomainProcessingTable(SQLModel, table=True): Attributes ---------- - id : int + processing_status_id : uuid.UUID Internal ID of the processing status - map_name : str + map_name : uuid.UUID Name of depth 1 map being tracked. Foreign into DepthOneMap - processing_start : float | None + processing_start : datetime | None Time processing started. None if not started. - processing_end : float | None + processing_end : datetime | None Time processing ended. None if not ended. processing_status : str Status of processing @@ -30,9 +45,11 @@ class TimeDomainProcessingTable(SQLModel, table=True): __tablename__ = "time_domain_processing" - processing_status_id: int = Field(primary_key=True) + processing_status_id: uuid.UUID = Field( + prdefault_factory=uuid.uuid7, primary_key=True + ) - map_id: int = Field( + map_id: uuid.UUID = Field( index=True, nullable=False, foreign_key="depth_one_maps.map_id", @@ -40,6 +57,23 @@ class TimeDomainProcessingTable(SQLModel, table=True): ) map: DepthOneMapTable = Relationship(back_populates="processing_status") - processing_start: float = Field(nullable=True) - processing_end: float = Field(nullable=True) + processing_start: datetime = Field(nullable=True) + processing_end: datetime = Field(nullable=True) processing_status: str = Field(index=True, nullable=False) + + def to_model(self) -> TimeDomainProcessing: + """ + Return a TimeDomainProcessing model from this table entry + + Returns + ------- + TimeDomainProcessing : TimeDomainProcessing + The TimeDomainProcessing model corresponding to this table entry. + """ + return TimeDomainProcessing( + processing_status_id=self.processing_status_id, + map_id=self.map_id, + processing_start=Time(self.processing_start), + processing_end=Time(self.processing_end), + processing_status=self.processing_status, + ) diff --git a/mapcat/database/tod.py b/mapcat/database/tod.py index dcc8ec4..8470691 100644 --- a/mapcat/database/tod.py +++ b/mapcat/database/tod.py @@ -2,19 +2,51 @@ Table for TODs """ +import uuid +from datetime import datetime + +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable from .links import TODToMapTable +class TODDepthOne(SQLModel): + tod_id: uuid.UUID + obs_id: str + pwv: float | None + ctime: AstroPydanticTime + start_time: AstroPydanticTime | None + stop_time: AstroPydanticTime | None + nsamples: int | None + telescope: str + telescope_flavor: str | None + tube_slot: str + tube_flavor: str | None + frequency: str + scan_type: str + subtype: str + wafer_count: int + duration: float + az_center: float + az_throw: float + el_center: float + el_throw: float + roll_center: float + roll_throw: float + wafer_slots_list: str + stream_ids_list: str + + class TODDepthOneTable(SQLModel, table=True): """ Table of TODs used in making depth 1 maps. Attributes ---------- - id : int + tod_id : uuid.UUID Unique TOD identifier. Internal to SO map_name : str Name of map this TOD went into. Foreign key @@ -22,11 +54,11 @@ class TODDepthOneTable(SQLModel, table=True): SO ID of TOD pwv : float Precipitable water vapor at time of obs - ctime : float + ctime : datetime Mean unix time of obs - start_time : float + start_time : datetime Start time of obs - stop_time : float + stop_time : datetime End time of obs nsamples : int Number of samps in obs @@ -67,12 +99,12 @@ class TODDepthOneTable(SQLModel, table=True): """ __tablename__ = "tod_depth_one" - tod_id: int = Field(primary_key=True) + tod_id: uuid.UUID = Field(default_factory=uuid.uuid7, primary_key=True) obs_id: str = Field(nullable=False) pwv: float | None = Field(index=True, nullable=True) - ctime: float = Field(index=True, nullable=False) - start_time: float | None = Field(index=True, nullable=True) - stop_time: float | None = Field(index=True, nullable=True) + ctime: datetime = Field(index=True, nullable=False) + start_time: datetime | None = Field(index=True, nullable=True) + stop_time: datetime | None = Field(index=True, nullable=True) nsamples: int | None = Field() telescope: str = Field(index=True, nullable=False) telescope_flavor: str | None = Field() @@ -94,3 +126,39 @@ class TODDepthOneTable(SQLModel, table=True): maps: list[DepthOneMapTable] = Relationship( back_populates="tods", link_model=TODToMapTable ) + + def to_model(self) -> TODDepthOne: + """ + Return an TODDepthOne model from this table entry. + + Returns + ------- + TODDepthOne : TODDepthOne + The TODDepthOne model corresponding to this table entry. + """ + return TODDepthOne( + tod_id=self.tod_id, + obs_id=self.obs_id, + pwv=self.pwv, + ctime=Time(self.ctime), + start_time=Time(self.start_time), + stop_time=Time(self.stop_time), + nsamples=self.nsamples, + telescope=self.telescope, + telescope_flavor=self.telescope_flavor, + tube_slot=self.tube_slot, + tube_flavor=self.tube_flavor, + frequency=self.frequency, + scan_type=self.scan_type, + subtype=self.subtype, + wafer_count=self.wafer_count, + duration=self.duration, + az_center=self.az_center, + az_throw=self.az_throw, + el_center=self.el_center, + el_throw=self.el_throw, + roll_center=self.roll_center, + roll_throw=self.roll_throw, + wafer_slots_list=self.wafer_slots_list, + stream_ids_list=self.stream_ids_list, + ) diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index 8586f65..d774235 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -3,6 +3,7 @@ """ import argparse as ap +import datetime from pathlib import Path import h5py @@ -70,7 +71,7 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT TODDepthOneTable( obs_id=obs_id, pwv=None, - ctime=float(obs_id[4:14]), + ctime=datetime.fromtimestamp(float(obs_id[4:14])), telescope=telescope, tube_slot=file_info["tube_slot"], frequency=file_info["frequency"], @@ -87,9 +88,9 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT mean_time_path=filenames.get("time"), tube_slot=file_info["tube_slot"], frequency=file_info["frequency"], - ctime=file_info["ctime"], - start_time=file_info["start_time"], - stop_time=file_info["stop_time"], + ctime=datetime.fromtimestamp(file_info["ctime"]), + start_time=datetime.fromtimestamp(file_info["start_time"]), + stop_time=datetime.fromtimestamp(file_info["stop_time"]), tods=tods, ) diff --git a/mapcat/toolkit/reset.py b/mapcat/toolkit/reset.py index 78cbc18..7187161 100644 --- a/mapcat/toolkit/reset.py +++ b/mapcat/toolkit/reset.py @@ -102,9 +102,13 @@ def core(session: sessionmaker, args: ap.Namespace): PointingResidualTable.map_id == DepthOneMapTable.map_id, ) if args.start_time is not None: - pr_stmt = pr_stmt.where(DepthOneMapTable.ctime >= args.start_time) + pr_stmt = pr_stmt.where( + DepthOneMapTable.ctime.timestamp() >= args.start_time + ) if args.end_time is not None: - pr_stmt = pr_stmt.where(DepthOneMapTable.ctime <= args.end_time) + pr_stmt = pr_stmt.where( + DepthOneMapTable.ctime.timestamp() <= args.end_time + ) pointing_residuals = cur_session.execute(pr_stmt).scalars().all() for pr in pointing_residuals: diff --git a/tests/test_act.py b/tests/test_act.py index 580c033..cc0af4d 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -218,7 +218,7 @@ def test_act(database_sessionmaker, downloaded_data_file): for map in maps: assert map.tube_slot in ["pa4", "pa6"] assert map.frequency == "f150" - assert map.ctime in [1505603190, 1505646390] + assert int(map.ctime.timestamp()) in [1505603190, 1505646390] # Clean up, otherewise we interfere with test_sky_coverage session.delete(map) @@ -248,7 +248,7 @@ def test_sky_coverage(database_sessionmaker, downloaded_data_file): cov.x ), # These should be ints, idk why I have to cast them (from str) int(cov.y), - ) in cov_mapping[str(d1map.ctime)] + ) in cov_mapping[str(d1map.ctime.timestamp())] with database_sessionmaker() as session: maps = session.query(DepthOneMapTable).all() From ac38e254480e0400967109c2d060c90591ab0b31 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Wed, 24 Jun 2026 13:21:11 -0400 Subject: [PATCH 04/15] [no ci] feat: finalize updating tests and add timezone specification to all fromtimestamp --- tests/test_mapcat.py | 74 +++++++++++++++++++++-------------------- tests/test_mapmaking.py | 38 +++++++++++---------- tests/test_pointing.py | 8 +++-- tests/test_reset.py | 13 +++++--- 4 files changed, 72 insertions(+), 61 deletions(-) diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index c92629e..fec2965 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -2,6 +2,8 @@ Test the core functions """ +from datetime import UTC, datetime + import pytest from astropy import units as u from sqlalchemy import create_engine @@ -70,9 +72,9 @@ def test_create_depth_one(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE", tube_slot="OTi1", frequency="f090", - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), ) session.add(data) @@ -90,13 +92,13 @@ def test_create_depth_one(database_sessionmaker): assert dmap.map_path == "/PATH/TO/DEPTH/ONE" assert dmap.tube_slot == "OTi1" assert dmap.frequency == "f090" - assert dmap.ctime == 1755787524.0 + assert dmap.ctime.unix == 1755787524.0 # Make child tables with database_sessionmaker() as session: processing_status = TimeDomainProcessingTable( - processing_start=1756787524.0, - processing_end=1756797524.0, + processing_start=datetime.fromtimestamp(1756787524.0, tz=UTC), + processing_end=datetime.fromtimestamp(1756797524.0, tz=UTC), processing_status="done", map_id=map_id, ) @@ -111,9 +113,9 @@ def test_create_depth_one(database_sessionmaker): tod = TODDepthOneTable( obs_id="obs_1753486724_lati6_111", pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -169,8 +171,8 @@ def test_create_depth_one(database_sessionmaker): assert proc.processing_status_id == proc_id assert proc.map_id == map_id - assert proc.processing_start == 1756787524.0 - assert proc.processing_end == 1756797524.0 + assert proc.processing_start.unix == 1756787524.0 + assert proc.processing_end.unix == 1756797524.0 assert proc.processing_status == "done" assert point.pointing_residual_id == point_id @@ -181,9 +183,9 @@ def test_create_depth_one(database_sessionmaker): assert tod.tod_id == tod_id assert tod.pwv == 0.7 assert tod.obs_id == "obs_1753486724_lati6_111" - assert tod.ctime == 1755787524.0 - assert tod.start_time == 1755687524.0 - assert tod.stop_time == 1755887524.0 + assert tod.ctime.unix == 1755787524.0 + assert tod.start_time.unix == 1755687524.0 + assert tod.stop_time.unix == 1755887524.0 assert tod.nsamples == 28562 assert tod.telescope == "lat" assert tod.telescope_flavor == "lat" @@ -228,14 +230,14 @@ def test_add_remove_child_tables(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE2", tube_slot="OTi1", frequency="f090", - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), ) processing_status = TimeDomainProcessingTable( - processing_start=1756787524.0, - processing_end=1756797524.0, + processing_start=datetime.fromtimestamp(1756787524.0, tz=UTC), + processing_end=datetime.fromtimestamp(1756797524.0, tz=UTC), processing_status="done", map=dmap, ) @@ -250,9 +252,9 @@ def test_add_remove_child_tables(database_sessionmaker): tod = TODDepthOneTable( obs_id="obs_1753486724_lati6_111", pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -345,8 +347,8 @@ def test_create_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/DAILY/COADD", platform="satp3", interval="daily", - start_time=1755604800.0, - stop_time=1755691200.0, + start_time=datetime.fromtimestamp(1755604800.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755691200.0, tz=UTC), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -367,8 +369,8 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert cmap.prefix_path == "/PATH/TO/DAILY/COADD" assert cmap.platform == "satp3" assert cmap.interval == "daily" - assert cmap.start_time == 1755604800.0 - assert cmap.stop_time == 1755691200.0 + assert cmap.start_time.unix == 1755604800.0 + assert cmap.stop_time.unix == 1755691200.0 assert cmap.freq_channel == "f090" assert cmap.geom_file_path == "/PATH/TO/GEOM/FILE" assert cmap.split_label == "full" @@ -380,7 +382,7 @@ def test_create_atomic_map_coadd(database_sessionmaker): telescope="satp3", freq_channel="f090", wafer="ws0", - ctime=1755643932, + ctime=datetime.fromtimestamp(1755643932, tz=UTC), split_label="full", map_path=None, ivar_path=None, @@ -431,7 +433,7 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert atomic.telescope == "satp3" assert atomic.freq_channel == "f090" assert atomic.wafer == "ws0" - assert atomic.ctime == 1755643932 + assert atomic.ctime.unix == 1755643932 assert atomic.split_label == "full" assert atomic.map_path is None assert atomic.ivar_path is None @@ -474,8 +476,8 @@ def test_create_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/WEEKLY/COADD", platform="satp3", interval="weekly", - start_time=1755432000.0, - stop_time=1756036800.0, + start_time=datetime.fromtimestamp(1755432000.0, tz=UTC), + stop_time=datetime.fromtimestamp(1756036800.0, tz=UTC), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -518,8 +520,8 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/DAILY/COADD", platform="satp3", interval="daily", - start_time=1755604800.0, - stop_time=1755691200.0, + start_time=datetime.fromtimestamp(1755604800.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755691200.0, tz=UTC), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -530,8 +532,8 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/WEEKLY/COADD", platform="satp3", interval="weekly", - start_time=1755432000.0, - stop_time=1756036800.0, + start_time=datetime.fromtimestamp(1755432000.0, tz=UTC), + stop_time=datetime.fromtimestamp(1756036800.0, tz=UTC), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -567,8 +569,8 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/WEEKLY/COADD", platform="satp3", interval="weekly", - start_time=1755432000.0, - stop_time=1756036800.0, + start_time=datetime.fromtimestamp(1755432000.0, tz=UTC), + stop_time=datetime.fromtimestamp(1756036800.0, tz=UTC), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", diff --git a/tests/test_mapmaking.py b/tests/test_mapmaking.py index a681cfc..983ebbc 100644 --- a/tests/test_mapmaking.py +++ b/tests/test_mapmaking.py @@ -1,3 +1,5 @@ +from datetime import UTC, datetime + from mapcat.database import DepthOneMapTable, TODDepthOneTable from mapcat.toolkit.mapmaking import build_obslists @@ -12,9 +14,9 @@ def test_build_obslists(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE", tube_slot="OTi1", frequency="f090", - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), ) data2 = DepthOneMapTable( @@ -22,9 +24,9 @@ def test_build_obslists(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE2", tube_slot="OTi4", frequency="f090", - ctime=1755788524.0, - start_time=1755787524.0, - stop_time=1755897524.0, + ctime=datetime.fromtimestamp(1755788524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755787524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755897524.0, tz=UTC), ) session.add(data1) @@ -54,9 +56,9 @@ def test_build_obslists(database_sessionmaker): tod1 = TODDepthOneTable( obs_id=obs_ids[0], pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -80,9 +82,9 @@ def test_build_obslists(database_sessionmaker): tod2 = TODDepthOneTable( obs_id=obs_ids[1], pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -106,9 +108,9 @@ def test_build_obslists(database_sessionmaker): tod3 = TODDepthOneTable( obs_id=obs_ids[2], pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -133,9 +135,9 @@ def test_build_obslists(database_sessionmaker): tod4 = TODDepthOneTable( obs_id=obs_ids[3], pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), nsamples=28562, telescope="lat", telescope_flavor="lat", diff --git a/tests/test_pointing.py b/tests/test_pointing.py index 44fdff7..fff0463 100644 --- a/tests/test_pointing.py +++ b/tests/test_pointing.py @@ -2,6 +2,8 @@ Tests for the pointing residual models. """ +from datetime import UTC, datetime + import numpy as np from astropy import units as u from astropy.coordinates import SkyCoord @@ -21,9 +23,9 @@ def test_add_retrieve_pointing(database_sessionmaker): map_path="DoesntExist/Map", tube_slot="i1", frequency="f090", - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), + start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), + stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), ) session.add(sample_map) diff --git a/tests/test_reset.py b/tests/test_reset.py index 2650631..30d2a46 100644 --- a/tests/test_reset.py +++ b/tests/test_reset.py @@ -3,6 +3,7 @@ """ import argparse +from datetime import UTC, datetime import pytest from sqlalchemy import create_engine @@ -44,14 +45,18 @@ def database_sessionmaker(tmp_path_factory): def _make_map(session, name, ctime, start_time=None, stop_time=None): """Helper to insert a DepthOneMapTable row and return its map_id.""" with session() as s: + if start_time is None: + start_time = ctime - 500 + if stop_time is None: + stop_time = ctime + 500 dmap = DepthOneMapTable( map_name=name, map_path=f"/path/{name}_map.fits", tube_slot="OTi1", frequency="f090", ctime=ctime, - start_time=start_time or ctime - 500, - stop_time=stop_time or ctime + 500, + start_time=datetime.fromtimestamp(start_time, tz=UTC), + stop_time=datetime.fromtimestamp(stop_time, tz=UTC), ) s.add(dmap) s.commit() @@ -64,8 +69,8 @@ def _make_proc(session, map_id, status): with session() as s: proc = TimeDomainProcessingTable( map_id=map_id, - processing_start=1756000000.0, - processing_end=1756001000.0, + processing_start=datetime.fromtimestamp(1756000000.0, tz=UTC), + processing_end=datetime.fromtimestamp(1756001000.0, tz=UTC), processing_status=status, ) s.add(proc) From 022edf61d5ba93d0e5dd10a04a6bbea9eb6b0959 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Wed, 24 Jun 2026 13:24:21 -0400 Subject: [PATCH 05/15] [no ci] fix: use correct uuid --- mapcat/database/atomic_coadd.py | 2 +- mapcat/database/atomic_map.py | 2 +- mapcat/database/depth_one_coadd.py | 2 +- mapcat/database/depth_one_map.py | 2 +- mapcat/database/links.py | 3 +-- mapcat/database/pipeline_information.py | 2 +- mapcat/database/pointing_residual.py | 3 +-- mapcat/database/sky_coverage.py | 3 +-- mapcat/database/time_domain_processing.py | 2 +- mapcat/database/tod.py | 2 +- 10 files changed, 10 insertions(+), 13 deletions(-) diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index 025a00d..c3be2e5 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -2,10 +2,10 @@ Atomic map coadds """ -import uuid from datetime import datetime from typing import TYPE_CHECKING +import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel diff --git a/mapcat/database/atomic_map.py b/mapcat/database/atomic_map.py index a888320..6cc8bc5 100644 --- a/mapcat/database/atomic_map.py +++ b/mapcat/database/atomic_map.py @@ -2,10 +2,10 @@ Table for atomic maps. """ -import uuid from datetime import datetime from typing import TYPE_CHECKING +import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel diff --git a/mapcat/database/depth_one_coadd.py b/mapcat/database/depth_one_coadd.py index f87dfb6..3cf2d27 100644 --- a/mapcat/database/depth_one_coadd.py +++ b/mapcat/database/depth_one_coadd.py @@ -2,9 +2,9 @@ Table containing information about Depth-1 map coadds. """ -import uuid from datetime import datetime +import uuid7 as uuid from astropy.time import Time from sqlmodel import Field, Relationship, SQLModel diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py index 8c63f98..0520541 100644 --- a/mapcat/database/depth_one_map.py +++ b/mapcat/database/depth_one_map.py @@ -2,10 +2,10 @@ Depth one map table. """ -import uuid from datetime import datetime from typing import TYPE_CHECKING, Any +import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import JSON, Field, Relationship, SQLModel diff --git a/mapcat/database/links.py b/mapcat/database/links.py index e32c81b..97593d4 100644 --- a/mapcat/database/links.py +++ b/mapcat/database/links.py @@ -2,8 +2,7 @@ Link tables. """ -import uuid - +import uuid7 as uuid from sqlmodel import Field, SQLModel diff --git a/mapcat/database/pipeline_information.py b/mapcat/database/pipeline_information.py index 4cdf988..2f4d9e6 100644 --- a/mapcat/database/pipeline_information.py +++ b/mapcat/database/pipeline_information.py @@ -2,9 +2,9 @@ Information on the map making pipeline run. """ -import uuid from typing import Any +import uuid7 as uuid from sqlmodel import JSON, Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable diff --git a/mapcat/database/pointing_residual.py b/mapcat/database/pointing_residual.py index 2d81e9a..28ffd35 100644 --- a/mapcat/database/pointing_residual.py +++ b/mapcat/database/pointing_residual.py @@ -2,8 +2,7 @@ Table containing pointing residuals. """ -import uuid - +import uuid7 as uuid from sqlmodel import Field, Relationship, SQLModel from mapcat.pointing.base import PointingModelStats diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index 3f21d1e..cb6ecd0 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -2,8 +2,7 @@ Sky coverage table. """ -import uuid - +import uuid7 as uuid from sqlalchemy import PrimaryKeyConstraint from sqlmodel import Field, Relationship, SQLModel diff --git a/mapcat/database/time_domain_processing.py b/mapcat/database/time_domain_processing.py index 1d37eaa..72d1e0f 100644 --- a/mapcat/database/time_domain_processing.py +++ b/mapcat/database/time_domain_processing.py @@ -2,9 +2,9 @@ Table containing information about processing status of the Depth-1 maps. """ -import uuid from datetime import datetime +import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel diff --git a/mapcat/database/tod.py b/mapcat/database/tod.py index 8470691..0df33fb 100644 --- a/mapcat/database/tod.py +++ b/mapcat/database/tod.py @@ -2,9 +2,9 @@ Table for TODs """ -import uuid from datetime import datetime +import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel From acb6e209cc83b0e90b2e56fbbac9580fc625723b Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Wed, 24 Jun 2026 13:26:23 -0400 Subject: [PATCH 06/15] [no ci] fix: use correct uuid factory --- mapcat/database/atomic_coadd.py | 2 +- mapcat/database/atomic_map.py | 2 +- mapcat/database/depth_one_coadd.py | 2 +- mapcat/database/depth_one_map.py | 2 +- mapcat/database/pipeline_information.py | 2 +- mapcat/database/pointing_residual.py | 2 +- mapcat/database/time_domain_processing.py | 2 +- mapcat/database/tod.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index c3be2e5..6f79fa5 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -34,7 +34,7 @@ class AtomicMapCoadd(SQLModel): class AtomicMapCoaddTable(SQLModel, table=True): __tablename__ = "atomic_map_coadds" - coadd_id: uuid.UUID = Field(default_factory=uuid.uuid7, primary_key=True) + coadd_id: uuid.UUID = Field(default_factory=uuid.create, primary_key=True) coadd_name: str = Field() prefix_path: str = Field() diff --git a/mapcat/database/atomic_map.py b/mapcat/database/atomic_map.py index 6cc8bc5..ab39043 100644 --- a/mapcat/database/atomic_map.py +++ b/mapcat/database/atomic_map.py @@ -61,7 +61,7 @@ class AtomicMap(SQLModel): class AtomicMapTable(SQLModel, table=True): __tablename__ = "atomic_maps" - atomic_map_id: uuid.UUID = Field(default_factory=uuid.uuid7, primary_key=True) + atomic_map_id: uuid.UUID = Field(default_factory=uuid.create, primary_key=True) obs_id: str = Field() telescope: str = Field() diff --git a/mapcat/database/depth_one_coadd.py b/mapcat/database/depth_one_coadd.py index 3cf2d27..9645e85 100644 --- a/mapcat/database/depth_one_coadd.py +++ b/mapcat/database/depth_one_coadd.py @@ -40,7 +40,7 @@ class DepthOneCoaddTable(SQLModel, table=True): __tablename__ = "depth_one_coadds" - coadd_id: int = Field(default_factory=uuid.uuid7, primary_key=True) + coadd_id: int = Field(default_factory=uuid.create, primary_key=True) coadd_name: str = Field(nullable=False) coadd_type: str = Field(nullable=False) diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py index 0520541..feded42 100644 --- a/mapcat/database/depth_one_map.py +++ b/mapcat/database/depth_one_map.py @@ -103,7 +103,7 @@ class DepthOneMapTable(SQLModel, table=True): __tablename__ = "depth_one_maps" - map_id: uuid.UUID = Field(default_factory=uuid.uuid7, primary_key=True) + map_id: uuid.UUID = Field(default_factory=uuid.create, primary_key=True) map_name: str = Field(index=True, unique=True, nullable=False) map_path: str | None = None diff --git a/mapcat/database/pipeline_information.py b/mapcat/database/pipeline_information.py index 2f4d9e6..3c04d37 100644 --- a/mapcat/database/pipeline_information.py +++ b/mapcat/database/pipeline_information.py @@ -31,7 +31,7 @@ class PipelineInformationTable(SQLModel, table=True): __tablename__ = "pipeline_information" pipeline_information_id: uuid.UUID = Field( - default_factory=uuid.uuid7, primary_key=True + default_factory=uuid.create, primary_key=True ) map_id: uuid.UUID = Field(foreign_key="depth_one_maps.map_id", nullable=False) map: DepthOneMapTable = Relationship(back_populates="pipeline_information") diff --git a/mapcat/database/pointing_residual.py b/mapcat/database/pointing_residual.py index 28ffd35..3da0127 100644 --- a/mapcat/database/pointing_residual.py +++ b/mapcat/database/pointing_residual.py @@ -33,7 +33,7 @@ class PointingResidualTable(SQLModel, table=True): __tablename__ = "depth_one_pointing_residuals" pointing_residual_id: uuid.UUID = Field( - default_factory=uuid.uuid7, primary_key=True + default_factory=uuid.create, primary_key=True ) map_id: uuid.UUID = Field( diff --git a/mapcat/database/time_domain_processing.py b/mapcat/database/time_domain_processing.py index 72d1e0f..62380b9 100644 --- a/mapcat/database/time_domain_processing.py +++ b/mapcat/database/time_domain_processing.py @@ -46,7 +46,7 @@ class TimeDomainProcessingTable(SQLModel, table=True): __tablename__ = "time_domain_processing" processing_status_id: uuid.UUID = Field( - prdefault_factory=uuid.uuid7, primary_key=True + prdefault_factory=uuid.create, primary_key=True ) map_id: uuid.UUID = Field( diff --git a/mapcat/database/tod.py b/mapcat/database/tod.py index 0df33fb..c01f01d 100644 --- a/mapcat/database/tod.py +++ b/mapcat/database/tod.py @@ -99,7 +99,7 @@ class TODDepthOneTable(SQLModel, table=True): """ __tablename__ = "tod_depth_one" - tod_id: uuid.UUID = Field(default_factory=uuid.uuid7, primary_key=True) + tod_id: uuid.UUID = Field(default_factory=uuid.create, primary_key=True) obs_id: str = Field(nullable=False) pwv: float | None = Field(index=True, nullable=True) ctime: datetime = Field(index=True, nullable=False) From cea86b1ba5e5a1a377120107b67eaf5bddacee33 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Thu, 25 Jun 2026 11:30:39 -0400 Subject: [PATCH 07/15] [no ci] fix: add time zones to act injest --- mapcat/database/time_domain_processing.py | 5 +++-- mapcat/toolkit/act.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/mapcat/database/time_domain_processing.py b/mapcat/database/time_domain_processing.py index 62380b9..e6b6378 100644 --- a/mapcat/database/time_domain_processing.py +++ b/mapcat/database/time_domain_processing.py @@ -4,11 +4,12 @@ from datetime import datetime -import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel +import uuid7 as uuid + from .depth_one_map import DepthOneMapTable @@ -46,7 +47,7 @@ class TimeDomainProcessingTable(SQLModel, table=True): __tablename__ = "time_domain_processing" processing_status_id: uuid.UUID = Field( - prdefault_factory=uuid.create, primary_key=True + default_factory=uuid.create, primary_key=True ) map_id: uuid.UUID = Field( diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index d774235..c351b4f 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -3,7 +3,7 @@ """ import argparse as ap -import datetime +from datetime import UTC, datetime from pathlib import Path import h5py @@ -71,7 +71,7 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT TODDepthOneTable( obs_id=obs_id, pwv=None, - ctime=datetime.fromtimestamp(float(obs_id[4:14])), + ctime=datetime.fromtimestamp(float(obs_id[4:14]), tz=UTC), telescope=telescope, tube_slot=file_info["tube_slot"], frequency=file_info["frequency"], @@ -88,9 +88,9 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT mean_time_path=filenames.get("time"), tube_slot=file_info["tube_slot"], frequency=file_info["frequency"], - ctime=datetime.fromtimestamp(file_info["ctime"]), - start_time=datetime.fromtimestamp(file_info["start_time"]), - stop_time=datetime.fromtimestamp(file_info["stop_time"]), + ctime=datetime.fromtimestamp(file_info["ctime"], tz=UTC), + start_time=datetime.fromtimestamp(file_info["start_time"], tz=UTC), + stop_time=datetime.fromtimestamp(file_info["stop_time"], tz=UTC), tods=tods, ) From 2dafdfc52c461e5a7e379ff687ef85013c0a6ee6 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Thu, 25 Jun 2026 11:31:13 -0400 Subject: [PATCH 08/15] [no ci] feat: first draft at an alembic migration for going to UUIDs --- ...87d072aac99_update_to_uuid_and_datetime.py | 243 ++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py diff --git a/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py b/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py new file mode 100644 index 0000000..8b7100d --- /dev/null +++ b/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py @@ -0,0 +1,243 @@ +"""update to uuid and datetime + +Revision ID: c87d072aac99 +Revises: 46575bc0d660 +Create Date: 2026-06-24 13:58:05.429510 + +""" + +from collections.abc import Sequence +from typing import Any + +import sqlalchemy as sa +import uuid7 as uuid +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "c87d072aac99" +down_revision: str | None = "46575bc0d660" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _table_exists(bind: Any, table_name: str) -> bool: + return bind.dialect.has_table(bind, table_name) + + +def _rename_tables(tables: Sequence[str]) -> None: + bind = op.get_bind() + for table_name in tables: + if _table_exists(bind, table_name): + op.rename_table(table_name, f"{table_name}_old") + + +def _copy_rows( + source_table: str, + target_table: str, + columns: Sequence[str], + *, + id_column: str | None = None, + id_map: dict[Any, Any] | None = None, + converters: dict[str, Any] | None = None, +) -> None: + bind = op.get_bind() + metadata = sa.MetaData() + source = sa.Table(source_table, metadata, autoload_with=bind) + target = sa.Table(target_table, metadata, autoload_with=bind) + + rows = bind.execute(sa.select(*[source.c[column] for column in columns])).fetchall() + + for row in rows: + values: dict[str, Any] = {} + for column in columns: + value = getattr(row, column) + if id_column is not None and column == id_column: + new_id = uuid.create() + if id_map is not None: + id_map[value] = new_id + values[column] = new_id + elif converters and column in converters: + values[column] = converters[column](value) + else: + values[column] = value + bind.execute(target.insert(), values) + + +def _update_links( + link_table_name: str, + link_column_name: str, + foreign_table_name: str, + foreign_key_name: str, + foreign_uuid_column_name: str, + tmp_column_name: str | None = None, +) -> None: + """ + Function for updating a link table to use a new UUID foreign key + from an old integer foreign key. Copilot assisted in writing this function: + I wrote pure SQL statements that achieved the desired result, as well as a + skeleton for _update_links, and Copilot translated the SQL statement into + an alembic migration using SQLAlchemy. + + Parameters + ---------- + link_table_name : str + Name of the link table to modify + link_column_name : str + Name of current foreign key column in link_table + foreign_table_name : str + Name of foreign table to link into + foreign_key_name : str + Name of current ID column (int) in foreign table that link_column_name foreign key into + foreign_uuid_column_name : str + Name of new ID column (UUID) in foreign table that we would like link_table to be foreign key into + + Returns + ------- + None + """ + + bind = op.get_bind() + metadata = sa.MetaData() + + tmp_column = f"{link_column_name}_tmp" + + # First make a temporary column of type UUID. This will be our foreign key column eventually + op.add_column( + table_name=link_table_name, + column=sa.Column(tmp_column, sa.Uuid(), nullable=True), + ) + + link_table = sa.Table(link_table_name, metadata, autoload_with=bind) + foreign_table = sa.Table(foreign_table_name, metadata, autoload_with=bind) + + # Update the link table temporary column we just made + # The value of this column with be the UUID column value in + # foreign table where the old index + stmt = sa.update(link_table).values( + **{ + tmp_column: ( + sa.select(foreign_table.c[foreign_uuid_column_name]) + .where( + foreign_table.c[foreign_key_name] == link_table.c[link_column_name] + ) + .scalar_subquery() + ) + } + ) + bind.execute(stmt) + + # We can now safely drop the original foreign key in link table as the link info is saved in tmp_column + op.drop_column(link_table_name, link_column_name) + + # Add a new, permanent link column, which we will copy values from tmp_column into. + # We intentionally leave the FK creation for later so it points at the final + # parent column name after the parent table is renamed. + # Note the foreign key constraint isn't actually made here yet since we + # will later rename the foreign key column in foreign_table from TABLE_id_uuid to TABLE_id. + op.add_column( + table_name=link_table_name, + column=sa.Column( + link_column_name, + sa.Uuid(), + nullable=False, + index=True, + ), + ) + + stmt = sa.update(link_table).values(**{link_column_name: link_table.c[tmp_column]}) + bind.execute(stmt) + + op.drop_column(link_table_name, tmp_column) + + +def _update_to_UUID_with_links( + table_name: str, old_key_name: str, link_table_list: list[tuple[str, str]] +) -> None: + """ + Function to update a table from integer ID to UUID. + This function will correcly update all link tables with + foreign keys into table_name to use the new UUID. + + Parameters + ---------- + table_name : str + Name of the table to update UUIDs. + old_key_name : str + Name of the column containing the old primary keys to be updated. + link_table_list : list[tuple[str, str]] + List of tuples where each entry is a pair of link table with a foreign key into table_name + to be updated and foreign key name in that link table. + + Returns + ------- + None + """ + bind = op.get_bind() + metadata = sa.MetaData() + + tmp_key_name = f"{old_key_name}_uuid" + # Add a new column which will hold our UUIDs + op.add_column( + table_name=table_name, + column=sa.Column(tmp_key_name, sa.Uuid(), nullable=True), + ) + + # Populate the UUID rows with uuids + parent_table = sa.Table(table_name, metadata, autoload_with=bind) + rows = bind.execute(sa.select(parent_table.c[old_key_name])).fetchall() + for (old_id,) in rows: + new_uuid = uuid.create() + bind.execute( + parent_table.update() + .where(parent_table.c[old_key_name] == old_id) + .values(tmp_key_name=new_uuid) + ) + + # Update the link tables + for link_table_name, link_column_name in link_table_list: + _update_links( + link_table_name=link_table_name, + link_column_name=link_column_name, + foreign_table_name=table_name, + foreign_key_name=old_key_name, + foreign_uuid_column_name=tmp_key_name, + ) + + # Now that we've done the sandbag swap we can drop the old ID + # column and alter the new column to have the correct name and be primary + op.drop_column(table_name=table_name, column_name=old_key_name) + + op.alter_column( + table_name=table_name, + column_name=tmp_key_name, + new_column_name=old_key_name, + ) + op.create_primary_key( + f"pk_{table_name}", + table_name, + [old_key_name], + ) + + # Recreate the foreign key constraints on the link tables + for link_table_name, link_column_name in link_table_list: + op.create_foreign_key( + f"fk_{link_table_name}_{link_column_name}", + link_table_name, + table_name, + [link_column_name], + [old_key_name], + ) + + +def upgrade() -> None: + + atomic_coadd_link_tables = [ + ("link_atomic_map_to_coadd", "coadd_id"), + ("link_coadd_map_to_coadd", "parent_coadd_id"), + ("link_coadd_map_to_coadd", "child_coadd_id"), + ] + _update_to_UUID_with_links( + table_name="atomic_map_coadds", + old_key_name="coadd_id", + link_table_list=atomic_coadd_link_tables, + ) From 8bc2afaca17c5a9949e5fbe50ac056ff1e891e04 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Thu, 25 Jun 2026 11:33:14 -0400 Subject: [PATCH 09/15] [no ci] fix: remove unused functions --- ...87d072aac99_update_to_uuid_and_datetime.py | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py b/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py index 8b7100d..17c4583 100644 --- a/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py +++ b/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py @@ -19,50 +19,6 @@ branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None - -def _table_exists(bind: Any, table_name: str) -> bool: - return bind.dialect.has_table(bind, table_name) - - -def _rename_tables(tables: Sequence[str]) -> None: - bind = op.get_bind() - for table_name in tables: - if _table_exists(bind, table_name): - op.rename_table(table_name, f"{table_name}_old") - - -def _copy_rows( - source_table: str, - target_table: str, - columns: Sequence[str], - *, - id_column: str | None = None, - id_map: dict[Any, Any] | None = None, - converters: dict[str, Any] | None = None, -) -> None: - bind = op.get_bind() - metadata = sa.MetaData() - source = sa.Table(source_table, metadata, autoload_with=bind) - target = sa.Table(target_table, metadata, autoload_with=bind) - - rows = bind.execute(sa.select(*[source.c[column] for column in columns])).fetchall() - - for row in rows: - values: dict[str, Any] = {} - for column in columns: - value = getattr(row, column) - if id_column is not None and column == id_column: - new_id = uuid.create() - if id_map is not None: - id_map[value] = new_id - values[column] = new_id - elif converters and column in converters: - values[column] = converters[column](value) - else: - values[column] = value - bind.execute(target.insert(), values) - - def _update_links( link_table_name: str, link_column_name: str, From fc5c9847d1144c1a4458ea97df7054c3fecf1464 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Thu, 25 Jun 2026 11:51:23 -0400 Subject: [PATCH 10/15] [no ci] feat: migrate all tables to UUID --- ...87d072aac99_update_to_uuid_and_datetime.py | 106 ++++++++++++++---- 1 file changed, 85 insertions(+), 21 deletions(-) diff --git a/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py b/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py index 17c4583..1aa5a1a 100644 --- a/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py +++ b/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py @@ -7,7 +7,6 @@ """ from collections.abc import Sequence -from typing import Any import sqlalchemy as sa import uuid7 as uuid @@ -19,6 +18,7 @@ branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None + def _update_links( link_table_name: str, link_column_name: str, @@ -107,7 +107,9 @@ def _update_links( def _update_to_UUID_with_links( - table_name: str, old_key_name: str, link_table_list: list[tuple[str, str]] + table_name: str, + old_key_name: str, + link_table_list: list[tuple[str, str]] | None = None, ) -> None: """ Function to update a table from integer ID to UUID. @@ -120,9 +122,10 @@ def _update_to_UUID_with_links( Name of the table to update UUIDs. old_key_name : str Name of the column containing the old primary keys to be updated. - link_table_list : list[tuple[str, str]] + link_table_list : list[tuple[str, str]] | None, default; None List of tuples where each entry is a pair of link table with a foreign key into table_name - to be updated and foreign key name in that link table. + to be updated and foreign key name in that link table. If none, table_name has no foreign keys + into it so we can just skip updating those links. Returns ------- @@ -150,14 +153,15 @@ def _update_to_UUID_with_links( ) # Update the link tables - for link_table_name, link_column_name in link_table_list: - _update_links( - link_table_name=link_table_name, - link_column_name=link_column_name, - foreign_table_name=table_name, - foreign_key_name=old_key_name, - foreign_uuid_column_name=tmp_key_name, - ) + if link_table_list: + for link_table_name, link_column_name in link_table_list: + _update_links( + link_table_name=link_table_name, + link_column_name=link_column_name, + foreign_table_name=table_name, + foreign_key_name=old_key_name, + foreign_uuid_column_name=tmp_key_name, + ) # Now that we've done the sandbag swap we can drop the old ID # column and alter the new column to have the correct name and be primary @@ -174,19 +178,25 @@ def _update_to_UUID_with_links( [old_key_name], ) - # Recreate the foreign key constraints on the link tables - for link_table_name, link_column_name in link_table_list: - op.create_foreign_key( - f"fk_{link_table_name}_{link_column_name}", - link_table_name, - table_name, - [link_column_name], - [old_key_name], - ) + if link_table_list: + # Recreate the foreign key constraints on the link tables + for link_table_name, link_column_name in link_table_list: + op.create_foreign_key( + f"fk_{link_table_name}_{link_column_name}", + link_table_name, + table_name, + [link_column_name], + [old_key_name], + ) + + +def _update_sky_coverage(): + pass def upgrade() -> None: + # TODO: check these for foreign keys not in links.py atomic_coadd_link_tables = [ ("link_atomic_map_to_coadd", "coadd_id"), ("link_coadd_map_to_coadd", "parent_coadd_id"), @@ -197,3 +207,57 @@ def upgrade() -> None: old_key_name="coadd_id", link_table_list=atomic_coadd_link_tables, ) + + atomic_map_link_tables = [("link_atomic_map_to_coadd", "atomic_map_id")] + _update_to_UUID_with_links( + table_name="atomic_maps", + old_key_name="atomic_map_id", + link_table_list=atomic_map_link_tables, + ) + + depth_one_coadd_link_tables = [("link_depth_one_map_to_coadd", "coadd_id")] + _update_to_UUID_with_links( + table_name="depth_one_coadds", + old_key_name="coadd_id", + link_table_list=depth_one_coadd_link_tables, + ) + + depth_one_link_tables = [ + ("link_depth_one_map_to_coadd", "map_id"), + ("link_tod_to_depth_one_map", "map_id"), + ] + _update_to_UUID_with_links( + table_name="depth_one_maps", + old_key_name="map_id", + link_table_list=depth_one_link_tables, + ) + + _update_to_UUID_with_links( + table_name="pipeline_information", + old_key_name="pipeline_information_id", + link_table_list=None, + ) + + _update_to_UUID_with_links( + table_name="depth_one_pointing_residuals", + old_key_name="pointing_residual_id", + link_table_list=None, + ) + + # TODO: write migration for sky coverage + _update_sky_coverage() + + _update_to_UUID_with_links( + table_name="time_domain_processing", + old_key_name="processing_status_id", + link_table_list=None, + ) + + tod_link_tables = [ + ("link_tod_to_depth_one_map", "tod_id"), + ] + _update_to_UUID_with_links( + table_name="tod_depth_one", + old_key_name="tod_id", + link_table_list=tod_link_tables, + ) From 2081366715291da0804d53de37bbb672fe4d3f13 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Mon, 27 Jul 2026 11:06:39 -0400 Subject: [PATCH 11/15] feat: remove uuids --- ...87d072aac99_update_to_uuid_and_datetime.py | 263 ------------------ mapcat/database/atomic_coadd.py | 5 +- mapcat/database/atomic_map.py | 5 +- mapcat/database/depth_one_coadd.py | 5 +- mapcat/database/depth_one_map.py | 7 +- mapcat/database/links.py | 17 +- mapcat/database/pipeline_information.py | 9 +- mapcat/database/pointing_residual.py | 9 +- mapcat/database/sky_coverage.py | 5 +- mapcat/database/time_domain_processing.py | 16 +- mapcat/database/tod.py | 7 +- 11 files changed, 34 insertions(+), 314 deletions(-) delete mode 100644 mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py diff --git a/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py b/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py deleted file mode 100644 index 1aa5a1a..0000000 --- a/mapcat/alembic/versions/c87d072aac99_update_to_uuid_and_datetime.py +++ /dev/null @@ -1,263 +0,0 @@ -"""update to uuid and datetime - -Revision ID: c87d072aac99 -Revises: 46575bc0d660 -Create Date: 2026-06-24 13:58:05.429510 - -""" - -from collections.abc import Sequence - -import sqlalchemy as sa -import uuid7 as uuid -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "c87d072aac99" -down_revision: str | None = "46575bc0d660" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def _update_links( - link_table_name: str, - link_column_name: str, - foreign_table_name: str, - foreign_key_name: str, - foreign_uuid_column_name: str, - tmp_column_name: str | None = None, -) -> None: - """ - Function for updating a link table to use a new UUID foreign key - from an old integer foreign key. Copilot assisted in writing this function: - I wrote pure SQL statements that achieved the desired result, as well as a - skeleton for _update_links, and Copilot translated the SQL statement into - an alembic migration using SQLAlchemy. - - Parameters - ---------- - link_table_name : str - Name of the link table to modify - link_column_name : str - Name of current foreign key column in link_table - foreign_table_name : str - Name of foreign table to link into - foreign_key_name : str - Name of current ID column (int) in foreign table that link_column_name foreign key into - foreign_uuid_column_name : str - Name of new ID column (UUID) in foreign table that we would like link_table to be foreign key into - - Returns - ------- - None - """ - - bind = op.get_bind() - metadata = sa.MetaData() - - tmp_column = f"{link_column_name}_tmp" - - # First make a temporary column of type UUID. This will be our foreign key column eventually - op.add_column( - table_name=link_table_name, - column=sa.Column(tmp_column, sa.Uuid(), nullable=True), - ) - - link_table = sa.Table(link_table_name, metadata, autoload_with=bind) - foreign_table = sa.Table(foreign_table_name, metadata, autoload_with=bind) - - # Update the link table temporary column we just made - # The value of this column with be the UUID column value in - # foreign table where the old index - stmt = sa.update(link_table).values( - **{ - tmp_column: ( - sa.select(foreign_table.c[foreign_uuid_column_name]) - .where( - foreign_table.c[foreign_key_name] == link_table.c[link_column_name] - ) - .scalar_subquery() - ) - } - ) - bind.execute(stmt) - - # We can now safely drop the original foreign key in link table as the link info is saved in tmp_column - op.drop_column(link_table_name, link_column_name) - - # Add a new, permanent link column, which we will copy values from tmp_column into. - # We intentionally leave the FK creation for later so it points at the final - # parent column name after the parent table is renamed. - # Note the foreign key constraint isn't actually made here yet since we - # will later rename the foreign key column in foreign_table from TABLE_id_uuid to TABLE_id. - op.add_column( - table_name=link_table_name, - column=sa.Column( - link_column_name, - sa.Uuid(), - nullable=False, - index=True, - ), - ) - - stmt = sa.update(link_table).values(**{link_column_name: link_table.c[tmp_column]}) - bind.execute(stmt) - - op.drop_column(link_table_name, tmp_column) - - -def _update_to_UUID_with_links( - table_name: str, - old_key_name: str, - link_table_list: list[tuple[str, str]] | None = None, -) -> None: - """ - Function to update a table from integer ID to UUID. - This function will correcly update all link tables with - foreign keys into table_name to use the new UUID. - - Parameters - ---------- - table_name : str - Name of the table to update UUIDs. - old_key_name : str - Name of the column containing the old primary keys to be updated. - link_table_list : list[tuple[str, str]] | None, default; None - List of tuples where each entry is a pair of link table with a foreign key into table_name - to be updated and foreign key name in that link table. If none, table_name has no foreign keys - into it so we can just skip updating those links. - - Returns - ------- - None - """ - bind = op.get_bind() - metadata = sa.MetaData() - - tmp_key_name = f"{old_key_name}_uuid" - # Add a new column which will hold our UUIDs - op.add_column( - table_name=table_name, - column=sa.Column(tmp_key_name, sa.Uuid(), nullable=True), - ) - - # Populate the UUID rows with uuids - parent_table = sa.Table(table_name, metadata, autoload_with=bind) - rows = bind.execute(sa.select(parent_table.c[old_key_name])).fetchall() - for (old_id,) in rows: - new_uuid = uuid.create() - bind.execute( - parent_table.update() - .where(parent_table.c[old_key_name] == old_id) - .values(tmp_key_name=new_uuid) - ) - - # Update the link tables - if link_table_list: - for link_table_name, link_column_name in link_table_list: - _update_links( - link_table_name=link_table_name, - link_column_name=link_column_name, - foreign_table_name=table_name, - foreign_key_name=old_key_name, - foreign_uuid_column_name=tmp_key_name, - ) - - # Now that we've done the sandbag swap we can drop the old ID - # column and alter the new column to have the correct name and be primary - op.drop_column(table_name=table_name, column_name=old_key_name) - - op.alter_column( - table_name=table_name, - column_name=tmp_key_name, - new_column_name=old_key_name, - ) - op.create_primary_key( - f"pk_{table_name}", - table_name, - [old_key_name], - ) - - if link_table_list: - # Recreate the foreign key constraints on the link tables - for link_table_name, link_column_name in link_table_list: - op.create_foreign_key( - f"fk_{link_table_name}_{link_column_name}", - link_table_name, - table_name, - [link_column_name], - [old_key_name], - ) - - -def _update_sky_coverage(): - pass - - -def upgrade() -> None: - - # TODO: check these for foreign keys not in links.py - atomic_coadd_link_tables = [ - ("link_atomic_map_to_coadd", "coadd_id"), - ("link_coadd_map_to_coadd", "parent_coadd_id"), - ("link_coadd_map_to_coadd", "child_coadd_id"), - ] - _update_to_UUID_with_links( - table_name="atomic_map_coadds", - old_key_name="coadd_id", - link_table_list=atomic_coadd_link_tables, - ) - - atomic_map_link_tables = [("link_atomic_map_to_coadd", "atomic_map_id")] - _update_to_UUID_with_links( - table_name="atomic_maps", - old_key_name="atomic_map_id", - link_table_list=atomic_map_link_tables, - ) - - depth_one_coadd_link_tables = [("link_depth_one_map_to_coadd", "coadd_id")] - _update_to_UUID_with_links( - table_name="depth_one_coadds", - old_key_name="coadd_id", - link_table_list=depth_one_coadd_link_tables, - ) - - depth_one_link_tables = [ - ("link_depth_one_map_to_coadd", "map_id"), - ("link_tod_to_depth_one_map", "map_id"), - ] - _update_to_UUID_with_links( - table_name="depth_one_maps", - old_key_name="map_id", - link_table_list=depth_one_link_tables, - ) - - _update_to_UUID_with_links( - table_name="pipeline_information", - old_key_name="pipeline_information_id", - link_table_list=None, - ) - - _update_to_UUID_with_links( - table_name="depth_one_pointing_residuals", - old_key_name="pointing_residual_id", - link_table_list=None, - ) - - # TODO: write migration for sky coverage - _update_sky_coverage() - - _update_to_UUID_with_links( - table_name="time_domain_processing", - old_key_name="processing_status_id", - link_table_list=None, - ) - - tod_link_tables = [ - ("link_tod_to_depth_one_map", "tod_id"), - ] - _update_to_UUID_with_links( - table_name="tod_depth_one", - old_key_name="tod_id", - link_table_list=tod_link_tables, - ) diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index 6f79fa5..e6caaac 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -5,7 +5,6 @@ from datetime import datetime from typing import TYPE_CHECKING -import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel @@ -17,7 +16,7 @@ class AtomicMapCoadd(SQLModel): - coadd_id: uuid.UUID + coadd_id: int coadd_name: str prefix_path: str @@ -34,7 +33,7 @@ class AtomicMapCoadd(SQLModel): class AtomicMapCoaddTable(SQLModel, table=True): __tablename__ = "atomic_map_coadds" - coadd_id: uuid.UUID = Field(default_factory=uuid.create, primary_key=True) + coadd_id: int = Field(primary_key=True) coadd_name: str = Field() prefix_path: str = Field() diff --git a/mapcat/database/atomic_map.py b/mapcat/database/atomic_map.py index ab39043..56088d5 100644 --- a/mapcat/database/atomic_map.py +++ b/mapcat/database/atomic_map.py @@ -5,7 +5,6 @@ from datetime import datetime from typing import TYPE_CHECKING -import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel @@ -17,7 +16,7 @@ class AtomicMap(SQLModel): - atomic_map_id: uuid.UUID + atomic_map_id: int obs_id: str telescope: str @@ -61,7 +60,7 @@ class AtomicMap(SQLModel): class AtomicMapTable(SQLModel, table=True): __tablename__ = "atomic_maps" - atomic_map_id: uuid.UUID = Field(default_factory=uuid.create, primary_key=True) + atomic_map_id: int = Field(primary_key=True) obs_id: str = Field() telescope: str = Field() diff --git a/mapcat/database/depth_one_coadd.py b/mapcat/database/depth_one_coadd.py index 9645e85..a2e0702 100644 --- a/mapcat/database/depth_one_coadd.py +++ b/mapcat/database/depth_one_coadd.py @@ -4,7 +4,6 @@ from datetime import datetime -import uuid7 as uuid from astropy.time import Time from sqlmodel import Field, Relationship, SQLModel @@ -13,7 +12,7 @@ class DepthOneCoadd(SQLModel): - coadd_id: uuid.UUID + coadd_id: int coadd_name: str coadd_type: str @@ -40,7 +39,7 @@ class DepthOneCoaddTable(SQLModel, table=True): __tablename__ = "depth_one_coadds" - coadd_id: int = Field(default_factory=uuid.create, primary_key=True) + coadd_id: int = Field(primary_key=True) coadd_name: str = Field(nullable=False) coadd_type: str = Field(nullable=False) diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py index feded42..cc338f0 100644 --- a/mapcat/database/depth_one_map.py +++ b/mapcat/database/depth_one_map.py @@ -5,7 +5,6 @@ from datetime import datetime from typing import TYPE_CHECKING, Any -import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import JSON, Field, Relationship, SQLModel @@ -22,7 +21,7 @@ class DepthOneMap(SQLModel): - map_id: uuid.UUID + map_id: int map_name: str map_path: str | None @@ -49,7 +48,7 @@ class DepthOneMapTable(SQLModel, table=True): Attributes ---------- - id : uuid.UUID + id : int Unique map identifiers. Internal to SO map_name : str Name of depth 1 map @@ -103,7 +102,7 @@ class DepthOneMapTable(SQLModel, table=True): __tablename__ = "depth_one_maps" - map_id: uuid.UUID = Field(default_factory=uuid.create, primary_key=True) + map_id: int = Field(primary_key=True) map_name: str = Field(index=True, unique=True, nullable=False) map_path: str | None = None diff --git a/mapcat/database/links.py b/mapcat/database/links.py index 97593d4..1db33b2 100644 --- a/mapcat/database/links.py +++ b/mapcat/database/links.py @@ -2,7 +2,6 @@ Link tables. """ -import uuid7 as uuid from sqlmodel import Field, SQLModel @@ -13,14 +12,14 @@ class DepthOneToCoaddTable(SQLModel, table=True): __tablename__ = "link_depth_one_map_to_coadd" - map_id: uuid.UUID = Field( + map_id: int = Field( foreign_key="depth_one_maps.map_id", primary_key=True, nullable=False, index=True, ondelete="CASCADE", ) - coadd_id: uuid.UUID = Field( + coadd_id: int = Field( foreign_key="depth_one_coadds.coadd_id", primary_key=True, nullable=False, @@ -36,14 +35,14 @@ class TODToMapTable(SQLModel, table=True): __tablename__ = "link_tod_to_depth_one_map" - tod_id: uuid.UUID = Field( + tod_id: int = Field( foreign_key="tod_depth_one.tod_id", primary_key=True, nullable=False, index=True, ondelete="CASCADE", ) - map_id: uuid.UUID = Field( + map_id: int = Field( foreign_key="depth_one_maps.map_id", primary_key=True, nullable=False, @@ -59,7 +58,7 @@ class AtomicMapToCoaddTable(SQLModel, table=True): __tablename__ = "link_atomic_map_to_coadd" - atomic_map_id: uuid.UUID = Field( + atomic_map_id: int = Field( foreign_key="atomic_maps.atomic_map_id", primary_key=True, nullable=False, @@ -67,7 +66,7 @@ class AtomicMapToCoaddTable(SQLModel, table=True): ondelete="CASCADE", ) - coadd_id: uuid.UUID = Field( + coadd_id: int = Field( foreign_key="atomic_map_coadds.coadd_id", primary_key=True, nullable=False, @@ -83,7 +82,7 @@ class CoaddMapToCoaddTable(SQLModel, table=True): __tablename__ = "link_coadd_map_to_coadd" - parent_coadd_id: uuid.UUID = Field( + parent_coadd_id: int = Field( foreign_key="atomic_map_coadds.coadd_id", primary_key=True, nullable=False, @@ -91,7 +90,7 @@ class CoaddMapToCoaddTable(SQLModel, table=True): ondelete="CASCADE", ) - child_coadd_id: uuid.UUID = Field( + child_coadd_id: int = Field( foreign_key="atomic_map_coadds.coadd_id", primary_key=True, nullable=False, diff --git a/mapcat/database/pipeline_information.py b/mapcat/database/pipeline_information.py index 3c04d37..8ca6ac2 100644 --- a/mapcat/database/pipeline_information.py +++ b/mapcat/database/pipeline_information.py @@ -4,7 +4,6 @@ from typing import Any -import uuid7 as uuid from sqlmodel import JSON, Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable @@ -16,7 +15,7 @@ class PipelineInformationTable(SQLModel, table=True): Attributes ---------- - id : uuid.UUID + id : int Internal ID of the pipeline info map_name : str Name of depth 1 map being tracked. Foreign into DepthOneMap @@ -30,10 +29,8 @@ class PipelineInformationTable(SQLModel, table=True): __tablename__ = "pipeline_information" - pipeline_information_id: uuid.UUID = Field( - default_factory=uuid.create, primary_key=True - ) - map_id: uuid.UUID = Field(foreign_key="depth_one_maps.map_id", nullable=False) + pipeline_information_id: int = Field(primary_key=True) + map_id: int = Field(foreign_key="depth_one_maps.map_id", nullable=False) map: DepthOneMapTable = Relationship(back_populates="pipeline_information") sotodlib_version: str diff --git a/mapcat/database/pointing_residual.py b/mapcat/database/pointing_residual.py index 3da0127..7f9a14e 100644 --- a/mapcat/database/pointing_residual.py +++ b/mapcat/database/pointing_residual.py @@ -2,7 +2,6 @@ Table containing pointing residuals. """ -import uuid7 as uuid from sqlmodel import Field, Relationship, SQLModel from mapcat.pointing.base import PointingModelStats @@ -23,7 +22,7 @@ class PointingResidualTable(SQLModel, table=True): Attributes ---------- - map_id : uuid.UUID + map_id : int Internal ID of the depth one map residual_model: ConstantPointingModel | PolynomialPointingModel The pointing model to actually store in the database. @@ -32,11 +31,9 @@ class PointingResidualTable(SQLModel, table=True): """ __tablename__ = "depth_one_pointing_residuals" - pointing_residual_id: uuid.UUID = Field( - default_factory=uuid.create, primary_key=True - ) + pointing_residual_id: int = Field(primary_key=True) - map_id: uuid.UUID = Field( + map_id: int = Field( index=True, nullable=False, foreign_key="depth_one_maps.map_id", diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index cb6ecd0..fa5021f 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -2,7 +2,6 @@ Sky coverage table. """ -import uuid7 as uuid from sqlalchemy import PrimaryKeyConstraint from sqlmodel import Field, Relationship, SQLModel @@ -20,7 +19,7 @@ class SkyCoverageTable(SQLModel, table=True): Composite ID from map_id, x, and y map : DepthOneMapTable Depth 1 map being tracked. Foreign into DepthOneMap - map_id : uuid.UUID + map_id : int ID of depth 1 map being tracked x : int x-index of coverage patch. x=0 runs from RA 0 to 10,, etc. @@ -33,7 +32,7 @@ class SkyCoverageTable(SQLModel, table=True): x: int = Field(index=True, primary_key=True) y: int = Field(index=True, primary_key=True) - map_id: uuid.UUID = Field( + map_id: int = Field( foreign_key="depth_one_maps.map_id", nullable=False, ondelete="CASCADE", diff --git a/mapcat/database/time_domain_processing.py b/mapcat/database/time_domain_processing.py index e6b6378..e21e62a 100644 --- a/mapcat/database/time_domain_processing.py +++ b/mapcat/database/time_domain_processing.py @@ -8,15 +8,13 @@ from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel -import uuid7 as uuid - from .depth_one_map import DepthOneMapTable class TimeDomainProcessing(SQLModel): - processing_status_id: uuid.UUID + processing_status_id: int - map_id: uuid.UUID + map_id: int processing_start: AstroPydanticTime | None processing_end: AstroPydanticTime | None @@ -32,9 +30,9 @@ class TimeDomainProcessingTable(SQLModel, table=True): Attributes ---------- - processing_status_id : uuid.UUID + processing_status_id : int Internal ID of the processing status - map_name : uuid.UUID + map_name : int Name of depth 1 map being tracked. Foreign into DepthOneMap processing_start : datetime | None Time processing started. None if not started. @@ -46,11 +44,9 @@ class TimeDomainProcessingTable(SQLModel, table=True): __tablename__ = "time_domain_processing" - processing_status_id: uuid.UUID = Field( - default_factory=uuid.create, primary_key=True - ) + processing_status_id: int = Field(primary_key=True) - map_id: uuid.UUID = Field( + map_id: int = Field( index=True, nullable=False, foreign_key="depth_one_maps.map_id", diff --git a/mapcat/database/tod.py b/mapcat/database/tod.py index c01f01d..882f5db 100644 --- a/mapcat/database/tod.py +++ b/mapcat/database/tod.py @@ -4,7 +4,6 @@ from datetime import datetime -import uuid7 as uuid from astropy.time import Time from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel @@ -14,7 +13,7 @@ class TODDepthOne(SQLModel): - tod_id: uuid.UUID + tod_id: int obs_id: str pwv: float | None ctime: AstroPydanticTime @@ -46,7 +45,7 @@ class TODDepthOneTable(SQLModel, table=True): Attributes ---------- - tod_id : uuid.UUID + tod_id : int Unique TOD identifier. Internal to SO map_name : str Name of map this TOD went into. Foreign key @@ -99,7 +98,7 @@ class TODDepthOneTable(SQLModel, table=True): """ __tablename__ = "tod_depth_one" - tod_id: uuid.UUID = Field(default_factory=uuid.create, primary_key=True) + tod_id: int = Field(primary_key=True) obs_id: str = Field(nullable=False) pwv: float | None = Field(index=True, nullable=True) ctime: datetime = Field(index=True, nullable=False) From 0b1f51881165f14f0b171b855c9d0fa7bbfbfc6a Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Mon, 27 Jul 2026 16:14:27 -0400 Subject: [PATCH 12/15] feat: add alembic migration for update to datetime and update tests --- .../0762b3c7694d_swapping_to_datetime.py | 165 ++++++++++++++++++ mapcat/database/atomic_coadd.py | 4 +- mapcat/database/atomic_map.py | 2 +- mapcat/toolkit/reset.py | 17 +- tests/test_act.py | 13 +- tests/test_mapcat.py | 48 ++++- tests/test_reset.py | 2 +- 7 files changed, 232 insertions(+), 19 deletions(-) create mode 100644 mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py diff --git a/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py b/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py new file mode 100644 index 0000000..c8c07cb --- /dev/null +++ b/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py @@ -0,0 +1,165 @@ +"""Swapping to datetime + +Revision ID: 0762b3c7694d +Revises: 46575bc0d660 +Create Date: 2026-07-27 11:19:49.803412 + +""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import datetime, timezone + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0762b3c7694d" +down_revision: str | None = "46575bc0d660" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +TABLE_DEFINITIONS = { + "atomic_maps": { + "primary_key_name": "atomic_map_id", + "column_names": [("ctime", False)], + }, + "atomic_map_coadds": { + "primary_key_name": "coadd_id", + "column_names": [("start_time", False), ("stop_time", False)], + }, + "depth_one_maps": { + "primary_key_name": "map_id", + "column_names": [("ctime", True), ("start_time", True), ("stop_time", True)], + }, + "depth_one_coadds": { + "primary_key_name": "coadd_id", + "column_names": [("ctime", False), ("start_time", False), ("stop_time", False)], + }, +} + + +def unix_to_datetime( + table_name: str, + primary_key_name: str, + column_names: list[tuple[str, bool]], +) -> None: + """ + Convert a column from unix time to datetime. + + Parameters + ---------- + table_name : str + The name of the table to modify. + primary_key_name : str + The name of the primary key column for the table. + column_names : list[tuple[str, bool]] + The names of the columns to convert and boolean defining whether the column is indexed. + """ + bind = op.get_bind() + metadata = sa.MetaData() + cur_table = sa.Table(table_name, metadata, autoload_with=bind) + for column_name, is_indexed in column_names: + temp_col_name = f"temp_datetime_{column_name}" + if is_indexed: + op.drop_index(f"ix_{table_name}_{column_name}", table_name=table_name) + + op.add_column( + table_name, sa.Column(temp_col_name, sa.DateTime(), nullable=True) + ) + + stmt = sa.select(cur_table.c[column_name], cur_table.c[primary_key_name]) + results = bind.execute(stmt).fetchall() + for row in results: + unix_time = row[column_name] + primary_key_value = row[primary_key_name] + datetime_value = datetime.fromtimestamp(int(unix_time), tz=timezone.utc) + update_stmt = ( + cur_table.update() + .where(cur_table.c[primary_key_name] == primary_key_value) + .values({temp_col_name: datetime_value}) + ) + bind.execute(update_stmt) + + with op.batch_alter_table(table_name) as batch_op: + batch_op.drop_column(column_name) + batch_op.alter_column( + temp_col_name, + new_column_name=column_name, + nullable=False, + ) + + if is_indexed: + op.create_index(f"ix_{table_name}_{column_name}", table_name, [column_name]) + + +def _datetime_to_unix_value(datetime_value: datetime) -> int | None: + datetime_value = datetime_value.astimezone(timezone.utc) + return int(datetime_value.timestamp()) + + +def datetime_to_unix( + table_name: str, + primary_key_name: str, + column_names: list[tuple[str, bool]], +) -> None: + """ + Convert a column from datetime to unix time. + + Parameters + ---------- + table_name : str + The name of the table to modify. + primary_key_name : str + The name of the primary key column for the table. + column_names : list[tuple[str, bool]] + The names of the columns to convert and a boolean defining whether the column is indexed. + """ + bind = op.get_bind() + metadata = sa.MetaData() + cur_table = sa.Table(table_name, metadata, autoload_with=bind) + for column_name, is_indexed in column_names: + temp_col_name = f"temp_unix_{column_name}" + if is_indexed: + op.drop_index(f"ix_{table_name}_{column_name}", table_name=table_name) + + op.add_column(table_name, sa.Column(temp_col_name, sa.String(), nullable=True)) + + stmt = sa.select(cur_table.c[column_name], cur_table.c[primary_key_name]) + results = bind.execute(stmt).fetchall() + for row in results: + datetime_value = row[column_name] + primary_key_value = row[primary_key_name] + unix_time = _datetime_to_unix_value(datetime_value) + update_stmt = ( + cur_table.update() + .where(cur_table.c[primary_key_name] == primary_key_value) + .values({temp_col_name: unix_time}) + ) + bind.execute(update_stmt) + + with op.batch_alter_table(table_name) as batch_op: + batch_op.drop_column(column_name) + batch_op.alter_column( + temp_col_name, + new_column_name=column_name, + nullable=False, + ) + + if is_indexed: + op.create_index(f"ix_{table_name}_{column_name}", table_name, [column_name]) + + +def upgrade() -> None: + for table_name, table_data in TABLE_DEFINITIONS.items(): + unix_to_datetime( + table_name, table_data["primary_key_name"], table_data["column_names"] + ) + + +def downgrade() -> None: + for table_name, table_data in TABLE_DEFINITIONS.items(): + datetime_to_unix( + table_name, table_data["primary_key_name"], table_data["column_names"] + ) diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index e6caaac..4dd4fe7 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -40,8 +40,8 @@ class AtomicMapCoaddTable(SQLModel, table=True): platform: str = Field() interval: str = Field() - start_time: datetime = Field() - stop_time: datetime = Field() + start_time: datetime = Field(nullable=False) + stop_time: datetime = Field(nullable=False) freq_channel: str = Field() geom_file_path: str = Field() split_label: str = Field() diff --git a/mapcat/database/atomic_map.py b/mapcat/database/atomic_map.py index 56088d5..0b6e266 100644 --- a/mapcat/database/atomic_map.py +++ b/mapcat/database/atomic_map.py @@ -66,7 +66,7 @@ class AtomicMapTable(SQLModel, table=True): telescope: str = Field() freq_channel: str = Field() wafer: str = Field() - ctime: datetime = Field() + ctime: datetime = Field(nullable=False) split_label: str = Field() map_path: str | None = Field() diff --git a/mapcat/toolkit/reset.py b/mapcat/toolkit/reset.py index 7187161..a83dd65 100644 --- a/mapcat/toolkit/reset.py +++ b/mapcat/toolkit/reset.py @@ -3,6 +3,7 @@ """ import argparse as ap +from datetime import UTC, datetime from sqlalchemy import select from sqlalchemy.orm import sessionmaker @@ -72,9 +73,15 @@ def core(session: sessionmaker, args: ap.Namespace): TimeDomainProcessingTable.map_id == DepthOneMapTable.map_id, ) if args.start_time is not None: - stmt = stmt.where(DepthOneMapTable.ctime >= args.start_time) + stmt = stmt.where( + DepthOneMapTable.ctime + >= datetime.fromtimestamp(args.start_time, tz=UTC) + ) if args.end_time is not None: - stmt = stmt.where(DepthOneMapTable.ctime <= args.end_time) + stmt = stmt.where( + DepthOneMapTable.ctime + <= datetime.fromtimestamp(args.end_time, tz=UTC) + ) if args.from_status is not None: stmt = stmt.where( @@ -103,11 +110,13 @@ def core(session: sessionmaker, args: ap.Namespace): ) if args.start_time is not None: pr_stmt = pr_stmt.where( - DepthOneMapTable.ctime.timestamp() >= args.start_time + DepthOneMapTable.ctime + >= datetime.fromtimestamp(args.start_time, tz=UTC) ) if args.end_time is not None: pr_stmt = pr_stmt.where( - DepthOneMapTable.ctime.timestamp() <= args.end_time + DepthOneMapTable.ctime + <= datetime.fromtimestamp(args.end_time, tz=UTC) ) pointing_residuals = cur_session.execute(pr_stmt).scalars().all() diff --git a/tests/test_act.py b/tests/test_act.py index cc0af4d..574833e 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -1,5 +1,6 @@ import argparse as ap import os +from datetime import timezone from pathlib import Path import astropy.units as u @@ -218,7 +219,11 @@ def test_act(database_sessionmaker, downloaded_data_file): for map in maps: assert map.tube_slot in ["pa4", "pa6"] assert map.frequency == "f150" - assert int(map.ctime.timestamp()) in [1505603190, 1505646390] + ctime = map.ctime + ctime = ( + ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime + ) + assert int(ctime.timestamp()) in [1505603190, 1505646390] # Clean up, otherewise we interfere with test_sky_coverage session.delete(map) @@ -239,6 +244,10 @@ def test_sky_coverage(database_sessionmaker, downloaded_data_file): with database_sessionmaker() as session: d1maps = session.query(DepthOneMapTable).all() for d1map in d1maps: + ctime = d1map.ctime + ctime = ( + ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime + ) assert len(d1map.depth_one_sky_coverage) > 0 for cov in d1map.depth_one_sky_coverage: # Shitty test to make sure the coverage tiles are correct, by checking against the known coverage for these two maps. @@ -248,7 +257,7 @@ def test_sky_coverage(database_sessionmaker, downloaded_data_file): cov.x ), # These should be ints, idk why I have to cast them (from str) int(cov.y), - ) in cov_mapping[str(d1map.ctime.timestamp())] + ) in cov_mapping[str(ctime.timestamp())] with database_sessionmaker() as session: maps = session.query(DepthOneMapTable).all() diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index fec2965..af4f0df 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -92,7 +92,9 @@ def test_create_depth_one(database_sessionmaker): assert dmap.map_path == "/PATH/TO/DEPTH/ONE" assert dmap.tube_slot == "OTi1" assert dmap.frequency == "f090" - assert dmap.ctime.unix == 1755787524.0 + ctime = dmap.ctime + ctime = ctime.replace(tzinfo=UTC) if ctime.tzinfo is None else ctime + assert int(ctime.timestamp()) == 1755787524.0 # Make child tables with database_sessionmaker() as session: @@ -171,8 +173,20 @@ def test_create_depth_one(database_sessionmaker): assert proc.processing_status_id == proc_id assert proc.map_id == map_id - assert proc.processing_start.unix == 1756787524.0 - assert proc.processing_end.unix == 1756797524.0 + processing_start = proc.processing_start + processing_start = ( + processing_start.replace(tzinfo=UTC) + if processing_start.tzinfo is None + else processing_start + ) + processing_end = proc.processing_end + processing_end = ( + processing_end.replace(tzinfo=UTC) + if processing_end.tzinfo is None + else processing_end + ) + assert int(processing_start.timestamp()) == 1756787524.0 + assert int(processing_end.timestamp()) == 1756797524.0 assert proc.processing_status == "done" assert point.pointing_residual_id == point_id @@ -183,9 +197,17 @@ def test_create_depth_one(database_sessionmaker): assert tod.tod_id == tod_id assert tod.pwv == 0.7 assert tod.obs_id == "obs_1753486724_lati6_111" - assert tod.ctime.unix == 1755787524.0 - assert tod.start_time.unix == 1755687524.0 - assert tod.stop_time.unix == 1755887524.0 + ctime = tod.ctime + ctime = ctime.replace(tzinfo=UTC) if ctime.tzinfo is None else ctime + start_time = tod.start_time + start_time = ( + start_time.replace(tzinfo=UTC) if start_time.tzinfo is None else start_time + ) + stop_time = tod.stop_time + stop_time = stop_time.replace(tzinfo=UTC) if stop_time.tzinfo is None else stop_time + assert int(ctime.timestamp()) == 1755787524.0 + assert int(start_time.timestamp()) == 1755687524.0 + assert int(stop_time.timestamp()) == 1755887524.0 assert tod.nsamples == 28562 assert tod.telescope == "lat" assert tod.telescope_flavor == "lat" @@ -369,8 +391,14 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert cmap.prefix_path == "/PATH/TO/DAILY/COADD" assert cmap.platform == "satp3" assert cmap.interval == "daily" - assert cmap.start_time.unix == 1755604800.0 - assert cmap.stop_time.unix == 1755691200.0 + start_time = cmap.start_time + start_time = ( + start_time.replace(tzinfo=UTC) if start_time.tzinfo is None else start_time + ) + stop_time = cmap.stop_time + stop_time = stop_time.replace(tzinfo=UTC) if stop_time.tzinfo is None else stop_time + assert int(start_time.timestamp()) == 1755604800.0 + assert int(stop_time.timestamp()) == 1755691200.0 assert cmap.freq_channel == "f090" assert cmap.geom_file_path == "/PATH/TO/GEOM/FILE" assert cmap.split_label == "full" @@ -433,7 +461,9 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert atomic.telescope == "satp3" assert atomic.freq_channel == "f090" assert atomic.wafer == "ws0" - assert atomic.ctime.unix == 1755643932 + ctime = atomic.ctime + ctime = ctime.replace(tzinfo=UTC) if ctime.tzinfo is None else ctime + assert int(ctime.timestamp()) == 1755643932.0 assert atomic.split_label == "full" assert atomic.map_path is None assert atomic.ivar_path is None diff --git a/tests/test_reset.py b/tests/test_reset.py index 30d2a46..739ec2d 100644 --- a/tests/test_reset.py +++ b/tests/test_reset.py @@ -54,7 +54,7 @@ def _make_map(session, name, ctime, start_time=None, stop_time=None): map_path=f"/path/{name}_map.fits", tube_slot="OTi1", frequency="f090", - ctime=ctime, + ctime=datetime.fromtimestamp(ctime, tz=UTC), start_time=datetime.fromtimestamp(start_time, tz=UTC), stop_time=datetime.fromtimestamp(stop_time, tz=UTC), ) From 10f6f48e36b522ee08eee1fe0f2d8dfe1ae94f7c Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Mon, 27 Jul 2026 16:22:55 -0400 Subject: [PATCH 13/15] fix: update UTC to timezone.utc as UTC is unsupported before python3.11 --- mapcat/toolkit/act.py | 10 ++--- mapcat/toolkit/reset.py | 10 ++--- tests/test_mapcat.py | 86 +++++++++++++++++++++++------------------ tests/test_mapmaking.py | 38 +++++++++--------- tests/test_pointing.py | 8 ++-- 5 files changed, 82 insertions(+), 70 deletions(-) diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index c351b4f..0244753 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -3,7 +3,7 @@ """ import argparse as ap -from datetime import UTC, datetime +from datetime import datetime, timezone from pathlib import Path import h5py @@ -71,7 +71,7 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT TODDepthOneTable( obs_id=obs_id, pwv=None, - ctime=datetime.fromtimestamp(float(obs_id[4:14]), tz=UTC), + ctime=datetime.fromtimestamp(float(obs_id[4:14]), tz=timezone.utc), telescope=telescope, tube_slot=file_info["tube_slot"], frequency=file_info["frequency"], @@ -88,9 +88,9 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT mean_time_path=filenames.get("time"), tube_slot=file_info["tube_slot"], frequency=file_info["frequency"], - ctime=datetime.fromtimestamp(file_info["ctime"], tz=UTC), - start_time=datetime.fromtimestamp(file_info["start_time"], tz=UTC), - stop_time=datetime.fromtimestamp(file_info["stop_time"], tz=UTC), + ctime=datetime.fromtimestamp(file_info["ctime"], tz=timezone.utc), + start_time=datetime.fromtimestamp(file_info["start_time"], tz=timezone.utc), + stop_time=datetime.fromtimestamp(file_info["stop_time"], tz=timezone.utc), tods=tods, ) diff --git a/mapcat/toolkit/reset.py b/mapcat/toolkit/reset.py index a83dd65..d7c61cc 100644 --- a/mapcat/toolkit/reset.py +++ b/mapcat/toolkit/reset.py @@ -3,7 +3,7 @@ """ import argparse as ap -from datetime import UTC, datetime +from datetime import datetime, timezone from sqlalchemy import select from sqlalchemy.orm import sessionmaker @@ -75,12 +75,12 @@ def core(session: sessionmaker, args: ap.Namespace): if args.start_time is not None: stmt = stmt.where( DepthOneMapTable.ctime - >= datetime.fromtimestamp(args.start_time, tz=UTC) + >= datetime.fromtimestamp(args.start_time, tz=timezone.utc) ) if args.end_time is not None: stmt = stmt.where( DepthOneMapTable.ctime - <= datetime.fromtimestamp(args.end_time, tz=UTC) + <= datetime.fromtimestamp(args.end_time, tz=timezone.utc) ) if args.from_status is not None: @@ -111,12 +111,12 @@ def core(session: sessionmaker, args: ap.Namespace): if args.start_time is not None: pr_stmt = pr_stmt.where( DepthOneMapTable.ctime - >= datetime.fromtimestamp(args.start_time, tz=UTC) + >= datetime.fromtimestamp(args.start_time, tz=timezone.utc) ) if args.end_time is not None: pr_stmt = pr_stmt.where( DepthOneMapTable.ctime - <= datetime.fromtimestamp(args.end_time, tz=UTC) + <= datetime.fromtimestamp(args.end_time, tz=timezone.utc) ) pointing_residuals = cur_session.execute(pr_stmt).scalars().all() diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index af4f0df..bca7e44 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -2,7 +2,7 @@ Test the core functions """ -from datetime import UTC, datetime +from datetime import datetime, timezone import pytest from astropy import units as u @@ -72,9 +72,9 @@ def test_create_depth_one(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE", tube_slot="OTi1", frequency="f090", - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), ) session.add(data) @@ -93,14 +93,14 @@ def test_create_depth_one(database_sessionmaker): assert dmap.tube_slot == "OTi1" assert dmap.frequency == "f090" ctime = dmap.ctime - ctime = ctime.replace(tzinfo=UTC) if ctime.tzinfo is None else ctime + ctime = ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime assert int(ctime.timestamp()) == 1755787524.0 # Make child tables with database_sessionmaker() as session: processing_status = TimeDomainProcessingTable( - processing_start=datetime.fromtimestamp(1756787524.0, tz=UTC), - processing_end=datetime.fromtimestamp(1756797524.0, tz=UTC), + processing_start=datetime.fromtimestamp(1756787524.0, tz=timezone.utc), + processing_end=datetime.fromtimestamp(1756797524.0, tz=timezone.utc), processing_status="done", map_id=map_id, ) @@ -115,9 +115,9 @@ def test_create_depth_one(database_sessionmaker): tod = TODDepthOneTable( obs_id="obs_1753486724_lati6_111", pwv=0.7, - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -175,13 +175,13 @@ def test_create_depth_one(database_sessionmaker): assert proc.map_id == map_id processing_start = proc.processing_start processing_start = ( - processing_start.replace(tzinfo=UTC) + processing_start.replace(tzinfo=timezone.utc) if processing_start.tzinfo is None else processing_start ) processing_end = proc.processing_end processing_end = ( - processing_end.replace(tzinfo=UTC) + processing_end.replace(tzinfo=timezone.utc) if processing_end.tzinfo is None else processing_end ) @@ -198,13 +198,19 @@ def test_create_depth_one(database_sessionmaker): assert tod.pwv == 0.7 assert tod.obs_id == "obs_1753486724_lati6_111" ctime = tod.ctime - ctime = ctime.replace(tzinfo=UTC) if ctime.tzinfo is None else ctime + ctime = ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime start_time = tod.start_time start_time = ( - start_time.replace(tzinfo=UTC) if start_time.tzinfo is None else start_time + start_time.replace(tzinfo=timezone.utc) + if start_time.tzinfo is None + else start_time ) stop_time = tod.stop_time - stop_time = stop_time.replace(tzinfo=UTC) if stop_time.tzinfo is None else stop_time + stop_time = ( + stop_time.replace(tzinfo=timezone.utc) + if stop_time.tzinfo is None + else stop_time + ) assert int(ctime.timestamp()) == 1755787524.0 assert int(start_time.timestamp()) == 1755687524.0 assert int(stop_time.timestamp()) == 1755887524.0 @@ -252,14 +258,14 @@ def test_add_remove_child_tables(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE2", tube_slot="OTi1", frequency="f090", - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), ) processing_status = TimeDomainProcessingTable( - processing_start=datetime.fromtimestamp(1756787524.0, tz=UTC), - processing_end=datetime.fromtimestamp(1756797524.0, tz=UTC), + processing_start=datetime.fromtimestamp(1756787524.0, tz=timezone.utc), + processing_end=datetime.fromtimestamp(1756797524.0, tz=timezone.utc), processing_status="done", map=dmap, ) @@ -274,9 +280,9 @@ def test_add_remove_child_tables(database_sessionmaker): tod = TODDepthOneTable( obs_id="obs_1753486724_lati6_111", pwv=0.7, - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -369,8 +375,8 @@ def test_create_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/DAILY/COADD", platform="satp3", interval="daily", - start_time=datetime.fromtimestamp(1755604800.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755691200.0, tz=UTC), + start_time=datetime.fromtimestamp(1755604800.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755691200.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -393,10 +399,16 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert cmap.interval == "daily" start_time = cmap.start_time start_time = ( - start_time.replace(tzinfo=UTC) if start_time.tzinfo is None else start_time + start_time.replace(tzinfo=timezone.utc) + if start_time.tzinfo is None + else start_time ) stop_time = cmap.stop_time - stop_time = stop_time.replace(tzinfo=UTC) if stop_time.tzinfo is None else stop_time + stop_time = ( + stop_time.replace(tzinfo=timezone.utc) + if stop_time.tzinfo is None + else stop_time + ) assert int(start_time.timestamp()) == 1755604800.0 assert int(stop_time.timestamp()) == 1755691200.0 assert cmap.freq_channel == "f090" @@ -410,7 +422,7 @@ def test_create_atomic_map_coadd(database_sessionmaker): telescope="satp3", freq_channel="f090", wafer="ws0", - ctime=datetime.fromtimestamp(1755643932, tz=UTC), + ctime=datetime.fromtimestamp(1755643932, tz=timezone.utc), split_label="full", map_path=None, ivar_path=None, @@ -462,7 +474,7 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert atomic.freq_channel == "f090" assert atomic.wafer == "ws0" ctime = atomic.ctime - ctime = ctime.replace(tzinfo=UTC) if ctime.tzinfo is None else ctime + ctime = ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime assert int(ctime.timestamp()) == 1755643932.0 assert atomic.split_label == "full" assert atomic.map_path is None @@ -506,8 +518,8 @@ def test_create_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/WEEKLY/COADD", platform="satp3", interval="weekly", - start_time=datetime.fromtimestamp(1755432000.0, tz=UTC), - stop_time=datetime.fromtimestamp(1756036800.0, tz=UTC), + start_time=datetime.fromtimestamp(1755432000.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1756036800.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -550,8 +562,8 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/DAILY/COADD", platform="satp3", interval="daily", - start_time=datetime.fromtimestamp(1755604800.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755691200.0, tz=UTC), + start_time=datetime.fromtimestamp(1755604800.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755691200.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -562,8 +574,8 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/WEEKLY/COADD", platform="satp3", interval="weekly", - start_time=datetime.fromtimestamp(1755432000.0, tz=UTC), - stop_time=datetime.fromtimestamp(1756036800.0, tz=UTC), + start_time=datetime.fromtimestamp(1755432000.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1756036800.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -599,8 +611,8 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/WEEKLY/COADD", platform="satp3", interval="weekly", - start_time=datetime.fromtimestamp(1755432000.0, tz=UTC), - stop_time=datetime.fromtimestamp(1756036800.0, tz=UTC), + start_time=datetime.fromtimestamp(1755432000.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1756036800.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", diff --git a/tests/test_mapmaking.py b/tests/test_mapmaking.py index 983ebbc..5e9df7b 100644 --- a/tests/test_mapmaking.py +++ b/tests/test_mapmaking.py @@ -1,4 +1,4 @@ -from datetime import UTC, datetime +from datetime import datetime, timezone from mapcat.database import DepthOneMapTable, TODDepthOneTable from mapcat.toolkit.mapmaking import build_obslists @@ -14,9 +14,9 @@ def test_build_obslists(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE", tube_slot="OTi1", frequency="f090", - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), ) data2 = DepthOneMapTable( @@ -24,9 +24,9 @@ def test_build_obslists(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE2", tube_slot="OTi4", frequency="f090", - ctime=datetime.fromtimestamp(1755788524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755787524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755897524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755788524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755897524.0, tz=timezone.utc), ) session.add(data1) @@ -56,9 +56,9 @@ def test_build_obslists(database_sessionmaker): tod1 = TODDepthOneTable( obs_id=obs_ids[0], pwv=0.7, - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -82,9 +82,9 @@ def test_build_obslists(database_sessionmaker): tod2 = TODDepthOneTable( obs_id=obs_ids[1], pwv=0.7, - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -108,9 +108,9 @@ def test_build_obslists(database_sessionmaker): tod3 = TODDepthOneTable( obs_id=obs_ids[2], pwv=0.7, - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -135,9 +135,9 @@ def test_build_obslists(database_sessionmaker): tod4 = TODDepthOneTable( obs_id=obs_ids[3], pwv=0.7, - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", diff --git a/tests/test_pointing.py b/tests/test_pointing.py index da0a072..bcdf4c0 100644 --- a/tests/test_pointing.py +++ b/tests/test_pointing.py @@ -2,7 +2,7 @@ Tests for the pointing residual models. """ -from datetime import UTC, datetime +from datetime import datetime, timezone import numpy as np from astropy import units as u @@ -23,9 +23,9 @@ def test_add_retrieve_pointing(database_sessionmaker): map_path="DoesntExist/Map", tube_slot="i1", frequency="f090", - ctime=datetime.fromtimestamp(1755787524.0, tz=UTC), - start_time=datetime.fromtimestamp(1755687524.0, tz=UTC), - stop_time=datetime.fromtimestamp(1755887524.0, tz=UTC), + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), ) session.add(sample_map) From e9945c74fdf9c7eae100e82ae4531b6313b753e2 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Thu, 6 Aug 2026 10:56:18 -0400 Subject: [PATCH 14/15] fix: missed a UTC import --- tests/test_reset.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_reset.py b/tests/test_reset.py index 739ec2d..9c52d26 100644 --- a/tests/test_reset.py +++ b/tests/test_reset.py @@ -3,7 +3,7 @@ """ import argparse -from datetime import UTC, datetime +from datetime import timezone, datetime import pytest from sqlalchemy import create_engine @@ -54,9 +54,9 @@ def _make_map(session, name, ctime, start_time=None, stop_time=None): map_path=f"/path/{name}_map.fits", tube_slot="OTi1", frequency="f090", - ctime=datetime.fromtimestamp(ctime, tz=UTC), - start_time=datetime.fromtimestamp(start_time, tz=UTC), - stop_time=datetime.fromtimestamp(stop_time, tz=UTC), + ctime=datetime.fromtimestamp(ctime, tz=timezone.utc), + start_time=datetime.fromtimestamp(start_time, tz=timezone.utc), + stop_time=datetime.fromtimestamp(stop_time, tz=timezone.utc), ) s.add(dmap) s.commit() @@ -69,8 +69,8 @@ def _make_proc(session, map_id, status): with session() as s: proc = TimeDomainProcessingTable( map_id=map_id, - processing_start=datetime.fromtimestamp(1756000000.0, tz=UTC), - processing_end=datetime.fromtimestamp(1756001000.0, tz=UTC), + processing_start=datetime.fromtimestamp(1756000000.0, tz=timezone.utc), + processing_end=datetime.fromtimestamp(1756001000.0, tz=timezone.utc), processing_status=status, ) s.add(proc) From 63e8f53bd5486b676908b72c039947f84749c9e0 Mon Sep 17 00:00:00 2001 From: Sulla2012 Date: Thu, 6 Aug 2026 10:56:59 -0400 Subject: [PATCH 15/15] fix: missed a UTC import --- tests/test_reset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_reset.py b/tests/test_reset.py index 9c52d26..87826f3 100644 --- a/tests/test_reset.py +++ b/tests/test_reset.py @@ -3,7 +3,7 @@ """ import argparse -from datetime import timezone, datetime +from datetime import datetime, timezone import pytest from sqlalchemy import create_engine