diff --git a/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py b/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py new file mode 100644 index 0000000..c8c07cb --- /dev/null +++ b/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py @@ -0,0 +1,165 @@ +"""Swapping to datetime + +Revision ID: 0762b3c7694d +Revises: 46575bc0d660 +Create Date: 2026-07-27 11:19:49.803412 + +""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import datetime, timezone + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0762b3c7694d" +down_revision: str | None = "46575bc0d660" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +TABLE_DEFINITIONS = { + "atomic_maps": { + "primary_key_name": "atomic_map_id", + "column_names": [("ctime", False)], + }, + "atomic_map_coadds": { + "primary_key_name": "coadd_id", + "column_names": [("start_time", False), ("stop_time", False)], + }, + "depth_one_maps": { + "primary_key_name": "map_id", + "column_names": [("ctime", True), ("start_time", True), ("stop_time", True)], + }, + "depth_one_coadds": { + "primary_key_name": "coadd_id", + "column_names": [("ctime", False), ("start_time", False), ("stop_time", False)], + }, +} + + +def unix_to_datetime( + table_name: str, + primary_key_name: str, + column_names: list[tuple[str, bool]], +) -> None: + """ + Convert a column from unix time to datetime. + + Parameters + ---------- + table_name : str + The name of the table to modify. + primary_key_name : str + The name of the primary key column for the table. + column_names : list[tuple[str, bool]] + The names of the columns to convert and boolean defining whether the column is indexed. + """ + bind = op.get_bind() + metadata = sa.MetaData() + cur_table = sa.Table(table_name, metadata, autoload_with=bind) + for column_name, is_indexed in column_names: + temp_col_name = f"temp_datetime_{column_name}" + if is_indexed: + op.drop_index(f"ix_{table_name}_{column_name}", table_name=table_name) + + op.add_column( + table_name, sa.Column(temp_col_name, sa.DateTime(), nullable=True) + ) + + stmt = sa.select(cur_table.c[column_name], cur_table.c[primary_key_name]) + results = bind.execute(stmt).fetchall() + for row in results: + unix_time = row[column_name] + primary_key_value = row[primary_key_name] + datetime_value = datetime.fromtimestamp(int(unix_time), tz=timezone.utc) + update_stmt = ( + cur_table.update() + .where(cur_table.c[primary_key_name] == primary_key_value) + .values({temp_col_name: datetime_value}) + ) + bind.execute(update_stmt) + + with op.batch_alter_table(table_name) as batch_op: + batch_op.drop_column(column_name) + batch_op.alter_column( + temp_col_name, + new_column_name=column_name, + nullable=False, + ) + + if is_indexed: + op.create_index(f"ix_{table_name}_{column_name}", table_name, [column_name]) + + +def _datetime_to_unix_value(datetime_value: datetime) -> int | None: + datetime_value = datetime_value.astimezone(timezone.utc) + return int(datetime_value.timestamp()) + + +def datetime_to_unix( + table_name: str, + primary_key_name: str, + column_names: list[tuple[str, bool]], +) -> None: + """ + Convert a column from datetime to unix time. + + Parameters + ---------- + table_name : str + The name of the table to modify. + primary_key_name : str + The name of the primary key column for the table. + column_names : list[tuple[str, bool]] + The names of the columns to convert and a boolean defining whether the column is indexed. + """ + bind = op.get_bind() + metadata = sa.MetaData() + cur_table = sa.Table(table_name, metadata, autoload_with=bind) + for column_name, is_indexed in column_names: + temp_col_name = f"temp_unix_{column_name}" + if is_indexed: + op.drop_index(f"ix_{table_name}_{column_name}", table_name=table_name) + + op.add_column(table_name, sa.Column(temp_col_name, sa.String(), nullable=True)) + + stmt = sa.select(cur_table.c[column_name], cur_table.c[primary_key_name]) + results = bind.execute(stmt).fetchall() + for row in results: + datetime_value = row[column_name] + primary_key_value = row[primary_key_name] + unix_time = _datetime_to_unix_value(datetime_value) + update_stmt = ( + cur_table.update() + .where(cur_table.c[primary_key_name] == primary_key_value) + .values({temp_col_name: unix_time}) + ) + bind.execute(update_stmt) + + with op.batch_alter_table(table_name) as batch_op: + batch_op.drop_column(column_name) + batch_op.alter_column( + temp_col_name, + new_column_name=column_name, + nullable=False, + ) + + if is_indexed: + op.create_index(f"ix_{table_name}_{column_name}", table_name, [column_name]) + + +def upgrade() -> None: + for table_name, table_data in TABLE_DEFINITIONS.items(): + unix_to_datetime( + table_name, table_data["primary_key_name"], table_data["column_names"] + ) + + +def downgrade() -> None: + for table_name, table_data in TABLE_DEFINITIONS.items(): + datetime_to_unix( + table_name, table_data["primary_key_name"], table_data["column_names"] + ) diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index 73653e9..4dd4fe7 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -2,8 +2,11 @@ Atomic map coadds """ +from datetime import datetime from typing import TYPE_CHECKING +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel if TYPE_CHECKING: @@ -12,6 +15,21 @@ from .links import AtomicMapToCoaddTable, CoaddMapToCoaddTable # pragma: no cover +class AtomicMapCoadd(SQLModel): + coadd_id: int + + coadd_name: str + prefix_path: str + + platform: str + interval: str + start_time: AstroPydanticTime + stop_time: AstroPydanticTime + freq_channel: str + geom_file_path: str + split_label: str + + class AtomicMapCoaddTable(SQLModel, table=True): __tablename__ = "atomic_map_coadds" @@ -22,8 +40,8 @@ class AtomicMapCoaddTable(SQLModel, table=True): platform: str = Field() interval: str = Field() - start_time: float = Field() - stop_time: float = Field() + start_time: datetime = Field(nullable=False) + stop_time: datetime = Field(nullable=False) freq_channel: str = Field() geom_file_path: str = Field() split_label: str = Field() @@ -50,3 +68,25 @@ class AtomicMapCoaddTable(SQLModel, table=True): "secondaryjoin": "AtomicMapCoaddTable.coadd_id == CoaddMapToCoaddTable.parent_coadd_id", }, ) + + def to_model(self) -> AtomicMapCoadd: + """ + Return an AtomicMapCoadd model from this table entry. + + Returns + ------- + AtomicMapCoadd : AtomicMapCoadd + The AtomicMapCoadd model corresponding to this table entry. + """ + return AtomicMapCoadd( + coadd_id=self.coadd_id, + coadd_name=self.coadd_name, + prefix_path=self.prefix_path, + platform=self.platform, + interval=self.interval, + start_time=Time(self.start_time), + stop_time=Time(self.stop_time), + freq_channel=self.freq_channel, + geom_file_path=self.geom_file_path, + split_label=self.split_label, + ) diff --git a/mapcat/database/atomic_map.py b/mapcat/database/atomic_map.py index 08753a3..0b6e266 100644 --- a/mapcat/database/atomic_map.py +++ b/mapcat/database/atomic_map.py @@ -2,8 +2,11 @@ Table for atomic maps. """ +from datetime import datetime from typing import TYPE_CHECKING +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel from .links import AtomicMapToCoaddTable @@ -12,6 +15,48 @@ from .atomic_coadd import AtomicMapCoaddTable # pragma: no cover +class AtomicMap(SQLModel): + atomic_map_id: int + + obs_id: str + telescope: str + freq_channel: str + wafer: str + ctime: AstroPydanticTime + split_label: str + + map_path: str | None + ivar_path: str | None + + valid: bool | None + split_detail: str | None + prefix_path: str | None + azimuth: float | None + pwv: float | None + dpwv: float | None + total_weight_qu: float | None + mean_weight_qu: float | None + median_weight_qu: float | None + leakage_avg: float | None + noise_avg: float | None + ampl_2f_avg: float | None + gain_avg: float | None + f_hwp: float | None + roll_angle: float | None + scan_speed: float | None + scan_acc: float | None + sun_distance: float | None + ambient_temperature: float | None + uv: float | None + ra_center: float | None + dec_center: float | None + number_dets: int | None + moon_distance: float | None + wind_speed: float | None + wind_direction: float | None + rqu_avg: float | None + + class AtomicMapTable(SQLModel, table=True): __tablename__ = "atomic_maps" @@ -21,7 +66,7 @@ class AtomicMapTable(SQLModel, table=True): telescope: str = Field() freq_channel: str = Field() wafer: str = Field() - ctime: int = Field() + ctime: datetime = Field(nullable=False) split_label: str = Field() map_path: str | None = Field() @@ -61,3 +106,50 @@ class AtomicMapTable(SQLModel, table=True): back_populates="atomic_maps", link_model=AtomicMapToCoaddTable, ) + + def to_model(self) -> AtomicMap: + """ + Return an AtomicMap model from this table entry. + + Returns + ------- + AtomicMap : AtomicMap + The AtmoicMap model corresponding to this table entry. + """ + return AtomicMap( + atomic_map_id=self.atomic_map_id, + obs_id=self.obs_id, + telescope=self.telescope, + freq_channel=self.freq_channel, + wafer=self.wafer, + ctime=Time(self.ctime), + split_label=self.split_label, + map_path=self.map_path, + ivar_path=self.ivar_path, + valid=self.valid, + split_detail=self.split_detail, + prefix_path=self.prefix_path, + azimuth=self.azimuth, + pwv=self.pwv, + dpwv=self.dpwv, + total_weight_qu=self.total_weight_qu, + mean_weight_qu=self.mean_weight_qu, + leakage_avg=self.leakage_avg, + noise_avg=self.noise_avg, + ampl_2f_avg=self.ampl_2f_ave, + gain_avg=self.gain_avg, + f_hwp=self.f_hwp, + roll_angle=self.roll_angle, + scan_speed=self.scan_speed, + scan_acc=self.scan_acc, + sun_distance=self.sun_distance, + ambient_temperature=self.ambient_temperature, + uv=self.uv, + ra_center=self.ra_center, + dec_center=self.dec_center, + number_dets=self.number_dets, + moon_distance=self.moon_distance, + wind_speed=self.wind_speed, + wind_direction=self.wind_direction, + rqu_avg=self.rqu_avg, + ) diff --git a/mapcat/database/depth_one_coadd.py b/mapcat/database/depth_one_coadd.py index 9660be9..a2e0702 100644 --- a/mapcat/database/depth_one_coadd.py +++ b/mapcat/database/depth_one_coadd.py @@ -2,12 +2,35 @@ Table containing information about Depth-1 map coadds. """ +from datetime import datetime + +from astropy.time import Time from sqlmodel import Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable from .links import DepthOneToCoaddTable +class DepthOneCoadd(SQLModel): + coadd_id: int + coadd_name: str + coadd_type: str + + map_path: str + ivar_path: str | None + rho_path: str | None + kappa_path: str | None + + start_time_path: str | None + mean_time_path: str | None + end_time_path: str | None + + frequency: str + ctime: datetime + start_time: datetime + stop_time: datetime + + class DepthOneCoaddTable(SQLModel, table=True): """ A co-add of multiple depth-1 maps. This is the table model, @@ -30,11 +53,37 @@ class DepthOneCoaddTable(SQLModel, table=True): end_time_path: str | None = None frequency: str = Field(nullable=False) - ctime: float = Field(nullable=False) - start_time: float = Field(nullable=False) - stop_time: float = Field(nullable=False) + ctime: datetime = Field(nullable=False) + start_time: datetime = Field(nullable=False) + stop_time: datetime = Field(nullable=False) maps: list["DepthOneMapTable"] = Relationship( back_populates="coadds", link_model=DepthOneToCoaddTable, ) + + def to_model(self) -> DepthOneCoadd: + """ + Return an DepthOneCoadd model from this table entry. + + Returns + ------- + DepthOneCoadd : DepthOneCoadd + The DepthOneCoadd model corresponding to this table entry. + """ + return DepthOneCoadd( + coadd_id=self.coadd_id, + coadd_name=self.coadd_name, + coadd_type=self.coadd_type, + map_path=self.map_path, + ivar_path=self.ivar_path, + rho_path=self.rho_path, + kappa_path=self.kappa_path, + start_time_path=self.start_time_path, + mean_time_path=self.mean_time_path, + end_time_path=self.end_time_path, + frequency=self.frequency, + ctime=Time(self.ctime), + start_time=Time(self.start_time), + stop_time=Time(self.end_time), + ) diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py index 6760f6a..cc338f0 100644 --- a/mapcat/database/depth_one_map.py +++ b/mapcat/database/depth_one_map.py @@ -2,8 +2,11 @@ Depth one map table. """ +from datetime import datetime from typing import TYPE_CHECKING, Any +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import JSON, Field, Relationship, SQLModel if TYPE_CHECKING: # pragma: no cover @@ -17,6 +20,28 @@ from .links import DepthOneToCoaddTable, TODToMapTable +class DepthOneMap(SQLModel): + map_id: int + map_name: str + + map_path: str | None + ivar_path: str | None + rho_path: str | None + kappa_path: str | None + flux_path: str | None + snr_path: str | None + + start_time_path: str | None + mean_time_path: str | None + end_time_path: str | None + + tube_slot: str + frequency: str + ctime: AstroPydanticTime + start_time: AstroPydanticTime + stop_time: AstroPydanticTime + + class DepthOneMapTable(SQLModel, table=True): """ A depth-1 map. @@ -54,11 +79,11 @@ class DepthOneMapTable(SQLModel, table=True): Standardized names of wafers used in this map frequency : str Frequency channel of map - ctime : float + ctime : datetime Mean unix time of map - start_time : float + start_time : datetime Start unix time of map - stop_time : float + stop_time : datetime Stop unix time of map processing_status : list[TimeDomainProcessingTable] List of processing status tables associated with d1 map @@ -93,9 +118,9 @@ class DepthOneMapTable(SQLModel, table=True): tube_slot: str = Field(index=True, nullable=False) frequency: str = Field(index=True, nullable=False) - ctime: float = Field(index=True, nullable=False) - start_time: float = Field(index=True, nullable=False) - stop_time: float = Field(index=True, nullable=False) + ctime: datetime = Field(index=True, nullable=False) + start_time: datetime = Field(index=True, nullable=False) + stop_time: datetime = Field(index=True, nullable=False) processing_status: list["TimeDomainProcessingTable"] = Relationship( back_populates="map", @@ -146,3 +171,31 @@ def coverage_path(self) -> str: raise ValueError( f"No coverage map available for map {self.map_name} (id {self.map_id})" ) + + def to_model(self) -> DepthOneMap: + """ + Return an DepthOneMap model from this table entry. + + Returns + ------- + DepthOneMap : DepthOneMap + The DepthOneMap model corresponding to this table entry. + """ + return DepthOneMap( + map_id=self.map_id, + map_name=self.map_name, + map_path=self.map_path, + ivar_path=self.ivar_path, + rho_path=self.rho_path, + kappa_path=self.kappa_path, + flux_path=self.flux_path, + snr_path=self.snr_path, + start_time_path=self.start_time_path, + mean_time_path=self.mean_time_path, + end_time_path=self.end_time_path, + tube_slot=self.tube_slot, + frequency=self.frequency, + ctime=Time(self.ctime), + start_time=Time(self.start_time), + stop_time=Time(self.stop_time), + ) diff --git a/mapcat/database/pipeline_information.py b/mapcat/database/pipeline_information.py index 3f47ee0..8ca6ac2 100644 --- a/mapcat/database/pipeline_information.py +++ b/mapcat/database/pipeline_information.py @@ -15,7 +15,7 @@ class PipelineInformationTable(SQLModel, table=True): Attributes ---------- - id : str + id : int Internal ID of the pipeline info map_name : str Name of depth 1 map being tracked. Foreign into DepthOneMap diff --git a/mapcat/database/time_domain_processing.py b/mapcat/database/time_domain_processing.py index f271437..e21e62a 100644 --- a/mapcat/database/time_domain_processing.py +++ b/mapcat/database/time_domain_processing.py @@ -2,11 +2,25 @@ Table containing information about processing status of the Depth-1 maps. """ +from datetime import datetime + +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable +class TimeDomainProcessing(SQLModel): + processing_status_id: int + + map_id: int + + processing_start: AstroPydanticTime | None + processing_end: AstroPydanticTime | None + processing_status: str + + class TimeDomainProcessingTable(SQLModel, table=True): """ Table for tracking processing status of depth-1 maps @@ -16,13 +30,13 @@ class TimeDomainProcessingTable(SQLModel, table=True): Attributes ---------- - id : int + processing_status_id : int Internal ID of the processing status - map_name : str + map_name : int Name of depth 1 map being tracked. Foreign into DepthOneMap - processing_start : float | None + processing_start : datetime | None Time processing started. None if not started. - processing_end : float | None + processing_end : datetime | None Time processing ended. None if not ended. processing_status : str Status of processing @@ -40,6 +54,23 @@ class TimeDomainProcessingTable(SQLModel, table=True): ) map: DepthOneMapTable = Relationship(back_populates="processing_status") - processing_start: float = Field(nullable=True) - processing_end: float = Field(nullable=True) + processing_start: datetime = Field(nullable=True) + processing_end: datetime = Field(nullable=True) processing_status: str = Field(index=True, nullable=False) + + def to_model(self) -> TimeDomainProcessing: + """ + Return a TimeDomainProcessing model from this table entry + + Returns + ------- + TimeDomainProcessing : TimeDomainProcessing + The TimeDomainProcessing model corresponding to this table entry. + """ + return TimeDomainProcessing( + processing_status_id=self.processing_status_id, + map_id=self.map_id, + processing_start=Time(self.processing_start), + processing_end=Time(self.processing_end), + processing_status=self.processing_status, + ) diff --git a/mapcat/database/tod.py b/mapcat/database/tod.py index dcc8ec4..882f5db 100644 --- a/mapcat/database/tod.py +++ b/mapcat/database/tod.py @@ -2,19 +2,50 @@ Table for TODs """ +from datetime import datetime + +from astropy.time import Time +from astropydantic import AstroPydanticTime from sqlmodel import Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable from .links import TODToMapTable +class TODDepthOne(SQLModel): + tod_id: int + obs_id: str + pwv: float | None + ctime: AstroPydanticTime + start_time: AstroPydanticTime | None + stop_time: AstroPydanticTime | None + nsamples: int | None + telescope: str + telescope_flavor: str | None + tube_slot: str + tube_flavor: str | None + frequency: str + scan_type: str + subtype: str + wafer_count: int + duration: float + az_center: float + az_throw: float + el_center: float + el_throw: float + roll_center: float + roll_throw: float + wafer_slots_list: str + stream_ids_list: str + + class TODDepthOneTable(SQLModel, table=True): """ Table of TODs used in making depth 1 maps. Attributes ---------- - id : int + tod_id : int Unique TOD identifier. Internal to SO map_name : str Name of map this TOD went into. Foreign key @@ -22,11 +53,11 @@ class TODDepthOneTable(SQLModel, table=True): SO ID of TOD pwv : float Precipitable water vapor at time of obs - ctime : float + ctime : datetime Mean unix time of obs - start_time : float + start_time : datetime Start time of obs - stop_time : float + stop_time : datetime End time of obs nsamples : int Number of samps in obs @@ -70,9 +101,9 @@ class TODDepthOneTable(SQLModel, table=True): tod_id: int = Field(primary_key=True) obs_id: str = Field(nullable=False) pwv: float | None = Field(index=True, nullable=True) - ctime: float = Field(index=True, nullable=False) - start_time: float | None = Field(index=True, nullable=True) - stop_time: float | None = Field(index=True, nullable=True) + ctime: datetime = Field(index=True, nullable=False) + start_time: datetime | None = Field(index=True, nullable=True) + stop_time: datetime | None = Field(index=True, nullable=True) nsamples: int | None = Field() telescope: str = Field(index=True, nullable=False) telescope_flavor: str | None = Field() @@ -94,3 +125,39 @@ class TODDepthOneTable(SQLModel, table=True): maps: list[DepthOneMapTable] = Relationship( back_populates="tods", link_model=TODToMapTable ) + + def to_model(self) -> TODDepthOne: + """ + Return an TODDepthOne model from this table entry. + + Returns + ------- + TODDepthOne : TODDepthOne + The TODDepthOne model corresponding to this table entry. + """ + return TODDepthOne( + tod_id=self.tod_id, + obs_id=self.obs_id, + pwv=self.pwv, + ctime=Time(self.ctime), + start_time=Time(self.start_time), + stop_time=Time(self.stop_time), + nsamples=self.nsamples, + telescope=self.telescope, + telescope_flavor=self.telescope_flavor, + tube_slot=self.tube_slot, + tube_flavor=self.tube_flavor, + frequency=self.frequency, + scan_type=self.scan_type, + subtype=self.subtype, + wafer_count=self.wafer_count, + duration=self.duration, + az_center=self.az_center, + az_throw=self.az_throw, + el_center=self.el_center, + el_throw=self.el_throw, + roll_center=self.roll_center, + roll_throw=self.roll_throw, + wafer_slots_list=self.wafer_slots_list, + stream_ids_list=self.stream_ids_list, + ) diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index 8586f65..0244753 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -3,6 +3,7 @@ """ import argparse as ap +from datetime import datetime, timezone from pathlib import Path import h5py @@ -70,7 +71,7 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT TODDepthOneTable( obs_id=obs_id, pwv=None, - ctime=float(obs_id[4:14]), + ctime=datetime.fromtimestamp(float(obs_id[4:14]), tz=timezone.utc), telescope=telescope, tube_slot=file_info["tube_slot"], frequency=file_info["frequency"], @@ -87,9 +88,9 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT mean_time_path=filenames.get("time"), tube_slot=file_info["tube_slot"], frequency=file_info["frequency"], - ctime=file_info["ctime"], - start_time=file_info["start_time"], - stop_time=file_info["stop_time"], + ctime=datetime.fromtimestamp(file_info["ctime"], tz=timezone.utc), + start_time=datetime.fromtimestamp(file_info["start_time"], tz=timezone.utc), + stop_time=datetime.fromtimestamp(file_info["stop_time"], tz=timezone.utc), tods=tods, ) diff --git a/mapcat/toolkit/reset.py b/mapcat/toolkit/reset.py index 78cbc18..d7c61cc 100644 --- a/mapcat/toolkit/reset.py +++ b/mapcat/toolkit/reset.py @@ -3,6 +3,7 @@ """ import argparse as ap +from datetime import datetime, timezone from sqlalchemy import select from sqlalchemy.orm import sessionmaker @@ -72,9 +73,15 @@ def core(session: sessionmaker, args: ap.Namespace): TimeDomainProcessingTable.map_id == DepthOneMapTable.map_id, ) if args.start_time is not None: - stmt = stmt.where(DepthOneMapTable.ctime >= args.start_time) + stmt = stmt.where( + DepthOneMapTable.ctime + >= datetime.fromtimestamp(args.start_time, tz=timezone.utc) + ) if args.end_time is not None: - stmt = stmt.where(DepthOneMapTable.ctime <= args.end_time) + stmt = stmt.where( + DepthOneMapTable.ctime + <= datetime.fromtimestamp(args.end_time, tz=timezone.utc) + ) if args.from_status is not None: stmt = stmt.where( @@ -102,9 +109,15 @@ def core(session: sessionmaker, args: ap.Namespace): PointingResidualTable.map_id == DepthOneMapTable.map_id, ) if args.start_time is not None: - pr_stmt = pr_stmt.where(DepthOneMapTable.ctime >= args.start_time) + pr_stmt = pr_stmt.where( + DepthOneMapTable.ctime + >= datetime.fromtimestamp(args.start_time, tz=timezone.utc) + ) if args.end_time is not None: - pr_stmt = pr_stmt.where(DepthOneMapTable.ctime <= args.end_time) + pr_stmt = pr_stmt.where( + DepthOneMapTable.ctime + <= datetime.fromtimestamp(args.end_time, tz=timezone.utc) + ) pointing_residuals = cur_session.execute(pr_stmt).scalars().all() for pr in pointing_residuals: diff --git a/tests/test_act.py b/tests/test_act.py index 580c033..574833e 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -1,5 +1,6 @@ import argparse as ap import os +from datetime import timezone from pathlib import Path import astropy.units as u @@ -218,7 +219,11 @@ def test_act(database_sessionmaker, downloaded_data_file): for map in maps: assert map.tube_slot in ["pa4", "pa6"] assert map.frequency == "f150" - assert map.ctime in [1505603190, 1505646390] + ctime = map.ctime + ctime = ( + ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime + ) + assert int(ctime.timestamp()) in [1505603190, 1505646390] # Clean up, otherewise we interfere with test_sky_coverage session.delete(map) @@ -239,6 +244,10 @@ def test_sky_coverage(database_sessionmaker, downloaded_data_file): with database_sessionmaker() as session: d1maps = session.query(DepthOneMapTable).all() for d1map in d1maps: + ctime = d1map.ctime + ctime = ( + ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime + ) assert len(d1map.depth_one_sky_coverage) > 0 for cov in d1map.depth_one_sky_coverage: # Shitty test to make sure the coverage tiles are correct, by checking against the known coverage for these two maps. @@ -248,7 +257,7 @@ def test_sky_coverage(database_sessionmaker, downloaded_data_file): cov.x ), # These should be ints, idk why I have to cast them (from str) int(cov.y), - ) in cov_mapping[str(d1map.ctime)] + ) in cov_mapping[str(ctime.timestamp())] with database_sessionmaker() as session: maps = session.query(DepthOneMapTable).all() diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index c92629e..bca7e44 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -2,6 +2,8 @@ Test the core functions """ +from datetime import datetime, timezone + import pytest from astropy import units as u from sqlalchemy import create_engine @@ -70,9 +72,9 @@ def test_create_depth_one(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE", tube_slot="OTi1", frequency="f090", - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), ) session.add(data) @@ -90,13 +92,15 @@ def test_create_depth_one(database_sessionmaker): assert dmap.map_path == "/PATH/TO/DEPTH/ONE" assert dmap.tube_slot == "OTi1" assert dmap.frequency == "f090" - assert dmap.ctime == 1755787524.0 + ctime = dmap.ctime + ctime = ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime + assert int(ctime.timestamp()) == 1755787524.0 # Make child tables with database_sessionmaker() as session: processing_status = TimeDomainProcessingTable( - processing_start=1756787524.0, - processing_end=1756797524.0, + processing_start=datetime.fromtimestamp(1756787524.0, tz=timezone.utc), + processing_end=datetime.fromtimestamp(1756797524.0, tz=timezone.utc), processing_status="done", map_id=map_id, ) @@ -111,9 +115,9 @@ def test_create_depth_one(database_sessionmaker): tod = TODDepthOneTable( obs_id="obs_1753486724_lati6_111", pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -169,8 +173,20 @@ def test_create_depth_one(database_sessionmaker): assert proc.processing_status_id == proc_id assert proc.map_id == map_id - assert proc.processing_start == 1756787524.0 - assert proc.processing_end == 1756797524.0 + processing_start = proc.processing_start + processing_start = ( + processing_start.replace(tzinfo=timezone.utc) + if processing_start.tzinfo is None + else processing_start + ) + processing_end = proc.processing_end + processing_end = ( + processing_end.replace(tzinfo=timezone.utc) + if processing_end.tzinfo is None + else processing_end + ) + assert int(processing_start.timestamp()) == 1756787524.0 + assert int(processing_end.timestamp()) == 1756797524.0 assert proc.processing_status == "done" assert point.pointing_residual_id == point_id @@ -181,9 +197,23 @@ def test_create_depth_one(database_sessionmaker): assert tod.tod_id == tod_id assert tod.pwv == 0.7 assert tod.obs_id == "obs_1753486724_lati6_111" - assert tod.ctime == 1755787524.0 - assert tod.start_time == 1755687524.0 - assert tod.stop_time == 1755887524.0 + ctime = tod.ctime + ctime = ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime + start_time = tod.start_time + start_time = ( + start_time.replace(tzinfo=timezone.utc) + if start_time.tzinfo is None + else start_time + ) + stop_time = tod.stop_time + stop_time = ( + stop_time.replace(tzinfo=timezone.utc) + if stop_time.tzinfo is None + else stop_time + ) + assert int(ctime.timestamp()) == 1755787524.0 + assert int(start_time.timestamp()) == 1755687524.0 + assert int(stop_time.timestamp()) == 1755887524.0 assert tod.nsamples == 28562 assert tod.telescope == "lat" assert tod.telescope_flavor == "lat" @@ -228,14 +258,14 @@ def test_add_remove_child_tables(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE2", tube_slot="OTi1", frequency="f090", - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), ) processing_status = TimeDomainProcessingTable( - processing_start=1756787524.0, - processing_end=1756797524.0, + processing_start=datetime.fromtimestamp(1756787524.0, tz=timezone.utc), + processing_end=datetime.fromtimestamp(1756797524.0, tz=timezone.utc), processing_status="done", map=dmap, ) @@ -250,9 +280,9 @@ def test_add_remove_child_tables(database_sessionmaker): tod = TODDepthOneTable( obs_id="obs_1753486724_lati6_111", pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -345,8 +375,8 @@ def test_create_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/DAILY/COADD", platform="satp3", interval="daily", - start_time=1755604800.0, - stop_time=1755691200.0, + start_time=datetime.fromtimestamp(1755604800.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755691200.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -367,8 +397,20 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert cmap.prefix_path == "/PATH/TO/DAILY/COADD" assert cmap.platform == "satp3" assert cmap.interval == "daily" - assert cmap.start_time == 1755604800.0 - assert cmap.stop_time == 1755691200.0 + start_time = cmap.start_time + start_time = ( + start_time.replace(tzinfo=timezone.utc) + if start_time.tzinfo is None + else start_time + ) + stop_time = cmap.stop_time + stop_time = ( + stop_time.replace(tzinfo=timezone.utc) + if stop_time.tzinfo is None + else stop_time + ) + assert int(start_time.timestamp()) == 1755604800.0 + assert int(stop_time.timestamp()) == 1755691200.0 assert cmap.freq_channel == "f090" assert cmap.geom_file_path == "/PATH/TO/GEOM/FILE" assert cmap.split_label == "full" @@ -380,7 +422,7 @@ def test_create_atomic_map_coadd(database_sessionmaker): telescope="satp3", freq_channel="f090", wafer="ws0", - ctime=1755643932, + ctime=datetime.fromtimestamp(1755643932, tz=timezone.utc), split_label="full", map_path=None, ivar_path=None, @@ -431,7 +473,9 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert atomic.telescope == "satp3" assert atomic.freq_channel == "f090" assert atomic.wafer == "ws0" - assert atomic.ctime == 1755643932 + ctime = atomic.ctime + ctime = ctime.replace(tzinfo=timezone.utc) if ctime.tzinfo is None else ctime + assert int(ctime.timestamp()) == 1755643932.0 assert atomic.split_label == "full" assert atomic.map_path is None assert atomic.ivar_path is None @@ -474,8 +518,8 @@ def test_create_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/WEEKLY/COADD", platform="satp3", interval="weekly", - start_time=1755432000.0, - stop_time=1756036800.0, + start_time=datetime.fromtimestamp(1755432000.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1756036800.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -518,8 +562,8 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/DAILY/COADD", platform="satp3", interval="daily", - start_time=1755604800.0, - stop_time=1755691200.0, + start_time=datetime.fromtimestamp(1755604800.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755691200.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -530,8 +574,8 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/WEEKLY/COADD", platform="satp3", interval="weekly", - start_time=1755432000.0, - stop_time=1756036800.0, + start_time=datetime.fromtimestamp(1755432000.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1756036800.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", @@ -567,8 +611,8 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): prefix_path="/PATH/TO/WEEKLY/COADD", platform="satp3", interval="weekly", - start_time=1755432000.0, - stop_time=1756036800.0, + start_time=datetime.fromtimestamp(1755432000.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1756036800.0, tz=timezone.utc), freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", diff --git a/tests/test_mapmaking.py b/tests/test_mapmaking.py index a681cfc..5e9df7b 100644 --- a/tests/test_mapmaking.py +++ b/tests/test_mapmaking.py @@ -1,3 +1,5 @@ +from datetime import datetime, timezone + from mapcat.database import DepthOneMapTable, TODDepthOneTable from mapcat.toolkit.mapmaking import build_obslists @@ -12,9 +14,9 @@ def test_build_obslists(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE", tube_slot="OTi1", frequency="f090", - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), ) data2 = DepthOneMapTable( @@ -22,9 +24,9 @@ def test_build_obslists(database_sessionmaker): map_path="/PATH/TO/DEPTH/ONE2", tube_slot="OTi4", frequency="f090", - ctime=1755788524.0, - start_time=1755787524.0, - stop_time=1755897524.0, + ctime=datetime.fromtimestamp(1755788524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755897524.0, tz=timezone.utc), ) session.add(data1) @@ -54,9 +56,9 @@ def test_build_obslists(database_sessionmaker): tod1 = TODDepthOneTable( obs_id=obs_ids[0], pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -80,9 +82,9 @@ def test_build_obslists(database_sessionmaker): tod2 = TODDepthOneTable( obs_id=obs_ids[1], pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -106,9 +108,9 @@ def test_build_obslists(database_sessionmaker): tod3 = TODDepthOneTable( obs_id=obs_ids[2], pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", @@ -133,9 +135,9 @@ def test_build_obslists(database_sessionmaker): tod4 = TODDepthOneTable( obs_id=obs_ids[3], pwv=0.7, - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), nsamples=28562, telescope="lat", telescope_flavor="lat", diff --git a/tests/test_pointing.py b/tests/test_pointing.py index 35e4712..bcdf4c0 100644 --- a/tests/test_pointing.py +++ b/tests/test_pointing.py @@ -2,6 +2,8 @@ Tests for the pointing residual models. """ +from datetime import datetime, timezone + import numpy as np from astropy import units as u from astropy.coordinates import SkyCoord @@ -21,9 +23,9 @@ def test_add_retrieve_pointing(database_sessionmaker): map_path="DoesntExist/Map", tube_slot="i1", frequency="f090", - ctime=1755787524.0, - start_time=1755687524.0, - stop_time=1755887524.0, + ctime=datetime.fromtimestamp(1755787524.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(1755687524.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(1755887524.0, tz=timezone.utc), ) session.add(sample_map) diff --git a/tests/test_reset.py b/tests/test_reset.py index 2650631..87826f3 100644 --- a/tests/test_reset.py +++ b/tests/test_reset.py @@ -3,6 +3,7 @@ """ import argparse +from datetime import datetime, timezone import pytest from sqlalchemy import create_engine @@ -44,14 +45,18 @@ def database_sessionmaker(tmp_path_factory): def _make_map(session, name, ctime, start_time=None, stop_time=None): """Helper to insert a DepthOneMapTable row and return its map_id.""" with session() as s: + if start_time is None: + start_time = ctime - 500 + if stop_time is None: + stop_time = ctime + 500 dmap = DepthOneMapTable( map_name=name, map_path=f"/path/{name}_map.fits", tube_slot="OTi1", frequency="f090", - ctime=ctime, - start_time=start_time or ctime - 500, - stop_time=stop_time or ctime + 500, + ctime=datetime.fromtimestamp(ctime, tz=timezone.utc), + start_time=datetime.fromtimestamp(start_time, tz=timezone.utc), + stop_time=datetime.fromtimestamp(stop_time, tz=timezone.utc), ) s.add(dmap) s.commit() @@ -64,8 +69,8 @@ def _make_proc(session, map_id, status): with session() as s: proc = TimeDomainProcessingTable( map_id=map_id, - processing_start=1756000000.0, - processing_end=1756001000.0, + processing_start=datetime.fromtimestamp(1756000000.0, tz=timezone.utc), + processing_end=datetime.fromtimestamp(1756001000.0, tz=timezone.utc), processing_status=status, ) s.add(proc)