From a992abcf5b851ed63d3ba21a2e1533f14e45710c Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Fri, 23 Feb 2024 16:13:31 -0800 Subject: [PATCH 01/13] Import the chains instead of re-constructing them for HSC The extra fixup for HSC-RC2 isn't necessary and they can just be exported/imported. --- bin.src/make_hsc_rc2_export.py | 14 ++++++----- bin.src/make_remote_butler.py | 43 ---------------------------------- 2 files changed, 8 insertions(+), 49 deletions(-) diff --git a/bin.src/make_hsc_rc2_export.py b/bin.src/make_hsc_rc2_export.py index cd70e75d..1b4f0ef0 100755 --- a/bin.src/make_hsc_rc2_export.py +++ b/bin.src/make_hsc_rc2_export.py @@ -92,14 +92,16 @@ def _export_for_copy(butler): ) contents.saveDatasets(records) - # Save calibration collection + # Save selected collections and chains for collection in butler.registry.queryCollections( - expression="HSC/calib*", - collectionTypes=daf_butler.CollectionType.CALIBRATION, - ): + expression="HSC/calib", + flattenChains=True, + includeChains=True, + ) + [ + "HSC/templates", + "HSC/calib/unbounded", + ]: contents.saveCollection(collection) - # Do not export chains, as they will need to be reworked to satisfy - # prompt processing's assumptions. if __name__ == "__main__": diff --git a/bin.src/make_remote_butler.py b/bin.src/make_remote_butler.py index 4ad6c05e..8b7cd22a 100755 --- a/bin.src/make_remote_butler.py +++ b/bin.src/make_remote_butler.py @@ -55,7 +55,6 @@ def _make_parser(): help="The export file containing the repository contents. Defaults to ./export.yaml.") parser.add_argument("--instrument", help="The short name of the instrument (HSC, LATISS, etc).") - parser.add_argument("--hsc-rc2", action="store_true", help="Extra fix up for HSC-RC2 dataset.") return parser @@ -97,46 +96,6 @@ def _add_chains(butler, instrument_name): ) -def _hsc_rc2(butler): - """fix up some specifics of the HSC-RC2 dataset export - - Parameters - ---------- - butler: `lsst.daf.butler.Butler` - The source Butler from which datasets are exported - """ - # Chain calibration collections - instrument = Instrument.fromName("HSC", butler.registry) - butler.registry.setCollectionChain( - instrument.makeCalibrationCollectionName(), - [ - "HSC/calib/DM-32378", - "HSC/calib/gen2/20180117", - "HSC/calib/DM-28636", - ], - ) - - butler.registry.registerCollection( - instrument.makeUnboundedCalibrationRunName(), - type=CollectionType.CHAINED - ) - butler.registry.setCollectionChain( - instrument.makeUnboundedCalibrationRunName(), - [ - "HSC/calib/gen2/20180117/unbounded", - "HSC/calib/DM-28636/unbounded", - ], - ) - # Chain rerun collections to templates - # The export script should have guaranteed that there are only coadds in these collections. - current = butler.registry.getCollectionChain("templates") - addition = butler.registry.queryCollections("HSC/runs/*", - collectionTypes=CollectionType.RUN) - butler.registry.setCollectionChain("templates", - list(addition) + list(current), - flatten=False) - - def main(): logging.basicConfig(level=logging.INFO, stream=sys.stdout) @@ -150,8 +109,6 @@ def main(): with time_this(msg="Import", level=logging.INFO): butler.import_(directory=args.src_repo, filename=args.export_file, transfer="auto") _add_chains(butler, args.instrument) - if args.hsc_rc2: - _hsc_rc2(butler) if __name__ == "__main__": From 39fe81e1ddf19bc27d41b78d32c84308b4c9ed01 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Fri, 23 Feb 2024 15:03:59 -0800 Subject: [PATCH 02/13] Move the LATISS export selection to a yaml config file Instead of hard-coding the selection in the script, move them to a yaml file so the script can be used for other exports more easily. --- bin.src/make_latiss_export.py | 106 +++++++++++++++------------------- etc/export_latiss.yaml | 23 ++++++++ 2 files changed, 70 insertions(+), 59 deletions(-) create mode 100644 etc/export_latiss.yaml diff --git a/bin.src/make_latiss_export.py b/bin.src/make_latiss_export.py index 0dff97c6..97417528 100755 --- a/bin.src/make_latiss_export.py +++ b/bin.src/make_latiss_export.py @@ -21,9 +21,9 @@ # along with this program. If not, see . -"""Selectively export the contents of the LATISS dataset. +"""Selectively export some contents from a butler repo. -This script selects some LATISS data in a source butler repo, and makes an export +This script selects some data in a source butler repo, and makes an export file for importing to the test central prompt processing repository. """ @@ -32,6 +32,7 @@ import logging import sys import tempfile +import yaml import lsst.daf.butler as daf_butler from lsst.utils.timer import time_this @@ -54,6 +55,13 @@ def _make_parser(): "exported from the source repo. If no target repo is given, all " "selected datasets in the source repo will be exported.", ) + parser.add_argument( + "--select", + required=True, + help="URI to a YAML file containing expressions to identify the " + "datasets and collections to be exported. An example is at " + "etc/export_latiss.yaml." + ) return parser @@ -62,6 +70,8 @@ def main(): args = _make_parser().parse_args() src_butler = daf_butler.Butler(args.src_repo) + with open(args.select, "r") as file: + wants = yaml.safe_load(file) with tempfile.TemporaryDirectory() as temp_repo: if args.target_repo: @@ -72,10 +82,10 @@ def main(): target_butler = daf_butler.Butler(config) with time_this(msg="Datasets and collections exported", level=logging.INFO): - _export_for_copy(src_butler, target_butler) + _export_for_copy(src_butler, target_butler, wants) -def _export_for_copy(butler, target_butler): +def _export_for_copy(butler, target_butler, wants): """Export selected data to make copies in another butler repo. Parameters @@ -86,65 +96,43 @@ def _export_for_copy(butler, target_butler): The target Butler to which datasets are exported. It is checked to avoid exporting existing datasets. No checks are done to verify if datasets are really identical. + wants : `dict` + A dictionary to identify selections with optional keys: + + ``"datasets"``, optional + A list of dataset selection expressions (`list` of `dict`). + The list is iterated over to find matching datasets in the butler, + with the matching criteria provided via the selection expressions. + Each selection expression has the keyworded argument dictionary to + be passed to butler to query datasets; it has the same meanings + as the parameters of `lsst.daf.butler.Registry.queryDatasets`. + ``"collections"``, optional + A list of collection selection expressions (`list` of `dict`). + The list is iterated over to find matching collections in the butler, + with the matching criteria provided via the selection expressions. + Each selection expression has the keyworded argument dictionary to + be passed to butler to query collectionss; it has the same meanings + as the parameters of `lsst.daf.butler.Registry.queryCollections`. """ with butler.export(format="yaml") as contents: - logging.debug("Selecting goodSeeingCoadd datasets") - records = _filter_datasets( - butler, - target_butler, - datasetType="goodSeeingCoadd", - collections="LATISS/templates", - ) - contents.saveDatasets(records) - - refcats = {"atlas_refcat2_20220201", "gaia_dr3_20230707"} - logging.debug(f"Selecting refcats datasets {refcats}") - records = _filter_datasets( - butler, target_butler, datasetType=refcats, collections="refcats*" - ) - contents.saveDatasets(records) - - logging.debug("Selecting skymaps dataset") - records = _filter_datasets( - butler, target_butler, datasetType="skyMap", collections="skymaps" - ) - contents.saveDatasets(records) - - logging.debug("Selecting datasets in LATISS/calib") - records = _filter_datasets( - butler, - target_butler, - datasetType=..., - # Workaround: use a matching expression rather than a specific - # string "LATISS/calib" for the collection argument, so to avoid - # MissingCollectionError when the collection does not exist in - # the target repo. - collections="*LATISS/calib", - ) - contents.saveDatasets(records) - - logging.debug("Selecting pretrained ML model dataset") - records = _filter_datasets( - butler, target_butler, datasetType="pretrainedModelPackage", collections="pretrained_models" - ) - contents.saveDatasets(records) + if "datasets" in wants: + for selection in wants["datasets"]: + logging.debug(f"Selecting datasets: {selection}") + if "datasetType" not in selection: + selection["datasetType"] = ... + records = _filter_datasets(butler, target_butler, **selection) + contents.saveDatasets(records) # Save selected collections and chains - for collection in butler.registry.queryCollections( - expression="LATISS/calib", - flattenChains=True, - includeChains=True, - ) + [ - "LATISS/templates", - "LATISS/calib/unbounded", - "pretrained_models", - ]: - logging.debug(f"Selecting collection {collection}") - try: - target_butler.registry.queryCollections(collection) - except daf_butler.registry.MissingCollectionError: - # MissingCollectionError is raised if the collection does not exist in target_butler. - contents.saveCollection(collection) + if "collections" in wants: + for selection in wants["collections"]: + for collection in butler.registry.queryCollections(**selection): + logging.debug(f"Selecting collection {collection}") + try: + target_butler.registry.queryCollections(collection) + except daf_butler.registry.MissingCollectionError: + # MissingCollectionError is raised if the collection does not exist in target_butler. + contents.saveCollection(collection) if __name__ == "__main__": diff --git a/etc/export_latiss.yaml b/etc/export_latiss.yaml new file mode 100644 index 00000000..83a04fdf --- /dev/null +++ b/etc/export_latiss.yaml @@ -0,0 +1,23 @@ +datasets: +- collections: LATISS/templates + datasetType: goodSeeingCoadd +- collections: refcats* + datasetType: atlas_refcat2_20220201 +- collections: refcats* + datasetType: gaia_dr3_20230707 +- collections: skymaps + datasetType: skyMap +# Workaround for DM-43294: use a matching expression rather than +# a specific string "LATISS/calib" for the collection argument, so +# to avoid MissingCollectionError when the collection does not +# exist in the target repo. +- collections: "*LATISS/calib" +- collections: pretrained_models + datasetType: pretrainedModelPackage +collections: +- expression: LATISS/calib + flattenChains: True + includeChains: True +- expression: LATISS/templates +- expression: LATISS/calib/unbounded +- expression: pretrained_models From b1e1a1a970c54b1d5e0a11383a34d01ee69826e9 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Fri, 23 Feb 2024 15:17:04 -0800 Subject: [PATCH 03/13] Use one generic export-making script There were data-specific scripts for making the export yaml. They can now be different config files but used with one same generic script. --- .../{make_latiss_export.py => make_export.py} | 0 bin.src/make_hsc_rc2_export.py | 108 ------------------ bin.src/make_template_export.py | 84 -------------- doc/playbook.rst | 2 +- etc/export_hsc_rc2.yaml | 19 +++ etc/export_templates.yaml | 3 + 6 files changed, 23 insertions(+), 193 deletions(-) rename bin.src/{make_latiss_export.py => make_export.py} (100%) delete mode 100755 bin.src/make_hsc_rc2_export.py delete mode 100755 bin.src/make_template_export.py create mode 100644 etc/export_hsc_rc2.yaml create mode 100644 etc/export_templates.yaml diff --git a/bin.src/make_latiss_export.py b/bin.src/make_export.py similarity index 100% rename from bin.src/make_latiss_export.py rename to bin.src/make_export.py diff --git a/bin.src/make_hsc_rc2_export.py b/bin.src/make_hsc_rc2_export.py deleted file mode 100755 index 1b4f0ef0..00000000 --- a/bin.src/make_hsc_rc2_export.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python -# This file is part of prompt_processing. -# -# Developed for the LSST Data Management System. -# This product includes software developed by the LSST Project -# (https://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -"""Selectively export the contents of the HSC-RC2 dataset. - -This script selects some HSC-RC2 data in a source butler repo, and makes an export -file for making a central prompt processing repository. -""" - - -import argparse -import logging -import sys -import time - -import lsst.daf.butler as daf_butler - - -def _make_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--src-repo", - required=True, - help="The location of the repository to be exported.", - ) - return parser - - -def main(): - logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) - - args = _make_parser().parse_args() - butler = daf_butler.Butler(args.src_repo) - - logging.info("Exporting Gen 3 registry to configure new repos...") - start = time.time_ns() - _export_for_copy(butler) - end = time.time_ns() - logging.info("Export finished in %.3fs.", 1e-9 * (end - start)) - - -def _export_for_copy(butler): - """Export selected data to make copies in another butler repo. - - Parameters - ---------- - butler: `lsst.daf.butler.Butler` - The source Butler from which datasets are exported - """ - with butler.export(format="yaml") as contents: - logging.debug("Selecting goodSeeingCoadd datasets") - records = butler.registry.queryDatasets( - datasetType="goodSeeingCoadd", - collections="HSC/runs/RC2/w_2022_44/DM-36763", - ) - contents.saveDatasets(records) - - logging.debug("Selecting refcats datasets") - records = butler.registry.queryDatasets( - datasetType=..., collections="refcats" - ) - contents.saveDatasets(records) - - logging.debug("Selecting skymaps dataset") - records = butler.registry.queryDatasets( - datasetType="skyMap", collections="skymaps", dataId={"skymap": "hsc_rings_v1"}) - contents.saveDatasets(records) - - logging.debug("Selecting datasets in HSC/calib") - records = butler.registry.queryDatasets( - datasetType=..., collections="HSC/calib" - ) - contents.saveDatasets(records) - - # Save selected collections and chains - for collection in butler.registry.queryCollections( - expression="HSC/calib", - flattenChains=True, - includeChains=True, - ) + [ - "HSC/templates", - "HSC/calib/unbounded", - ]: - contents.saveCollection(collection) - - -if __name__ == "__main__": - main() diff --git a/bin.src/make_template_export.py b/bin.src/make_template_export.py deleted file mode 100755 index b50ec081..00000000 --- a/bin.src/make_template_export.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# This file is part of prompt_processing. -# -# Developed for the LSST Data Management System. -# This product includes software developed by the LSST Project -# (https://www.lsst.org). -# See the COPYRIGHT file at the top-level directory of this distribution -# for details of code ownership. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -"""Export goodSeeingCoadd from a collection and make an export file -for importing those data to a central prompt processing repository. -All of the goodSeeingCoadd datasets in this collection are exported -without selection. -""" - - -import argparse -import logging -import sys -import time - -import lsst.daf.butler as daf_butler - - -def _make_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--src-repo", - required=True, - help="The location of the repository to be exported.", - ) - parser.add_argument( - "--collection", - required=True, - help="The collection to query data to be exported.", - ) - return parser - - -def main(): - logging.basicConfig(level=logging.INFO, stream=sys.stdout) - - args = _make_parser().parse_args() - butler = daf_butler.Butler(args.src_repo, collections=args.collection) - - logging.info("Exporting from %s", butler) - start = time.time_ns() - _export_for_copy(butler) - end = time.time_ns() - logging.info("Export finished in %.3fs.", 1e-9 * (end - start)) - - -def _export_for_copy(butler): - """Export selected data to make copies in another butler repo. - - Parameters - ---------- - butler: `lsst.daf.butler.Butler` - The source Butler from which datasets are exported - """ - with butler.export(format="yaml") as contents: - logging.debug("Selecting goodSeeingCoadd datasets") - records = butler.registry.queryDatasets( - datasetType="goodSeeingCoadd", - ) - contents.saveDatasets(records) - - -if __name__ == "__main__": - main() diff --git a/doc/playbook.rst b/doc/playbook.rst index caaa9f42..718e56b4 100644 --- a/doc/playbook.rst +++ b/doc/playbook.rst @@ -151,7 +151,7 @@ The bucket ``rubin-pp-dev`` holds incoming raw images. The bucket ``rubin-pp-dev-users`` holds: * ``rubin-pp-dev-users/central_repo/`` contains the central repository described in `DMTN-219`_. - This repository currently contains HSC and LATISS data, uploaded with ``make_hsc_rc2_export.py``, ``make_latiss_export.py``, and ``make_template_export.py``. + This repository currently contains HSC and LATISS data, uploaded with ``make_export.py``. * ``rubin-pp-dev-users/unobserved/`` contains raw files that the upload scripts can draw from to create incoming raws. diff --git a/etc/export_hsc_rc2.yaml b/etc/export_hsc_rc2.yaml new file mode 100644 index 00000000..94bf7c0b --- /dev/null +++ b/etc/export_hsc_rc2.yaml @@ -0,0 +1,19 @@ +datasets: +- collections: HSC/runs/RC2/w_2022_44/DM-36763 + datasetType: goodSeeingCoadd +- collections: refcats* +- collections: skymaps + dataId: + skymap: hsc_rings_v1 + datasetType: skyMap +# Workaround for DM-43294: use a matching expression rather than +# a specific string "HSC/calib" for the collection argument, so +# to avoid MissingCollectionError when the collection does not +# exist in the target repo. +- collections: "*HSC/calib" +collections: +- expression: HSC/calib + flattenChains: True + includeChains: True +- expression: HSC/templates +- expression: HSC/calib/unbounded diff --git a/etc/export_templates.yaml b/etc/export_templates.yaml new file mode 100644 index 00000000..9e6834d6 --- /dev/null +++ b/etc/export_templates.yaml @@ -0,0 +1,3 @@ +datasets: +- collections: u/hchiang2/w_2023_01/DM-37751/templates + datasetType: goodSeeingCoadd From 1753c41dcaeaf5f9c5ded5ce27e02821bd5b4fc4 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Fri, 23 Feb 2024 16:41:45 -0800 Subject: [PATCH 04/13] Add export config file for LSSTComCamSim This exports an early version of the simulated data from /repo/ops-rehearsal-3-prep to prompt-processing's dev central repo. --- etc/export_comCamSim.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 etc/export_comCamSim.yaml diff --git a/etc/export_comCamSim.yaml b/etc/export_comCamSim.yaml new file mode 100644 index 00000000..811a085d --- /dev/null +++ b/etc/export_comCamSim.yaml @@ -0,0 +1,16 @@ +datasets: +- collections: refcats + datasetType: uw_stars_20240228 +- collections: skymaps + datasetType: skyMap +# Workaround for DM-43294: use a matching expression rather than +# a specific collection string for the collection argument, so +# to avoid MissingCollectionError when the collection does not +# exist in the target repo. +- collections: "*LSSTComCamSim/calib" + +collections: +- expression: LSSTComCamSim/calib + flattenChains: True + includeChains: True +- expression: LSSTComCamSim/calib/unbounded From 867b76461ddee12a194ba10db34684b0db0d9bc3 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Wed, 6 Mar 2024 14:59:54 -0800 Subject: [PATCH 05/13] Add new refcats uw_stars_20240228 for LSSTComCamSim --- python/activator/middleware_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/activator/middleware_interface.py b/python/activator/middleware_interface.py index a44a2a48..dd605ce4 100644 --- a/python/activator/middleware_interface.py +++ b/python/activator/middleware_interface.py @@ -538,7 +538,7 @@ def _export_refcats(self, center, radius): # collection, so we have to specify a list here. Replace this # with another solution ASAP. possible_refcats = ["gaia", "panstarrs", "gaia_dr2_20200414", "ps1_pv3_3pi_20170110", - "atlas_refcat2_20220201", "gaia_dr3_20230707"] + "atlas_refcat2_20220201", "gaia_dr3_20230707", "uw_stars_20240228"] _log.debug("Searching for refcats in %s...", shard_ids) refcats = set(_filter_datasets( self.central_butler, self.butler, From d518edd80512445ca37d60fa6e89f902ddef69b3 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Wed, 28 Feb 2024 15:15:14 -0800 Subject: [PATCH 06/13] Add support for raws from the new LsstComCamSim camera --- python/activator/raw.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/activator/raw.py b/python/activator/raw.py index 7aa13736..0c38c204 100644 --- a/python/activator/raw.py +++ b/python/activator/raw.py @@ -40,7 +40,7 @@ import re import time -from lsst.obs.lsst import LsstCam, LsstComCam +from lsst.obs.lsst import LsstCam, LsstComCam, LsstComCamSim from lsst.obs.lsst.translators.lsst import LsstBaseTranslator from lsst.resources import ResourcePath @@ -65,7 +65,7 @@ ################################ # The list of camera names that might be used for LSST -_LSST_CAMERA_LIST = ("LATISS", "ComCam", "LSSTComCam", "LSSTCam", "TS8", "LSST-TS8") +_LSST_CAMERA_LIST = ("LATISS", "ComCam", "LSSTComCam", "LSSTComCamSim", "LSSTCam", "TS8", "LSST-TS8") # Translate from Camera path prefixes to official names. _TRANSLATE_INSTRUMENT = { @@ -77,6 +77,7 @@ _CAMERA_ABBREV = { "LATISS": "AT", "LSSTComCam": "CC", + "LSSTComCamSim": "CC", "LSSTCam": "MC", "LSST-TS8": "TS", } @@ -90,10 +91,12 @@ _LSSTCAM = LsstCam.getCamera().getNameMap() _LSSTCOMCAM = LsstComCam.getCamera().getNameMap() +_LSSTCOMCAMSIM = LsstComCamSim.getCamera().getNameMap() _DETECTOR_FROM_RS = { "LATISS": {"R00_S00": 0}, "LSSTComCam": {name: value.getId() for name, value in _LSSTCOMCAM.items()}, + "LSSTComCamSim": {name: value.getId() for name, value in _LSSTCOMCAMSIM.items()}, "LSST-TS8": {f"R22_S{x}{y}": x * 3 + y for x in range(3) for y in range(3)}, "LSSTCam": {name: value.getId() for name, value in _LSSTCAM.items()}, } From 3e8c3187f0a7d9668969c2b594841bb8cb8ec635 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Thu, 7 Mar 2024 14:11:13 -0800 Subject: [PATCH 07/13] Factor out the raw_pool making --- python/tester/upload.py | 45 ++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/python/tester/upload.py b/python/tester/upload.py index 05588d9a..c7ce91fe 100644 --- a/python/tester/upload.py +++ b/python/tester/upload.py @@ -153,6 +153,38 @@ def main(): _log.error(f"No raw files found for {instrument}, aborting.") +def _add_to_raw_pool(raw_pool, snap_num, visit, blob): + """Add a detector-snap to the raw pool for uploading. + + Parameters + ---------- + raw_pool : mapping [`str`, mapping [`int`, mapping [`activator.FannedOutVisit`, `s3.ObjectSummary`]]] + Available raws as a mapping from group IDs to a mapping of snap ID. + The value of the innermost mapping is the observation metadata for + each detector, and a Blob representing the image taken in that + detector-snap. + visit : `activator.visit.FannedOutVisit` + The visit-detector combination to be added with this raw. + snap_num : `int` + The snap number for this raw. + blob : `s3.ObjectSummary` + The raw image for this detector-snap. + """ + group = visit.groupId + if group in raw_pool: + snap_dict = raw_pool[group] + if snap_num in snap_dict: + _log.debug(f"New detector {visit.detector} added to snap {snap_num} of group {group}.") + detector_dict = snap_dict[snap_num] + detector_dict[visit] = blob + else: + _log.debug(f"New snap {snap_num} added to group {group}.") + snap_dict[snap_num] = {visit: blob} + else: + _log.debug(f"New group {group} registered.") + raw_pool[group] = {snap_num: {visit: blob}} + + def get_samples_non_lsst(bucket, instrument): """Return any predefined raw exposures for a non-LSST instrument. @@ -222,18 +254,7 @@ def get_samples_non_lsst(bucket, instrument): private_sndStamp=hsc_metadata[exp_id]["time"]-2*duration, ) _log.debug(f"File {blob.key} parsed as snap {snap_num} of visit {visit}.") - if group in result: - snap_dict = result[group] - if snap_num in snap_dict: - _log.debug(f"New detector {visit.detector} added to snap {snap_num} of group {group}.") - detector_dict = snap_dict[snap_num] - detector_dict[visit] = blob - else: - _log.debug(f"New snap {snap_num} added to group {group}.") - snap_dict[snap_num] = {visit: blob} - else: - _log.debug(f"New group {group} registered.") - result[group] = {snap_num: {visit: blob}} + _add_to_raw_pool(result, snap_num, visit, blob) return result From 5e5e2844991821c7bbff12f98a50f5c4784c57a8 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Wed, 28 Feb 2024 16:28:38 -0800 Subject: [PATCH 08/13] Reuse the raw_pool making utility for LSST cameras Previuosly only the single-detector scenario, i.e. LATISS, was considered for LSST cameras. By using _add_to_raw_pool, multiple detectors per LSST exposure are supported in the upload test for other LSST cameras. --- python/tester/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tester/upload.py b/python/tester/upload.py index c7ce91fe..58fa897a 100644 --- a/python/tester/upload.py +++ b/python/tester/upload.py @@ -319,7 +319,7 @@ def get_samples_lsst(bucket, instrument): ).unix_tai-2*duration, ) _log.debug(f"File {blob.key} parsed as visit {visit} and registered as group {md['GROUPID']}.") - result[md["GROUPID"]] = {0: {visit: blob}} + _add_to_raw_pool(result, 0, visit, blob) return result From a06bbd5c3758327dfdc0ec96de828b249ab64e73 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Wed, 28 Feb 2024 15:33:21 -0800 Subject: [PATCH 09/13] Generalize the exposure ID maker from LATISS to any LSST cemera --- python/tester/utils.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/tester/utils.py b/python/tester/utils.py index 80f3121e..95819351 100644 --- a/python/tester/utils.py +++ b/python/tester/utils.py @@ -39,7 +39,7 @@ from lsst.obs.lsst.translators.lsst import LsstBaseTranslator -from activator.raw import _LSST_CAMERA_LIST +from activator.raw import _LSST_CAMERA_LIST, _CAMERA_ABBREV _log = logging.getLogger("lsst." + __name__) _log.setLevel(logging.INFO) @@ -132,7 +132,8 @@ def make_exposure_id(instrument, group_id, snap): case "HSC": return make_hsc_id(group_id, snap) case "LATISS": - return make_latiss_id(group_id, snap) + abbrev = _CAMERA_ABBREV[instrument] + return make_lsst_id(group_id, snap, abbrev) case _: raise NotImplementedError(f"Exposure ID generation not supported for {instrument}.") @@ -175,8 +176,8 @@ def make_hsc_id(group_id, snap): return exposure_id, {"EXP-ID": f"HSCE{exposure_id:08d}"} -def make_latiss_id(group_id, snap): - """Generate an exposure ID that the Butler can parse as a valid LATISS ID. +def make_lsst_id(group_id, snap, abbrev): + """Generate an exposure ID that the Butler can parse as a valid LSST ID. Parameters ---------- @@ -184,22 +185,28 @@ def make_latiss_id(group_id, snap): The mocked group ID. snap : `int` A snap ID. + abbrev : `str` + The abbreviation of the LSST camera. Returns ------- exposure_number : An exposure ID in the format expected by Gen 3 Middleware. headers : `dict` - The key-value pairs are in the form to appear in LATISS headers. + The key-value pairs are in the form to appear in LSST headers. """ day_obs, seq_num = decode_group(group_id) exposure_num = LsstBaseTranslator.compute_exposure_id(day_obs, seq_num) - obs_id = f"AT_O_{day_obs}_{seq_num:06d}" + # Just use the default ``O`` which may be different from the original + # controller in the header. + controller = "O" + obs_id = f"{abbrev}_{controller}_{day_obs}_{seq_num:06d}" return exposure_num, { "DAYOBS": day_obs, "SEQNUM": seq_num, "OBSID": obs_id, "GROUPID": group_id, + "CONTRLLR": controller, } From 6067aba7b5fd4db38e8041e8f563073cbae9066f Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Thu, 29 Feb 2024 16:00:31 -0800 Subject: [PATCH 10/13] Make an exposure ID for any LSST camera in upload test --- python/tester/utils.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/python/tester/utils.py b/python/tester/utils.py index 95819351..4f00238f 100644 --- a/python/tester/utils.py +++ b/python/tester/utils.py @@ -128,14 +128,13 @@ def make_exposure_id(instrument, group_id, snap): The header key-value pairs to accompany with the exposure ID in the format for ``instrument``'s header. """ - match instrument: - case "HSC": - return make_hsc_id(group_id, snap) - case "LATISS": - abbrev = _CAMERA_ABBREV[instrument] - return make_lsst_id(group_id, snap, abbrev) - case _: - raise NotImplementedError(f"Exposure ID generation not supported for {instrument}.") + if instrument in _LSST_CAMERA_LIST: + abbrev = _CAMERA_ABBREV[instrument] + return make_lsst_id(group_id, snap, abbrev) + elif instrument == "HSC": + return make_hsc_id(group_id, snap) + else: + raise NotImplementedError(f"Exposure ID generation not supported for {instrument}.") def make_hsc_id(group_id, snap): From 3a00a1c016b7d30d4a88c5a3e7956391d9fb3c36 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Thu, 29 Feb 2024 11:28:17 -0800 Subject: [PATCH 11/13] Parameterize salIndex for LSST camera in the upload test --- python/tester/upload.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/python/tester/upload.py b/python/tester/upload.py index 58fa897a..e1aa24b9 100644 --- a/python/tester/upload.py +++ b/python/tester/upload.py @@ -56,12 +56,14 @@ class Instrument: n_snaps: int n_detectors: int + sal_index: int = 0 INSTRUMENTS = { - "LSSTCam": Instrument(2, 189 + 8 + 8), - "LSSTComCam": Instrument(2, 9), - "LATISS": Instrument(1, 1), + "LSSTCam": Instrument(2, 189 + 8 + 8, 1), + "LSSTComCam": Instrument(2, 9, 1), + "LSSTComCamSim": Instrument(1, 9, 3), + "LATISS": Instrument(1, 1, 2), "DECam": Instrument(1, 62), "HSC": Instrument(1, 112), } @@ -310,7 +312,7 @@ def get_samples_lsst(bucket, instrument): rotationSystem=FannedOutVisit.RotSys.SKY, cameraAngle=md["ROTPA"], survey="SURVEY", - salIndex=2, # 2 is LATISS + salIndex=INSTRUMENTS[instrument].sal_index, scriptSalIndex=2, dome=FannedOutVisit.Dome.OPEN, duration=duration, From ac94fae830a3812dd72222b902ae8a694a3ae3dc Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Thu, 29 Feb 2024 11:35:06 -0800 Subject: [PATCH 12/13] Workaround null FILTBAND in ComCam headers Currently images coming from ComCam DAQ has null FILTBAND. The camera team plans to fix it. For our simulated nextVisit messages we need something that is not null. Using FILTER is the next best thing. --- python/tester/upload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/tester/upload.py b/python/tester/upload.py index e1aa24b9..7fee9d19 100644 --- a/python/tester/upload.py +++ b/python/tester/upload.py @@ -305,7 +305,8 @@ def get_samples_lsst(bucket, instrument): detector=_DETECTOR_FROM_RS[instrument][m["raft_sensor"]], groupId=md["GROUPID"], nimages=INSTRUMENTS[instrument].n_snaps, - filters=md["FILTBAND"], + # ComCam currently sets the FILTBAND header to null. + filters=md["FILTBAND"] or md["FILTER"], coordinateSystem=FannedOutVisit.CoordSys.ICRS, position=[md["RA"], md["DEC"]], startTime=astropy.time.Time(md["DATE-BEG"], format="isot", scale="tai").unix_tai, From 13470fc1040a02bdfecfc6a70e7416891dbc78d8 Mon Sep 17 00:00:00 2001 From: Hsin-Fang Chiang Date: Mon, 4 Mar 2024 16:23:10 -0800 Subject: [PATCH 13/13] Add LSSTComCamSim upload test support in the playbook --- doc/playbook.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/playbook.rst b/doc/playbook.rst index 718e56b4..b712f15b 100644 --- a/doc/playbook.rst +++ b/doc/playbook.rst @@ -151,7 +151,7 @@ The bucket ``rubin-pp-dev`` holds incoming raw images. The bucket ``rubin-pp-dev-users`` holds: * ``rubin-pp-dev-users/central_repo/`` contains the central repository described in `DMTN-219`_. - This repository currently contains HSC and LATISS data, uploaded with ``make_export.py``. + This repository currently contains HSC, LATISS, and LSSTComCamSim data, uploaded with ``make_export.py``. * ``rubin-pp-dev-users/unobserved/`` contains raw files that the upload scripts can draw from to create incoming raws. @@ -372,7 +372,7 @@ Install the Prompt Processing code, and set it up before use: The tester scripts send ``next_visit`` events for each detector via Kafka on the ``next-visit-topic`` topic. They then upload a batch of files representing the snaps of the visit to the ``rubin-pp-dev`` S3 bucket, simulating incoming raw images. -``python/tester/upload.py``: Command line arguments are the instrument name (currently HSC or LATISS) and the number of groups of images to send. +``python/tester/upload.py``: Command line arguments are the instrument name (currently HSC, LATISS, and LSSTComCamSim) and the number of groups of images to send. Sample command line: @@ -380,6 +380,7 @@ Sample command line: python upload.py HSC 3 python upload.py LATISS 3 + python upload.py LSSTComCamSim 1 This script draws images stored in the ``rubin-pp-dev-users`` bucket. @@ -389,6 +390,7 @@ This script draws images stored in the ``rubin-pp-dev-users`` bucket. One of the files, the unobserved group `2023-10-11T01:45:47.810`, has modified RA at a location with no templates. Astrometry is also expected to fail in WCS fitting. This visit can test pipeline fallback features. +* For LSSTComCamSim, 2 groups, in total 18 raw fits files and their corresponding json metadata files, are curated. ``python/tester/upload_hsc_rc2.py``: Command line argument is the number of groups of images to send. @@ -444,6 +446,7 @@ For passwordless login, create a ``~/.pgpass`` file with contents: usdf-prompt-processing-dev.slac.stanford.edu:5432:lsst-devl:rubin:PASSWORD usdf-prompt-processing-dev.slac.stanford.edu:5432:ppcentralbutler:latiss_prompt:PASSWORD usdf-prompt-processing-dev.slac.stanford.edu:5432:ppcentralbutler:hsc_prompt:PASSWORD + usdf-prompt-processing-dev.slac.stanford.edu:5432:ppcentralbutler:lsstcomcamsim_prompt:PASSWORD and execute ``chmod 0600 ~/.pgpass``. @@ -455,6 +458,8 @@ From ``rubin-devl``, new APDB schemas can be created in the usual way: -c db_url="postgresql://rubin@usdf-prompt-processing-dev.slac.stanford.edu/lsst-devl" make_apdb.py -c namespace="pp_apdb_hsc" \ -c db_url="postgresql://rubin@usdf-prompt-processing-dev.slac.stanford.edu/lsst-devl" + make_apdb.py -c namespace="pp_apdb_lsstcomcamsim" \ + -c db_url="postgresql://rubin@usdf-prompt-processing-dev.slac.stanford.edu/lsst-devl" Resetting the APDB ------------------ @@ -472,3 +477,9 @@ To restore the APDB to a clean state, run the following: psql -h usdf-prompt-processing-dev.slac.stanford.edu lsst-devl rubin -c 'drop schema "pp_apdb_hsc" cascade;' make_apdb.py -c namespace="pp_apdb_hsc" \ -c db_url="postgresql://rubin@usdf-prompt-processing-dev.slac.stanford.edu/lsst-devl" + +.. code-block:: sh + + psql -h usdf-prompt-processing-dev.slac.stanford.edu lsst-devl rubin -c 'drop schema "pp_apdb_lsstcomcamsim" cascade;' + make_apdb.py -c namespace="pp_apdb_lsstcomcamsim" \ + -c db_url="postgresql://rubin@usdf-prompt-processing-dev.slac.stanford.edu/lsst-devl"