diff --git a/bin.src/make_preloaded_export.py b/bin.src/make_preloaded_export.py new file mode 100755 index 00000000..62c173e4 --- /dev/null +++ b/bin.src/make_preloaded_export.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# This file is part of prompt_prototype. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +"""Selectively export the contents of an ap_verify dataset. + +This script selects the subset of an ap_verify dataset's preloaded repository that +matches what the central prompt processing repository ought to look like. +""" + + +import argparse +import logging +import os +import re +import sys +import time + +import lsst.daf.butler as daf_butler + + +def _make_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--src-repo", required=True, + help="The location of the repository to be exported.") + return parser + + +def main(): + logging.basicConfig(level=logging.INFO, stream=sys.stdout) + + args = _make_parser().parse_args() + gen3_repo = os.path.abspath(args.src_repo) + + logging.info("Exporting Gen 3 registry to configure new repos...") + start = time.time_ns() + _export_for_copy(gen3_repo) + end = time.time_ns() + logging.info("Export finished in %.3fs.", 1e-9 * (end - start)) + + +def _get_dataset_types(): + """Identify the dataset types that should be marked for export. + + Returns + ------- + types : iterable [`str` or `re.Pattern`] + The dataset types to include + """ + # Everything except raws and SS ephemerides + return [re.compile("^(?!raw|visitSsObjects).*")] + + +def _export_for_copy(repo): + """Export a Gen 3 repository so that a dataset can make copies later. + + Parameters + ---------- + repo : `str` + The location of the Gen 3 repository. + """ + butler = daf_butler.Butler(repo) + with butler.export(format="yaml") as contents: + # Need all detectors, even those without data, for visit definition + contents.saveDataIds(butler.registry.queryDataIds({"detector"}).expanded()) + contents.saveDatasets(butler.registry.queryDatasets( + datasetType=_get_dataset_types(), collections=...)) + # Save calibration collection + for collection in butler.registry.queryCollections( + collectionTypes=daf_butler.CollectionType.CALIBRATION): + contents.saveCollection(collection) + # Do not export chains, as they will need to be reworked to satisfy + # prompt processing's assumptions. + + +if __name__ == "__main__": + main() diff --git a/bin.src/make_remote_butler.py b/bin.src/make_remote_butler.py new file mode 100755 index 00000000..594f4077 --- /dev/null +++ b/bin.src/make_remote_butler.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# This file is part of prompt_prototype. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +"""Simple script for creating a repository at a remote URI, given +a source repository and export file. + +For most values of --target-repo and --seed-config, this script is only useful +if run from the prompt-proto project on Google Cloud. + +The user is responsible for clearing any old copies of the repository from +both the target URI and the registry database. +""" + + +import argparse +import logging +import os +import sys + +from lsst.utils import getPackageDir +from lsst.utils.timer import time_this +from lsst.daf.butler import Butler, CollectionType, Config +from lsst.obs.base import Instrument + + +def _make_parser(): + parser = argparse.ArgumentParser() + # Could reasonably be positional arguments, but keep them as keywords to + # prevent users from confusing --src-repo with --target-repo. + parser.add_argument("--src-repo", required=True, + help="The location of the repository whose files are to be copied.") + parser.add_argument("--target-repo", required=True, + help="The URI of the repository to create.") + parser.add_argument("--seed-config", + default=os.path.join(getPackageDir("prompt_prototype"), "etc", "db_butler.yaml"), + help="The config file to use for the new repository. Defaults to etc/db_butler.yaml.") + parser.add_argument("--export-file", default="export.yaml", + help="The export file containing the repository contents. Defaults to ./export.yaml.") + return parser + + +def _add_chains(butler): + """Create collections to serve as a uniform interface. + + Parameters + ---------- + butler : `lsst.daf.butler.Butler` + A Butler pointing to the repository to modify. Assumed to already contain the following collections: + + - standard calibration collection + - standard skymap collection + - templates/* + - refcats/* + """ + butler.registry.registerCollection("templates", type=CollectionType.CHAINED) + butler.registry.setCollectionChain( + "templates", + list(butler.registry.queryCollections("templates/*", collectionTypes=CollectionType.RUN)) + ) + + butler.registry.registerCollection("refcats", type=CollectionType.CHAINED) + butler.registry.setCollectionChain( + "refcats", + list(butler.registry.queryCollections("refcats/*", collectionTypes=CollectionType.RUN)) + ) + + instrument = Instrument.fromName(list(butler.registry.queryDataIds("instrument"))[0]["instrument"], + butler.registry) + defaults = instrument.makeCollectionName("defaults") + butler.registry.registerCollection(defaults, type=CollectionType.CHAINED) + butler.registry.setCollectionChain( + defaults, + [instrument.makeCalibrationCollectionName(), "templates", "skymaps", "refcats"] + ) + + +def main(): + logging.basicConfig(level=logging.INFO, stream=sys.stdout) + + args = _make_parser().parse_args() + seed_config = Config(args.seed_config) + logging.info("Creating repository at %s...", args.target_repo) + with time_this(msg="Repository creation", level=logging.INFO): + config = Butler.makeRepo(args.target_repo, config=seed_config, overwrite=False) + with time_this(msg="Butler creation", level=logging.INFO): + butler = Butler(config, writeable=True) + with time_this(msg="Import", level=logging.INFO): + butler.import_(directory=args.src_repo, filename=args.export_file, transfer="auto") + _add_chains(butler) + + +if __name__ == "__main__": + main() diff --git a/doc/playbook.rst b/doc/playbook.rst index dac12c06..37d0ab0a 100644 --- a/doc/playbook.rst +++ b/doc/playbook.rst @@ -2,6 +2,8 @@ Playbook for the Prompt Processing Proposal and Prototype ######################################################### +.. _DMTN-219: https://dmtn-219.lsst.io/ + Table of Contents ================= @@ -48,7 +50,7 @@ Pub/Sub Topics ============== One Google Pub/Sub topic is used for ``nextVisit`` events. -Additional topics are used for images from each instrument, where the instrument is one of ``LSSTCam``, ``LSSTComCam``, ``LATISS``, or ``DECam``. +Additional topics are used for images from each instrument, where the instrument is one of ``LSSTCam``, ``LSSTComCam``, ``LATISS``, ``DECam``, or ``HSC``. To create the topic, in the Google Cloud Console for the ``prompt-proto`` project: @@ -65,11 +67,14 @@ On the other hand, using multiple topics is also simple to do. Buckets ======= -A single bucket named ``rubin-prompt-proto-main`` has been created to hold incoming raw images. +A single bucket named ``rubin-prompt-proto-main`` has been created to hold the central repository described in `DMTN-219`_, as well as incoming raw images. + +The bucket ``rubin-prompt-proto-support-data-template`` contains a pristine copy of the calibration datasets and templates. +This bucket is not intended for direct use by the prototype, but can be used to restore the central repository to its state at the start of an observing run. -An additional bucket will be needed eventually to hold a Butler repo containing calibration datasets and templates. +The bucket ``rubin-prompt-proto-unobserved`` contains raw files that the upload script(s) can draw from to create incoming raws for ``rubin-prompt-proto-main``. -The raw image bucket has had notifications configured for it; these publish to a Google Pub/Sub topic as mentioned in the previous section. +The ``-main`` bucket has had notifications configured for it; these publish to a Google Pub/Sub topic as mentioned in the previous section. To configure these notifications, in a shell: .. code-block:: sh diff --git a/etc/db_butler.yaml b/etc/db_butler.yaml new file mode 100644 index 00000000..1be7363b --- /dev/null +++ b/etc/db_butler.yaml @@ -0,0 +1,3 @@ +registry: + db: postgresql://postgres@localhost:5432/ + namespace: support_data_template diff --git a/python/tester/upload.py b/python/tester/upload.py index a7e9d42a..c22241eb 100644 --- a/python/tester/upload.py +++ b/python/tester/upload.py @@ -20,6 +20,7 @@ class Instrument: "LSSTComCam": Instrument(2, 9), "LATISS": Instrument(1, 1), "DECam": Instrument(1, 62), + "HSC": Instrument(1, 112), } EXPOSURE_INTERVAL = 18 SLEW_INTERVAL = 2 diff --git a/ups/prompt_prototype.table b/ups/prompt_prototype.table index 26064978..93419c54 100644 --- a/ups/prompt_prototype.table +++ b/ups/prompt_prototype.table @@ -4,6 +4,7 @@ # the "base" package. setupRequired(base) setupRequired(sconsUtils) +setupRequired(utils) # Used by scripts in bin.src setupRequired(daf_butler) # Used by middleware_interface module. setupRequired(obs_base) setupRequired(obs_lsst)