From 9dd54945ec090f61c7113caded45cdbd5c927b69 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Thu, 24 Apr 2025 15:41:51 -0700 Subject: [PATCH 1/2] Fix definitions of SingleFrame pipelines. The pipelines included analyzeCalibrationMetrics (unknown label?) but did not include associateSolarSystemDirectSource. Making the pipeline definition opt-out instead of opt-in should keep it closer to its intended meaning of "ap_pipe/SingleFrame, without Preprocessing". --- pipelines/HSC/SingleFrame.yaml | 9 ++++++--- pipelines/LATISS/SingleFrame.yaml | 9 ++++++--- pipelines/LSSTCam-imSim/SingleFrame.yaml | 9 ++++++--- pipelines/LSSTCam/SingleFrame.yaml | 9 ++++++--- pipelines/LSSTComCam/SingleFrame.yaml | 9 ++++++--- pipelines/LSSTComCamSim/SingleFrame.yaml | 9 ++++++--- 6 files changed, 36 insertions(+), 18 deletions(-) diff --git a/pipelines/HSC/SingleFrame.yaml b/pipelines/HSC/SingleFrame.yaml index 2822564b..10cc5674 100644 --- a/pipelines/HSC/SingleFrame.yaml +++ b/pipelines/HSC/SingleFrame.yaml @@ -1,9 +1,12 @@ description: >- Single-frame pipeline for the case in which no templates exist, specialized for HSC + Unlike ap_pipe/SingleFrame.yaml, this pipeline is not standalone. imports: - location: $AP_PIPE_DIR/pipelines/HSC/SingleFrame.yaml - include: - - apPipeSingleFrame - - analyzeCalibrationMetrics + exclude: + # Run in prep_butler or Preprocessing.yaml + - getRegionTimeFromVisit + - mpSkyEphemerisQuery + - loadDiaCatalogs diff --git a/pipelines/LATISS/SingleFrame.yaml b/pipelines/LATISS/SingleFrame.yaml index 1edda16d..24b94d9d 100644 --- a/pipelines/LATISS/SingleFrame.yaml +++ b/pipelines/LATISS/SingleFrame.yaml @@ -1,9 +1,12 @@ description: >- Single-frame pipeline for the case in which no templates exist, specialized for LATISS + Unlike ap_pipe/SingleFrame.yaml, this pipeline is not standalone. imports: - location: $AP_PIPE_DIR/pipelines/LATISS/SingleFrame.yaml - include: - - apPipeSingleFrame - - analyzeCalibrationMetrics + exclude: + # Run in prep_butler or Preprocessing.yaml + - getRegionTimeFromVisit + - mpSkyEphemerisQuery + - loadDiaCatalogs diff --git a/pipelines/LSSTCam-imSim/SingleFrame.yaml b/pipelines/LSSTCam-imSim/SingleFrame.yaml index 2cba3a0e..c809e71b 100644 --- a/pipelines/LSSTCam-imSim/SingleFrame.yaml +++ b/pipelines/LSSTCam-imSim/SingleFrame.yaml @@ -1,9 +1,12 @@ description: >- Single-frame pipeline for the case in which no templates exist, specialized for LSSTCam-imSim + Unlike ap_pipe/SingleFrame.yaml, this pipeline is not standalone. imports: - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/SingleFrame.yaml - include: - - apPipeSingleFrame - - analyzeCalibrationMetrics + exclude: + # Run in prep_butler or Preprocessing.yaml + - getRegionTimeFromVisit + - mpSkyEphemerisQuery + - loadDiaCatalogs diff --git a/pipelines/LSSTCam/SingleFrame.yaml b/pipelines/LSSTCam/SingleFrame.yaml index 8080dfc7..840853d4 100644 --- a/pipelines/LSSTCam/SingleFrame.yaml +++ b/pipelines/LSSTCam/SingleFrame.yaml @@ -1,9 +1,12 @@ description: >- Single-frame pipeline for the case in which no templates exist, specialized for LSSTCam + Unlike ap_pipe/SingleFrame.yaml, this pipeline is not standalone. imports: - location: $AP_PIPE_DIR/pipelines/LSSTCam/SingleFrame.yaml - include: - - apPipeSingleFrame - - analyzeCalibrationMetrics + exclude: + # Run in prep_butler or Preprocessing.yaml + - getRegionTimeFromVisit + - mpSkyEphemerisQuery + - loadDiaCatalogs diff --git a/pipelines/LSSTComCam/SingleFrame.yaml b/pipelines/LSSTComCam/SingleFrame.yaml index 5b61f4a3..d0a4fe71 100644 --- a/pipelines/LSSTComCam/SingleFrame.yaml +++ b/pipelines/LSSTComCam/SingleFrame.yaml @@ -1,9 +1,12 @@ description: >- Single-frame pipeline for the case in which no templates exist, specialized for LSSTComCam + Unlike ap_pipe/SingleFrame.yaml, this pipeline is not standalone. imports: - location: $AP_PIPE_DIR/pipelines/LSSTComCam/SingleFrame.yaml - include: - - apPipeSingleFrame - - analyzeCalibrationMetrics + exclude: + # Run in prep_butler or Preprocessing.yaml + - getRegionTimeFromVisit + - mpSkyEphemerisQuery + - loadDiaCatalogs diff --git a/pipelines/LSSTComCamSim/SingleFrame.yaml b/pipelines/LSSTComCamSim/SingleFrame.yaml index 38b2ae6a..fa6d65db 100644 --- a/pipelines/LSSTComCamSim/SingleFrame.yaml +++ b/pipelines/LSSTComCamSim/SingleFrame.yaml @@ -1,9 +1,12 @@ description: >- Single-frame pipeline for the case in which no templates exist, specialized for LSSTComCamSim + Unlike ap_pipe/SingleFrame.yaml, this pipeline is not standalone. imports: - location: $AP_PIPE_DIR/pipelines/LSSTComCamSim/SingleFrame.yaml - include: - - apPipeSingleFrame - - analyzeCalibrationMetrics + exclude: + # Run in prep_butler or Preprocessing.yaml + - getRegionTimeFromVisit + - mpSkyEphemerisQuery + - loadDiaCatalogs From e11e3194ae0a66559c4e345f9c644d608a512403 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Thu, 24 Apr 2025 16:15:31 -0700 Subject: [PATCH 2/2] Add row to dataset type instructions. SingleFrame.yaml has tasks not found in ApPipe.yaml, so it must be updated separately. --- doc/playbook.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/playbook.rst b/doc/playbook.rst index 21f68fdf..88abf119 100644 --- a/doc/playbook.rst +++ b/doc/playbook.rst @@ -349,6 +349,7 @@ One raw was ingested, visit-defined, and kept in the development central repo, s apdb-cli create-sql "sqlite:///apdb.db" apdb_config.yaml pipetask run -b s3://rubin-pp-dev-users/central_repo_2 -i LSSTComCamSim/raw/all,LSSTComCamSim/defaults,LSSTComCamSim/templates -o u/${USER}/add-dataset-types -d "instrument='LSSTComCamSim' and exposure=7024062700235 and detector=8" -p $AP_PIPE_DIR/pipelines/LSSTComCamSim/ApPipe.yaml -c parameters:apdb_config=apdb_config.yaml -c associateApdb:doPackageAlerts=False --register-dataset-types --init-only + pipetask run -b s3://rubin-pp-dev-users/central_repo_2 -i LSSTComCamSim/raw/all,LSSTComCamSim/defaults,LSSTComCamSim/templates -o u/${USER}/add-dataset-types -d "instrument='LSSTComCamSim' and exposure=7024062700235 and detector=8" -p $AP_PIPE_DIR/pipelines/LSSTComCamSim/SingleFrame.yaml -c parameters:apdb_config=apdb_config.yaml --register-dataset-types --init-only .. note::