Merge pull request #17 from lsst/tickets/DM-26629

DM- 26629: switch to calibration collections instead of the calibration_label dimension

.gitattributes

@@ -4,5 +4,4 @@
 *.sqlite3 filter=lfs diff=lfs merge=lfs -text
 
 # These look like fits files, but are really symbolic links
-preloaded/calib/LSST-ImSim/bias !filter !diff !merge
-preloaded/calib/LSST-ImSim/flat !filter !diff !merge
+preloaded/calib/LSST-ImSim/gen2 !filter !diff !merge

README.md

@@ -3,9 +3,13 @@
 
 Data from `testdata_lsst` for small-scale tests of [`ap_verify`](https://github.com/lsst-dm/ap_verify/) functionality.
 
-The data are identical to the files provided by [`testdata_lsst`](https://github.com/lsst/testdata_lsst/).
+The data were originally identical to the files provided by [`testdata_lsst`](https://github.com/lsst/testdata_lsst/).
 See that package for documentation of the file contents.
 
+However, [DM-26138](https://jira.lsstcorp.org/browse/DM-26138) updated the filter names used for LSST imSim data to include the throughputs version (e.g. "i" -> "i_sim_1.4").
+This affects the headers of both raw and master calibration files, as well as their usual filenames.
+These updates have been applied to the files in `ap_verify_testdata`, but not (yet) to the files in `testdata_lsst`.
+
 Relevant Files and Directories
 ------------------------------
 path                  | description

config/export.yaml

@@ -9,21 +9,6 @@ data:
     exposure_max: 9999999999
     detector_max: 999
     class_name: lsst.obs.lsst.LsstImSim
-- type: dimension
-  element: calibration_label
-  records:
-  - instrument: LSSTCam-imSim
-    name: unbounded
-    datetime_begin: null
-    datetime_end: null
-  - instrument: LSSTCam-imSim
-    name: gen2/bias_2022-01-01_037
-    datetime_begin: !butler_time/tai/iso '1994-08-17 00:00:00.000000000'
-    datetime_end: !butler_time/tai/iso '2049-05-19 00:00:00.000000000'
-  - instrument: LSSTCam-imSim
-    name: gen2/flat_2022-08-06_037_i_sim_1.4
-    datetime_begin: !butler_time/tai/iso '1995-03-22 00:00:00.000000000'
-    datetime_end: !butler_time/tai/iso '2049-12-22 00:00:00.000000000'
 - type: dimension
   element: detector
   records:
@@ -1167,77 +1152,94 @@ data:
   - instrument: LSSTCam-imSim
     name: i_sim_1.4
     band: i
+- type: collection
+  collection_type: CALIBRATION
+  name: LSSTCam-imSim/calib
+- type: collection
+  collection_type: RUN
+  name: LSSTCam-imSim/calib/gen2/2022-01-01
+  host: null
+  timespan_begin: null
+  timespan_end: null
+- type: collection
+  collection_type: RUN
+  name: LSSTCam-imSim/calib/gen2/2022-08-06
+  host: null
+  timespan_begin: null
+  timespan_end: null
+- type: collection
+  collection_type: RUN
+  name: LSSTCam-imSim/calib/unbounded
+  host: null
+  timespan_begin: null
+  timespan_end: null
+- type: collection
+  collection_type: RUN
+  name: refcats
+  host: null
+  timespan_begin: null
+  timespan_end: null
 - type: dataset_type
   name: camera
   dimensions:
   - instrument
-  - calibration_label
   storage_class: Camera
-- type: run
-  name: LSSTCam-imSim/calib
+  is_calibration: true
 - type: dataset
   dataset_type: camera
-  run: LSSTCam-imSim/calib
+  run: LSSTCam-imSim/calib/unbounded
   records:
   - dataset_id:
     - 1
     data_id:
     - instrument: LSSTCam-imSim
-      calibration_label: unbounded
-    path: LSSTCam-imSim/calib/camera/camera_unbounded_LSSTCam-imSim_LSSTCam-imSim_calib.fits
+    path: LSSTCam-imSim/calib/unbounded/camera/camera_LSSTCam-imSim_LSSTCam-imSim_calib_unbounded.fits
     formatter: lsst.obs.base.formatters.fitsGeneric.FitsGenericFormatter
 - type: dataset_type
   name: bias
   dimensions:
   - instrument
-  - calibration_label
   - detector
   storage_class: ExposureF
-- type: run
-  name: LSSTCam-imSim/calib
+  is_calibration: true
 - type: dataset
   dataset_type: bias
-  run: LSSTCam-imSim/calib
+  run: LSSTCam-imSim/calib/gen2/2022-01-01
   records:
   - dataset_id:
     - 2
     data_id:
     - instrument: LSSTCam-imSim
-      calibration_label: gen2/bias_2022-01-01_037
       detector: 37
-    path: LSSTCam-imSim/calib/bias/bias_gen2_bias_2022-01-01_037_37_LSSTCam-imSim_LSSTCam-imSim_calib.fits
+    path: LSSTCam-imSim/calib/gen2/2022-01-01/bias/bias_37_LSSTCam-imSim_LSSTCam-imSim_calib_gen2_2022-01-01.fits
     formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
 - type: dataset_type
   name: flat
   dimensions:
   - band
   - instrument
-  - calibration_label
   - detector
   - physical_filter
   storage_class: ExposureF
-- type: run
-  name: LSSTCam-imSim/calib
+  is_calibration: true
 - type: dataset
   dataset_type: flat
-  run: LSSTCam-imSim/calib
+  run: LSSTCam-imSim/calib/gen2/2022-08-06
   records:
   - dataset_id:
     - 3
     data_id:
     - instrument: LSSTCam-imSim
-      calibration_label: gen2/flat_2022-08-06_037_i_sim_1.4
       detector: 37
       physical_filter: i_sim_1.4
-    path: LSSTCam-imSim/calib/flat/i/i_sim_1.4/flat_i_i_sim_1_4_gen2_flat_2022-08-06_037_i_sim_1_4_37_LSSTCam-imSim_LSSTCam-imSim_calib.fits
+    path: LSSTCam-imSim/calib/gen2/2022-08-06/flat/i/i_sim_1.4/flat_i_i_sim_1_4_37_LSSTCam-imSim_LSSTCam-imSim_calib_gen2_2022-08-06.fits
     formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
 - type: dataset_type
   name: gaia
   dimensions:
   - htm7
   storage_class: SimpleCatalog
-- type: run
-  name: refcats
+  is_calibration: false
 - type: dataset
   dataset_type: gaia
   run: refcats
@@ -1284,3 +1286,19 @@ data:
     - htm7: 232344
     path: refcats/gaia/gaia_232344_refcats.fits
     formatter: lsst.obs.base.formatters.fitsGeneric.FitsGenericFormatter
+- type: associations
+  collection: LSSTCam-imSim/calib
+  collection_type: CALIBRATION
+  validity_ranges:
+  - begin: null
+    end: null
+    dataset_ids:
+    - 1
+  - begin: !butler_time/tai/iso '1994-08-17 00:00:00.000000000'
+    end: !butler_time/tai/iso '2049-05-19 00:00:00.000000000'
+    dataset_ids:
+    - 2
+  - begin: !butler_time/tai/iso '1995-03-22 00:00:00.000000000'
+    end: !butler_time/tai/iso '2049-12-22 00:00:00.000000000'
+    dataset_ids:
+    - 3

preloaded/LSSTCam-imSim/calib/gen2/2022-01-01/bias/bias_37_LSSTCam-imSim_LSSTCam-imSim_calib_gen2_2022-01-01.fits

@@ -0,0 +1 @@
+../../../../../../calib/bias-R11-S01-det037_2022-01-01.fits.gz

preloaded/LSSTCam-imSim/calib/gen2/2022-08-06/flat/i/i_sim_1.4/flat_i_i_sim_1_4_37_LSSTCam-imSim_LSSTCam-imSim_calib_gen2_2022-08-06.fits

@@ -0,0 +1 @@
+../../../../../../../../calib/flat_i-R11-S01-det037_2022-08-06.fits.gz

scripts/add_gen3_repo.py

@@ -117,6 +117,12 @@ def _export_for_copy(dataset, repo):
         # Need all detectors, even those without data, for visit definition
         contents.saveDataIds(butler.registry.queryDataIds({"detector"}).expanded())
         contents.saveDatasets(butler.registry.queryDatasets(datasetType=..., collections=...))
+        # Explicitly save the calibration collection.
+        # Do _not_ include the RUN collections here because that will export
+        # an empty raws collection, which ap_verify assumes does not exist
+        # before ingest.
+        for collection in butler.registry.queryCollections(..., collectionTypes={daf_butler.CollectionType.CALIBRATION}):
+            contents.saveCollection(collection)
 
 
 if __name__ == "__main__":