DM-23231: Change everywhere to use expId rather than visit+snap

config/ingest.py

@@ -45,6 +45,7 @@
     'controller': 'translate_controller',
     'obsid': 'translate_obsid',
     'expGroup': 'translate_expGroup',
+    'expId': 'translate_expId',
 }
 config.parse.defaults = {
 }
@@ -69,6 +70,7 @@
     'controller': 'text',
     'obsid': 'text',
     'expGroup': 'text',
+    'expId': 'int',
 }
-config.register.unique = ["visit", "detector"]
+config.register.unique = ["expId", "detector", "visit"]
 config.register.visit = ['visit', 'filter', 'dateObs', 'expTime']

config/latiss/ingest.py

@@ -50,6 +50,9 @@
     'object': 'text',
     'imageType': 'text',
     'wavelength': 'int',
+    'expGroup': 'text',
+    'obsid': 'text',
+    'expId': 'int'
 }
-config.register.unique = ["visit", "detector"]
+config.register.unique = ["expId", "visit"]
 config.register.visit = list(config.register.columns.keys())

config/ts3/ingest.py

@@ -38,5 +38,5 @@
 config.register.columns['dayObs'] = 'text'
 config.register.columns['testSeqNum'] = 'int'
 
-config.register.unique = ["visit", "detector"]
+config.register.unique = ["expId", "detector", "visit"]
 config.register.visit = list(config.register.columns.keys())

config/ts8/ingest.py

@@ -38,5 +38,5 @@
 config.register.columns['dayObs'] = 'text'
 config.register.columns['testSeqNum'] = 'int'
 
-config.register.unique = ["visit", "detector"]
+config.register.unique = ["expId", "detector", "visit"]
 config.register.visit = list(config.register.columns.keys())

config/ucd/ingest.py

@@ -17,5 +17,5 @@
 config.register.columns['dayObs'] = 'text'
 config.register.columns['testSeqNum'] = 'int'
 
-config.register.unique = ["visit", "detector"]
+config.register.unique = ["expId", "detector", "visit"]
 config.register.visit = list(config.register.columns.keys())

policy/.gitignore

@@ -1,8 +1,8 @@
-latiss.yaml
-imsim.yaml
-lsstCam.yaml
-phosim.yaml
-ts8*.yaml
-ucd.yaml
-ts3.yaml
-comCam.yaml
+/latiss.yaml
+/imsim.yaml
+/lsstCam.yaml
+/phosim.yaml
+/ts8.yaml
+/ucd.yaml
+/ts3.yaml
+/comCam.yaml

policy/latiss/latissMapper.yaml

@@ -25,20 +25,20 @@ calibrations:
 
 datasets:
   raw_hdu:
-    template: raw/%(dayObs)s/%(visit)08d-det%(detector)03d.fits[0]
+    template: raw/%(dayObs)s/%(expId)08d-det%(detector)03d.fits[0]
   apPipe_metadata:
-    template: apPipe_metadata/%(visit)08d-%(filter)s/apPipeMetadata_%(visit)08d-%(filter)s-%(detectorName)s-det%(detector)03d.yaml
+    template: apPipe_metadata/%(expId)08d-%(filter)s/apPipeMetadata_%(expId)08d-%(filter)s-%(detectorName)s-det%(detector)03d.yaml
   runIsr_metadata:
-    template: runIsr_metadata/%(visit)08d-%(filter)s/runIsrMetadata_%(visit)08d-%(filter)s-%(detectorName)s-det%(detector)03d.yaml
+    template: runIsr_metadata/%(expId)08d-%(filter)s/runIsrMetadata_%(expId)08d-%(filter)s-%(detectorName)s-det%(detector)03d.yaml
   brighterFatterKernel:
     template: calibrations/bfKernel-%(detectorName)s-det%(detector)03d.pkl
 
 exposures:
   _raw:
-    template: raw/%(dayObs)s/%(visit)08d-det%(detector)03d.fits[0]
+    template: raw/%(dayObs)s/%(expId)08d-det%(detector)03d.fits[0]
   raw_amp:
-    template: raw/%(dayObs)s/%(visit)08d-det%(detector)03d.fits[%(channel)d]
+    template: raw/%(dayObs)s/%(expId)08d-det%(detector)03d.fits[%(channel)d]
   postISRCCD:
-    template: postISRCCD/%(dayObs)s/postISRCCD_%(visit)08d-%(filter)s-det%(detector)03d.fits
+    template: postISRCCD/%(dayObs)s/postISRCCD_%(expId)08d-%(filter)s-det%(detector)03d.fits
   calexp:
-    template: calexp/%(dayObs)s/calexp_%(visit)08d-%(filter)s-det%(detector)03d.fits
+    template: calexp/%(dayObs)s/calexp_%(expId)08d-%(filter)s-det%(detector)03d.fits

policy/ts3/ts3Mapper.yaml

@@ -10,16 +10,16 @@ calibrations:
 
 exposures:
   _raw:
-    template: raw/%(run)s/%(visit)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[0]
+    template: raw/%(run)s/%(expId)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[0]
   raw_amp:
-    template: raw/%(run)s/%(visit)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[%(channel)d]
+    template: raw/%(run)s/%(expId)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[%(channel)d]
   postISRCCD:
-    template: postISRCCD/postISRCCD_%(visit)011d-%(filter)s-det%(detector)03d.fits
+    template: postISRCCD/postISRCCD_%(expId)011d-%(filter)s-det%(detector)03d.fits
   calexp:
-    template: calexp/calexp_%(visit)011d-%(filter)s-det%(detector)03d.fits
+    template: calexp/calexp_%(expId)011d-%(filter)s-det%(detector)03d.fits
 
 datasets:
   eotest_gain:  # set of amplifier gains, as measured by eotest
     template: gain/%(run)s/gain_%(run)s-det%(detector)03d.pickle
   raw_hdu:
-    template: raw/%(run)s/%(visit)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[0]
+    template: raw/%(run)s/%(expId)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[0]

policy/ts8/ts8Mapper.yaml

@@ -10,16 +10,16 @@ calibrations:
 
 exposures:
   _raw:
-    template: raw/%(run)s/%(visit)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[0]
+    template: raw/%(run)s/%(expId)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[0]
   raw_amp:
-    template: raw/%(run)s/%(visit)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[%(channel)d]
+    template: raw/%(run)s/%(expId)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[%(channel)d]
   postISRCCD:
-    template: postISRCCD/postISRCCD_%(visit)011d-%(filter)s-det%(detector)03d.fits
+    template: postISRCCD/postISRCCD_%(expId)011d-%(filter)s-det%(detector)03d.fits
   calexp:
-    template: calexp/calexp_%(visit)011d-%(filter)s-det%(detector)03d.fits
+    template: calexp/calexp_%(expId)011d-%(filter)s-det%(detector)03d.fits
 
 datasets:
   eotest_gain:  # set of amplifier gains, as measured by eotest
     template: gain/%(run)s/gain_%(run)s-det%(detector)03d.pickle
   raw_hdu:
-    template: raw/%(run)s/%(visit)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[0]
+    template: raw/%(run)s/%(expId)011d-%(raftName)s-%(detectorName)s-det%(detector)03d.fits[0]

policy/ucd/ucdMapper.yaml

@@ -10,16 +10,16 @@ calibrations:
 
 exposures:
   _raw:
-    template: raw/%(run)s/%(visit)011d-%(detectorName)s-det%(detector)03d.fits[0]
+    template: raw/%(run)s/%(expId)011d-%(detectorName)s-det%(detector)03d.fits[0]
   raw_amp:
-    template: raw/%(run)s/%(visit)011d-%(detectorName)s-det%(detector)03d.fits[%(channel)d]
+    template: raw/%(run)s/%(expId)011d-%(detectorName)s-det%(detector)03d.fits[%(channel)d]
   postISRCCD:
-    template: postISRCCD/postISRCCD_%(visit)011d-%(filter)s-det%(detector)03d.fits
+    template: postISRCCD/postISRCCD_%(expId)011d-%(filter)s-det%(detector)03d.fits
   calexp:
-    template: calexp/calexp_%(visit)011d-%(filter)s-det%(detector)03d.fits
+    template: calexp/calexp_%(expId)011d-%(filter)s-det%(detector)03d.fits
 
 datasets:
   eotest_gain:  # set of amplifier gains, as measured by eotest
     template: gain/%(run)s/gain_%(run)s-det%(detector)03d.pickle
   raw_hdu:
-    template: raw/%(run)s/%(visit)011d-%(detectorName)s-det%(detector)03d.fits[0]
+    template: raw/%(run)s/%(expId)011d-%(detectorName)s-det%(detector)03d.fits[0]

python/lsst/obs/lsst/ingest.py

@@ -280,6 +280,9 @@ def translate_obsid(self, md):
     def translate_expGroup(self, md):
         return self.observationInfo.exposure_group
 
+    def translate_expId(self, md):
+        return self.observationInfo.exposure_id
+
     def translate_controller(self, md):
         if "CONTRLLR" in md:
             if md["CONTRLLR"]:

python/lsst/obs/lsst/script/phosimToRafts.py

@@ -94,7 +94,8 @@ def processPhosimData(ids, visit, inputDir, outputDir):
     ----------
     ids : `str`
         DataId to be used to access the data. Can only be given as
-        ``visit=NNN`` form. ``visit`` is used if this is `None`
+        ``visit=NNN`` form. ``visit`` is used if this is `None`.
+        ``expId`` is assumed to be an alias for ``visit`` in PhoSim.
     visit : `int`
         Explicit visit number to read from repository. Only used if
         ``ids`` is `None` or to check that it is consistent with the value
@@ -141,7 +142,8 @@ def processPhosimData(ids, visit, inputDir, outputDir):
             raise RuntimeError(f"No raw data found in butler repository at {inputDir}")
         visit = rawData[0]
 
-    dataId = dict(visit=visit)
+    # For phosim we declare that visit==expId
+    dataId = dict(expId=visit)
     #
     # Due to butler stupidity it can't/won't lookup things when you also
     # specify a channel.  Sigh

tests/bin/reingest.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Reingest all the raw files.
+# Run this if there have been registry or translator changes.
+
+export DYLD_LIBRARY_PATH=${LSST_LIBRARY_PATH}
+
+set -e
+DATADIR=${OBS_LSST_DIR}/data/input
+cd ${DATADIR}
+
+for instrument in *; do
+  echo
+  echo $instrument
+
+  rm "${instrument}/registry.sqlite3"
+  mv "${instrument}/raw" "${instrument}/old"
+
+  ingest_files=$(find "${instrument}/old" -type f)
+  echo ingestImages.py "${instrument}" --mode=copy ${ingest_files}
+  ingestImages.py "${instrument}" --mode=copy ${ingest_files}
+  rm -rf ${instrument}/old
+done

tests/headers/latiss-AT_O_20200128_000379.yaml

@@ -0,0 +1,95 @@
+SIMPLE: true
+EXTEND: true
+COMMENT: '---- Checksums ----'
+ORIGIN: LSST DM Header Service
+TIMESYS: TAI
+DATE: '2020-01-29T07:26:00.522'
+DATE-OBS: '2020-01-29T07:25:52.373'
+DATE-BEG: '2020-01-29T07:25:52.373'
+DATE-END: '2020-01-29T07:26:37.217'
+MJD: 58877.3097282593
+MJD-OBS: 58877.3096339468
+MJD-BEG: 58877.3096339468
+MJD-END: 58877.3101529745
+OBSID: AT_O_20200128_000379
+GROUPID: '2020-01-29T07:25:52.166'
+IMGTYPE: OBJECT
+BUNIT: adu
+TELESCOP: LSST AuxTelescope
+INSTRUME: LATISS
+OBSERVER: LSST
+OBS-LONG: -70.749417
+OBS-LAT: -30.244639
+OBS-ELEV: 2663.0
+OBSGEO-X: 1818938.94
+OBSGEO-Y: -5208470.95
+OBSGEO-Z: -3195172.08
+RA: 185.705958333333
+DEC: -57.6761388888889
+RASTART: 343.801645144617
+DECSTART: -76.9544144126301
+RAEND: 343.842366953198
+DECEND: -76.9697483864031
+ROTPA: 165.248859104852
+ROTCOORD: sky
+HASTART: null
+ELSTART: 60.0244513734215
+AZSTART: 161.246452708219
+AMSTART: null
+HAEND: null
+ELEND: 60.0337090178364
+AZEND: 161.277061630462
+AMEND: null
+INSTPORT: 2
+ATM3PORT: 1
+TRACKSYS: ''
+RADESYS: ''
+FOCUSZ: 0.35416568455
+OBJECT: HD107696
+TELCODE: AT
+CONTRLLR: O
+DAYOBS: '20200128'
+SEQNUM: 379
+CCD_MANU: ''
+CCD_TYPE: ''
+CCD_SERN: ''
+LSST_NUM: ''
+SEQCKSUM: 2
+SEQNAME: PathFinder
+REBNAME: ''
+CONTNUM: ''
+IMAGETAG: ''
+TEMP_SET: -104.150001525879
+CCDTEMP: null
+OVERH: null
+OVERV: null
+PREH: null
+FILTER: KPNO_406_828nm
+FILTPOS: 4
+GRATING: empty_1
+GRATPOS: 1
+LINSPOS: 60.0
+EXPTIME: 5.0
+SHUTTIME: 5.029
+DARKTIME: 0.0
+FILENAME: AT_O_20200128_000379.fits
+HEADVER: 1.2.1
+XTENSION: IMAGE
+BITPIX: 32
+NAXIS: 2
+NAXIS1: 576
+NAXIS2: 2048
+PCOUNT: 0
+GCOUNT: 1
+EXTNAME: Segment10
+INHERIT: true
+DATASEC: '[4:512,1:2000]'
+DETSEC: '[509:1,1:2000]'
+DETSIZE: '[1:4072,1:4000]'
+DTV1: 513
+DTV2: 0
+DTM1_1: -1.0
+DTM2_2: 1.0
+DTM1_2: 0
+DTM2_1: 0
+

tests/test_comCam.py

@@ -78,10 +78,10 @@ def setUp(self):
                               )
 
         path_to_raw = os.path.join(self.data_dir, "raw", "unknown", "R22",
-                                   "2019053000001-R22-S00-det000-000.fits")
+                                   "2019053000001-R22-S00-det000.fits")
         keys = set(('filter', 'patch', 'tract', 'visit', 'channel', 'amp', 'style', 'detector', 'dstype',
                     'snap', 'run', 'calibDate', 'half', 'detectorName', 'raftName', 'label',
-                    'numSubfilters', 'fgcmcycle', 'name', 'pixel_id', 'description', 'subfilter'))
+                    'numSubfilters', 'fgcmcycle', 'name', 'pixel_id', 'description', 'subfilter', 'expId'))
         query_format = ["visit", "filter"]
         queryMetadata = (({'visit': 3019053000001}, [(3019053000001, 'NONE')]),
                          ({'filter': 'NONE'}, [(3019053000001, 'NONE')]),
@@ -92,11 +92,11 @@ def setUp(self):
         map_storage_name = 'FitsStorage'
         metadata_output_path = os.path.join("processCcd_metadata/3019053000001-NONE/R22",
                                             "processCcdMetadata_3019053000001-NONE-R22-S00-det000.yaml")
-        raw_filename = '3019053000001-R22-S00-det000-000.fits'
+        raw_filename = '3019053000001-R22-S00-det000.fits'
         default_level = 'visit'
-        raw_levels = (('skyTile', set(['visit', 'detector', 'snap', 'run', 'detectorName', 'raftName'])),
-                      ('filter', set(['visit', 'detector', 'snap', 'run', 'detectorName', 'raftName'])),
-                      ('visit', set(['visit', 'detector', 'snap', 'run', 'detectorName', 'raftName']))
+        raw_levels = (('skyTile', set(['expId', 'detector', 'run', 'detectorName', 'raftName'])),
+                      ('filter', set(['expId', 'detector', 'run', 'detectorName', 'raftName'])),
+                      ('visit', set(['expId', 'detector', 'run', 'detectorName', 'raftName']))
                       )
         self.setUp_mapper(output=self.data_dir,
                           path_to_raw=path_to_raw,

tests/test_imsim.py

@@ -97,10 +97,10 @@ def setUp(self):
                               linearizer_type=linearizer_type
                               )
 
-        path_to_raw = os.path.join(self.data_dir, "raw", "204595", "R11", "00204595-R11-S20-det042-000.fits")
+        path_to_raw = os.path.join(self.data_dir, "raw", "204595", "R11", "00204595-R11-S20-det042.fits")
         keys = set(('filter', 'patch', 'tract', 'visit', 'channel', 'amp', 'style', 'detector', 'dstype',
                     'snap', 'run', 'calibDate', 'half', 'detectorName', 'raftName', 'label',
-                    'numSubfilters', 'fgcmcycle', 'name', 'pixel_id', 'description', 'subfilter'))
+                    'numSubfilters', 'fgcmcycle', 'name', 'pixel_id', 'description', 'subfilter', 'expId'))
         query_format = ["visit", "filter"]
         queryMetadata = (({'visit': 204595}, [(204595, 'i')]),
                          ({'filter': 'i'}, [(204595, 'i')]),
@@ -111,11 +111,11 @@ def setUp(self):
         map_storage_name = 'FitsStorage'
         metadata_output_path = os.path.join('processCcd_metadata', '00204595-i', 'R11',
                                             'processCcdMetadata_00204595-i-R11-S20-det042.yaml')
-        raw_filename = '00204595-R11-S20-det042-000.fits'
+        raw_filename = '00204595-R11-S20-det042.fits'
         default_level = 'visit'
-        raw_levels = (('skyTile', set(['visit', 'detector', 'snap', 'run', 'detectorName', 'raftName'])),
-                      ('filter', set(['visit', 'detector', 'snap', 'run', 'detectorName', 'raftName'])),
-                      ('visit', set(['visit', 'detector', 'snap', 'run', 'detectorName', 'raftName']))
+        raw_levels = (('skyTile', set(['expId', 'detector', 'run', 'detectorName', 'raftName'])),
+                      ('filter', set(['expId', 'detector', 'run', 'detectorName', 'raftName'])),
+                      ('visit', set(['expId', 'detector', 'run', 'detectorName', 'raftName']))
                       )
         self.setUp_mapper(output=self.data_dir,
                           path_to_raw=path_to_raw,

tests/test_latiss.py

@@ -74,7 +74,7 @@ def setUp(self):
         path_to_raw = os.path.join(self.data_dir, "raw", "2018-09-20", "3018092000065-det000.fits")
         keys = set(('filter', 'patch', 'tract', 'visit', 'channel', 'amp', 'style', 'detector', 'dstype',
                     'calibDate', 'half', 'label', 'dayObs', 'run', 'snap', 'detectorName', 'raftName',
-                    'numSubfilters', 'fgcmcycle', 'name', 'pixel_id', 'description', 'subfilter'))
+                    'numSubfilters', 'fgcmcycle', 'name', 'pixel_id', 'description', 'subfilter', 'expId'))
         query_format = ["visit", "seqNum", "dayObs"]
         queryMetadata = (({'visit': 3018092000065}, [(3018092000065, 65, '2018-09-20')]),
                          ({'detector': 0}, [(3018092000065, 65, '2018-09-20')]),
@@ -88,9 +88,9 @@ def setUp(self):
 
         raw_filename = '3018092000065-det000.fits'
         default_level = 'visit'
-        raw_levels = (('skyTile', set(['visit', 'detector', 'dayObs'])),
-                      ('filter', set(['visit', 'detector', 'dayObs'])),
-                      ('visit', set(['visit', 'detector', 'dayObs']))
+        raw_levels = (('skyTile', set(['expId', 'detector', 'dayObs'])),
+                      ('filter', set(['expId', 'detector', 'dayObs'])),
+                      ('visit', set(['expId', 'detector', 'dayObs']))
                       )
         self.setUp_mapper(output=self.data_dir,
                           path_to_raw=path_to_raw,