Regtest / integration fixes for B14

JenkinsfileRT

@@ -52,7 +52,7 @@ def pip_install_args = "--index-url ${pip_index} --progress-bar=off"
 // Define environment variables needed for the regression tests
 env_vars = [
     "TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory",
-    "CRDS_CONTEXT=roman_0056.pmap",
+    "CRDS_CONTEXT=roman_0058.pmap",
     "CRDS_SERVER_URL=https://roman-serverless",
     'CRDS_PATH=/grp/crds/roman/test/',
     'DD_ENV=ci',

JenkinsfileRT_dev

@@ -52,7 +52,7 @@ def pip_install_args = "--index-url ${pip_index} --progress-bar=off"
 // Define environment variables needed for the regression tests
 env_vars = [
     "TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory",
-    "CRDS_CONTEXT=roman_0056.pmap",
+    "CRDS_CONTEXT=roman_0058.pmap",
     "CRDS_SERVER_URL=https://roman-serverless",
     'CRDS_PATH=/grp/crds/roman/test/',
     'WEBBPSF_PATH=/grp/jwst/ote/webbpsf-data',

pyproject.toml

@@ -17,7 +17,7 @@ dependencies = [
     "asdf-astropy >=0.5.0",
     "astropy >=5.3.0",
     "crds >=11.16.16",
-    "gwcs >=0.18.1",
+    "gwcs >=0.21.0",
     "jsonschema >=4.8",
     "numpy >=1.22",
     "photutils >=1.10.0",

requirements-sdp.txt

@@ -5,5 +5,5 @@
 #
 # Add a CRDS context number for the delivery as a record here
 #
-# CRDS_CONTEXT = roman_0056.pmap
+# CRDS_CONTEXT = roman_0058.pmap
 #

romancal/pipeline/highlevel_pipeline.py

@@ -76,7 +76,10 @@ def process(self, input):
             #
             # check to see if the product name contains a skycell name & if true get the skycell record
             product_name = input.asn_table["products"][0]["name"]
-            skycell_name = product_name.split("_")[3]
+            try:
+                skycell_name = product_name.split("_")[3]
+            except IndexError:
+                skycell_name = ""
             skycell_record = []
 
             # if this is a valid skycell name load the database and get the skycell record

romancal/regtest/conftest.py

@@ -205,6 +205,9 @@ def ignore_asdf_paths():
         "roman.cal_logs",
         "roman.meta.date",
         "roman.individual_image_cal_logs",
+        "roman.meta.individual_image_meta",
+        # individual image meta includes string arrays, not supported by
+        # compare_asdf at present.
         # roman.meta.filename is used by the ExposurePipeline so should likely
         # not be ignored here
         # "roman.meta.filename",

romancal/regtest/regtestdata.py

@@ -614,6 +614,8 @@ def give_up_diffing(self, level, diff_instance):
                 level.t2.meta,
                 ignore_nan_inequality=self.equal_nan,
                 math_epsilon=self.atol,
+                exclude_paths=["root['date']"],
+                # creation date sometimes stored here
             )
             if meta_difference:
                 difference["metas_differ"] = meta_difference

romancal/regtest/test_catalog.py

@@ -55,7 +55,7 @@ def test_catalog_l3(rtdata, ignore_asdf_paths):
     # DMS376: type of source
     # DMS386: flux uncertainties
     # DMS387: DQ flags
-    inputfn = "r0099101001001001001_F158_visit_0.900.0.50_178199.5_-0.5_i2d.asdf"
+    inputfn = "r0099101001001001001_F158_visit_i2d.asdf"
     outputfn = inputfn.replace("_i2d", "_cat")
     rtdata.get_data(f"WFI/image/{inputfn}")
     rtdata.input = inputfn

romancal/regtest/test_hlp_pipeline.py

@@ -38,7 +38,7 @@ def test_level3_hlp_pipeline(rtdata, ignore_asdf_paths):
     rtdata.input = input_asn
 
     # Test Pipeline
-    output = "r0099101001001001001_F158_visit_0.900.0.50_178199.5_-0.5_i2d.asdf"
+    output = "r0099101001001001001_F158_visit_i2d.asdf"
     rtdata.output = output
     args = [
         "--disable-crds-steppars",
@@ -51,18 +51,17 @@ def test_level3_hlp_pipeline(rtdata, ignore_asdf_paths):
     assert diff.identical, diff.report()
 
     # Generate thumbnail image
-    input_file = "r0099101001001001001_F158_visit_0.900.0.50_178199.5_-0.5_i2d.asdf"
-    thumbnail_file = (
-        "r0099101001001001001_F158_visit_0.900.0.50_178199.5_-0.5_thumb.png"
-    )
+    input_file = "r0099101001001001001_F158_visit_i2d.asdf"
+    thumbnail_file = "r0099101001001001001_F158_visit_thumb.png"
+    input_file = "r0099101001001001001_F158_visit_i2d.asdf"
+    thumbnail_file = "r0099101001001001001_F158_visit_thumb.png"
+
     preview_cmd = f"stpreview to {input_file} {thumbnail_file} 256 256 roman"
     os.system(preview_cmd)  # nosec
 
     # Generate preview image
-    input_file = "r0099101001001001001_F158_visit_0.900.0.50_178199.5_-0.5_i2d.asdf"
-    preview_file = (
-        "r0099101001001001001_F158_visit_0.900.0.50_178199.5_-0.5_preview.png"
-    )
+    input_file = "r0099101001001001001_F158_visit_i2d.asdf"
+    preview_file = "r0099101001001001001_F158_visit_preview.png"
     preview_cmd = f"stpreview to {input_file} {preview_file} 1080 1080 roman"
     os.system(preview_cmd)  # nosec
 

romancal/scripts/make_regtestdata.sh

@@ -154,15 +154,15 @@ import roman_datamodels as rdm
 from roman_datamodels import stnode
 from romancal.assign_wcs.assign_wcs_step import AssignWcsStep
 model = rdm.open('${basename}_cal.asdf', lazy_load=False)
-model.meta.filename = stnode.Filename(f'{basename}_shift_cal.asdf')
+model.meta.filename = stnode.Filename(f'${basename}_shift_cal.asdf')
 delta = [1 / 3600., 1 / 3600.]
 wcsinfo = model.meta.wcsinfo
 wcsinfo.ra_ref += delta[0]
 wcsinfo.dec_ref += delta[1]
 model = AssignWcsStep.call(model)
 model.to_asdf(f'${basename}_shift_cal.asdf')"
     strun romancal.step.TweakRegStep ${basename}_shift_cal.asdf
-    cp ${basename}_shift_cal.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
+    cp ${basename}_shift_cal.asdf $outdir/roman-pipeline/dev/WFI/image/
     cp ${basename}_shift_tweakregstep.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 done
 
@@ -172,10 +172,11 @@ strun roman_elp r0000201001001001001_01101_0004_WFI01_uncal.asdf
 cp r0000101001001001001_01101_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/image/
 cp r0000201001001001001_01101_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/grism/
 
-l3name="r0099101001001001001_F158_visit_0.900.0.50_178199.5_-0.5"
+l3name="r0099101001001001001_F158_visit"
 asn_from_list r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf r0000101001001001001_01101_0003_WFI01_cal.asdf -o L3_regtest_asn.json --product-name $l3name
 strun --disable-crds-steppars roman_hlp L3_regtest_asn.json
 cp L3_regtest_asn.json $outdir/roman-pipeline/dev/WFI/image/
+cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/WFI/image/
 cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
 # L3 catalog
@@ -185,3 +186,8 @@ cp ${l3name}_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 # L2 catalog
 strun romancal.step.SourceCatalogStep r0000101001001001001_01101_0001_WFI01_cal.asdf
 cp r0000101001001001001_01101_0001_WFI01_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
+
+l3name="r0099101001001001001_F158_visit_r274dp63x31y81"
+asn_from_list --product-name=$l3name r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf r0000101001001001001_01101_0003_WFI01_cal.asdf -o L3_m1_asn.json
+strun --disable-crds-steppars roman_hlp L3_m1_asn.json
+cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/