Merge branch 'master' into JP-3584

CHANGES.rst

@@ -103,6 +103,8 @@ outlier_detection
   to detect outliers in TSO and coronagraphic data, with user-defined
   rolling window width via the ``n_ints`` parameter. [#8473]
 
+- Added tests for changes made in #8464. [#8481]
+
 photom
 ------
 
@@ -151,11 +153,16 @@ tweakreg
 
 - Improved how a image group name is determined. [#8426]
 
+- Refactor step to work towards performance improvements. [#8424]
+
 - Changed default settings for ``abs_separation`` parameter for the ``tweakreg``
   step to have a value compatible with the ``abs_tolerance`` parameter. [#8445]
 
 - Improve error handling in the absolute alignment. [#8450, #8477]
 
+- Change code default to use IRAF StarFinder instead of
+  DAO StarFinder [#8487]
+
 wfss_contam
 -----------
 

docs/jwst/tweakreg/README.rst

@@ -218,7 +218,7 @@ The ``tweakreg`` step has the following optional arguments:
   in pixels. (Default=400)
 
 * ``starfinder``: A `str` indicating the source detection algorithm to use.
-  Allowed values: `'iraf'`, `'dao'`, `'segmentation'`. (Default= `'dao'`)
+  Allowed values: `'iraf'`, `'dao'`, `'segmentation'`. (Default= `'iraf'`)
 
 * ``snr_threshold``: A `float` value indicating SNR threshold above the
   background. Required for all star finders. (Default=10.0)

jwst/assign_wcs/util.py

@@ -199,7 +199,7 @@ def calc_rotation_matrix(roll_ref: float, v3i_yang: float, vparity: int = 1) ->
 
 def wcs_from_footprints(dmodels, refmodel=None, transform=None, bounding_box=None,
                         pscale_ratio=None, pscale=None, rotation=None,
-                        shape=None, crpix=None, crval=None):
+                        shape=None, crpix=None, crval=None, wcslist=None):
     """
     Create a WCS from a list of input data models.
 
@@ -259,7 +259,8 @@ def wcs_from_footprints(dmodels, refmodel=None, transform=None, bounding_box=Non
 
     """
     bb = bounding_box
-    wcslist = [im.meta.wcs for im in dmodels]
+    if wcslist is None:
+        wcslist = [im.meta.wcs for im in dmodels]
 
     if not isiterable(wcslist):
         raise ValueError("Expected 'wcslist' to be an iterable of WCS objects.")

jwst/coron/tests/test_coron.py

@@ -141,8 +141,8 @@ def test_align_array():
         ]
     )
 
-    npt.assert_allclose(aligned, truth_aligned, atol=1e-6)
-    npt.assert_allclose(shifts, truth_shifts, atol=1e-6)
+    npt.assert_allclose(aligned, truth_aligned, atol=1e-5)
+    npt.assert_allclose(shifts, truth_shifts, atol=1e-5)
 
 
 def test_align_models():

jwst/datamodels/container.py

@@ -160,6 +160,7 @@ def __init__(self, init=None, asn_exptypes=None, asn_n_members=None,
         self.asn_table = {}
         self.asn_table_name = None
         self.asn_pool_name = None
+        self.asn_file_path = None
 
         self._memmap = kwargs.get("memmap", False)
         self._return_open = kwargs.get('return_open', True)
@@ -196,7 +197,8 @@ def __init__(self, init=None, asn_exptypes=None, asn_n_members=None,
             self.from_asn(init)
         elif isinstance(init, str):
             init_from_asn = self.read_asn(init)
-            self.from_asn(init_from_asn, asn_file_path=init)
+            self.asn_file_path = init
+            self.from_asn(init_from_asn)
         else:
             raise TypeError('Input {0!r} is not a list of JwstDataModels or '
                             'an ASN file'.format(init))
@@ -275,17 +277,14 @@ def read_asn(filepath):
             raise IOError("Cannot read ASN file.") from e
         return asn_data
 
-    def from_asn(self, asn_data, asn_file_path=None):
+    def from_asn(self, asn_data):
         """
         Load fits files from a JWST association file.
 
         Parameters
         ----------
         asn_data : ~jwst.associations.Association
             An association dictionary
-
-        asn_file_path: str
-            Filepath of the association, if known.
         """
         # match the asn_exptypes to the exptype in the association and retain
         # only those file that match, as a list, if asn_exptypes is set to none
@@ -303,8 +302,8 @@ def from_asn(self, asn_data, asn_file_path=None):
             infiles = [member for member
                        in asn_data['products'][0]['members']]
 
-        if asn_file_path:
-            asn_dir = op.dirname(asn_file_path)
+        if self.asn_file_path:
+            asn_dir = op.dirname(self.asn_file_path)
         else:
             asn_dir = ''
 
@@ -348,8 +347,8 @@ def from_asn(self, asn_data, asn_file_path=None):
             self.meta.asn_table._instance, asn_data
         )
 
-        if asn_file_path is not None:
-            self.asn_table_name = op.basename(asn_file_path)
+        if self.asn_file_path is not None:
+            self.asn_table_name = op.basename(self.asn_file_path)
             self.asn_pool_name = asn_data['asn_pool']
             for model in self:
                 try:

jwst/outlier_detection/outlier_detection.py

@@ -387,7 +387,7 @@ def detect_outliers(self, blot_models):
 def _remove_file(fn):
     if isinstance(fn, str) and os.path.isfile(fn):
         os.remove(fn)
-        log.debug(f"    {fn}")
+        log.info(f"Removing file {fn}")
 
 
 def flag_cr(sci_image, blot_image, snr="5.0 4.0", scale="1.2 0.7", backg=0,

jwst/outlier_detection/outlier_detection_spec.py

@@ -110,6 +110,7 @@ def do_detection(self):
             if not pars['in_memory']:
                 for fn in drizzled_models._models:
                     _remove_file(fn)
+                    log.info(f"Removing file {fn}")
 
         if pars['resample_data'] is True:
             # Blot the median image back to recreate each input image specified
@@ -130,4 +131,5 @@ def do_detection(self):
         if not pars['save_intermediate_results']:
             for fn in blot_models._models:
                 _remove_file(fn)
+                log.info(f"Removing file {fn}")
         del median_model, blot_models

jwst/outlier_detection/outlier_detection_step.py

@@ -174,8 +174,6 @@ def process(self, input_data):
 
             state = 'COMPLETE'
             if self.input_container:
-                if not self.save_intermediate_results:
-                    self.log.debug("The following files will be deleted since save_intermediate_results=False:")
                 for model in self.input_models:
                     model.meta.cal_step.outlier_detection = state
             else:

jwst/outlier_detection/tests/test_outlier_detection.py

@@ -1,6 +1,8 @@
 import pytest
 import numpy as np
 from scipy.ndimage import gaussian_filter
+from glob import glob
+import os
 
 from stdatamodels.jwst import datamodels
 
@@ -15,7 +17,6 @@
 )
 from jwst.assign_wcs.pointing import create_fitswcs
 
-
 OUTLIER_DO_NOT_USE = np.bitwise_or(
     datamodels.dqflags.pixel["DO_NOT_USE"], datamodels.dqflags.pixel["OUTLIER"]
 )
@@ -184,6 +185,15 @@ def test_outlier_step(we_three_sci, tmp_cwd):
     # Drop a CR on the science array
     container[0].data[12, 12] += 1
 
+    # Verify that intermediary files are removed
+    OutlierDetectionStep.call(container)
+    i2d_files = glob(os.path.join(tmp_cwd, '*i2d.fits'))
+    median_files = glob(os.path.join(tmp_cwd, '*median.fits'))
+    blot_files = glob(os.path.join(tmp_cwd, '*blot.fits'))
+    assert len(i2d_files) == 0
+    assert len(median_files) == 0
+    assert len(blot_files) == 0
+
     result = OutlierDetectionStep.call(
         container, save_results=True, save_intermediate_results=True
     )
@@ -199,6 +209,14 @@ def test_outlier_step(we_three_sci, tmp_cwd):
     # Verify CR is flagged
     assert result[0].dq[12, 12] == OUTLIER_DO_NOT_USE
 
+    # Verify that intermediary files are saved at the specified location
+    i2d_files = glob(os.path.join(tmp_cwd, '*i2d.fits'))
+    median_files = glob(os.path.join(tmp_cwd, '*median.fits'))
+    blot_files = glob(os.path.join(tmp_cwd, '*blot.fits'))
+    assert len(i2d_files) != 0
+    assert len(median_files) != 0
+    assert len(blot_files) != 0
+
 
 def test_outlier_step_on_disk(we_three_sci, tmp_cwd):
     """Test whole step with an outlier including saving intermediate and results files"""

jwst/regtest/test_miri_lrs_slit_spec3.py

@@ -79,7 +79,7 @@ def test_miri_lrs_slit_spec3(run_pipeline, rtdata_module, fitsdiff_default_kwarg
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
     assert diff.identical, diff.report()
 
-    if "s2d" in output:
+    if output == "s2d":
         # Compare the calculated wavelengths
         tolerance = 1e-03
         dmt = datamodels.open(rtdata.truth)

jwst/regtest/test_niriss_image.py

@@ -54,17 +54,26 @@ def test_niriss_tweakreg_no_sources(rtdata, fitsdiff_default_kwargs):
     rtdata.input = "niriss/imaging/jw01537-o003_20240406t164421_image3_00004_asn.json"
     rtdata.get_asn("niriss/imaging/jw01537-o003_20240406t164421_image3_00004_asn.json")
 
-    args = ["jwst.tweakreg.TweakRegStep", rtdata.input, "--abs_refcat='GAIADR3'"]
+    args = [
+        "jwst.tweakreg.TweakRegStep",
+        rtdata.input,
+        "--abs_refcat='GAIADR3'",
+        "--save_results=True",
+    ]
     result = Step.from_cmdline(args)
     # Check that the step is skipped
     assert result.skip
 
     # Check the status of the step is set correctly in the files.
-    result = TweakRegStep.call(rtdata.input)
+    mc = datamodels.ModelContainer(rtdata.input)
 
-    for fi in result._models:
-        with datamodels.open(fi) as model:
-            assert model.meta.cal_step.tweakreg == 'SKIPPED'
+    for model in mc:
+        assert model.meta.cal_step.tweakreg != 'SKIPPED'
+
+    result = TweakRegStep.call(mc)
+
+    for model in result:
+        assert model.meta.cal_step.tweakreg == 'SKIPPED'
 
     result.close()
 

jwst/regtest/test_nirspec_fs_spec3.py

@@ -46,7 +46,7 @@ def test_nirspec_fs_spec3(run_pipeline, rtdata_module, fitsdiff_default_kwargs,
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
     assert diff.identical, diff.report()
 
-    if "s2d" in output:
+    if output == "s2d":
         # Compare the calculated wavelengths
         tolerance = 1e-03
         dmt = datamodels.open(rtdata.truth)

jwst/resample/resample.py

@@ -72,7 +72,7 @@ def __init__(self, input_models, output=None, single=False, blendheaders=True,
         self.input_models = input_models
         self.output_dir = None
         self.output_filename = output
-        if output is not None and '.fits' not in output:
+        if output is not None and '.fits' not in str(output):
             self.output_dir = output
             self.output_filename = None
 

jwst/resample/resample_spec.py

@@ -64,7 +64,7 @@ def __init__(self, input_models, output=None, single=False, blendheaders=False,
 
         self.output_filename = output
         self.output_dir = None
-        if output is not None and '.fits' not in output:
+        if output is not None and '.fits' not in str(output):
             self.output_dir = output
             self.output_filename = None
         self.pscale_ratio = pscale_ratio