Merge branch 'master' into abs-sep-tol-check

CHANGES.rst

@@ -57,11 +57,15 @@ extract_1d
   regions are specified and the lower and upper limits for one of them are
   outside the valid area for some data range. [#8433]
 
+- Correct the output slit name for non-primary slit extractions in the
+  spec3 pipeline, for NIRSpec fixed slit mode. [#8470]
+
 extract_2d
 ----------
 
 - Added handling for NIRCam GRISM time series pointing offsets. [#8449]
 
+
 flat_field
 ----------
 
@@ -91,6 +95,10 @@ outlier_detection
   ``OutlierDetectionStackStep``, ``outlierpars`` reference file handling,
   and ``scale_detection`` (an unused argument). [#8438]
 
+- Intermediate output files are now correctly removed after the step has
+  finished, unless save_intermediate_results is True. This PR also addressed
+  the _i2d files not being saved in the specified output directory. [#8464]
+
 photom
 ------
 
@@ -147,12 +155,15 @@ tweakreg
   when this condition is not satisfied and source confusion is possible during
   catalog matching. [#8476]
 
+- Improve error handling in the absolute alignment. [#8450, #8477]
+
 wfss_contam
 -----------
 
 - Fixed flux scaling issue in model contamination image by adding background
   subtraction and re-scaling fluxes to respect wavelength oversampling. [#8416]
 
+
 1.14.0 (2024-03-29)
 ===================
 

jwst/extract_1d/extract.py

@@ -2986,7 +2986,10 @@ def do_extract1d(
 
             # For NRS_FIXEDSLIT, the slit name comes from the FXD_SLIT keyword in the model meta
             if input_model.meta.exposure.type == 'NRS_FIXEDSLIT':
-                slitname = input_model.meta.instrument.fixed_slit
+                if hasattr(input_model, "name") and input_model.name is not None:
+                    slitname = input_model.name
+                else:
+                    slitname = input_model.meta.instrument.fixed_slit
 
             # Loop over all spectral orders available for extraction
             prev_offset = OFFSET_NOT_ASSIGNED_YET

jwst/extract_1d/tests/test_extract_1d_step.py

@@ -0,0 +1,118 @@
+import numpy as np
+import pytest
+import stdatamodels.jwst.datamodels as dm
+
+from jwst.assign_wcs.util import wcs_bbox_from_shape
+from jwst.extract_1d.extract_1d_step import Extract1dStep
+
+
+@pytest.fixture
+def simple_wcs():
+    shape = (50, 50)
+    xcenter = shape[1] // 2.0
+
+    def simple_wcs_function(x, y):
+        """ Simple WCS for testing """
+        crpix1 = xcenter
+        crpix3 = 1.0
+        cdelt1 = 0.1
+        cdelt2 = 0.1
+        cdelt3 = 0.01
+
+        crval1 = 45.0
+        crval2 = 45.0
+        crval3 = 7.5
+
+        wave = (x + 1 - crpix3) * cdelt3 + crval3
+        ra = (x + 1 - crpix1) * cdelt1 + crval1
+        dec = np.full_like(ra, crval2 + 1 * cdelt2)
+
+        return ra, dec, wave
+
+    simple_wcs_function.bounding_box = wcs_bbox_from_shape(shape)
+
+    return simple_wcs_function
+
+
+@pytest.fixture()
+def mock_nirspec_fs_one_slit(simple_wcs):
+    model = dm.SlitModel()
+    model.meta.instrument.name = 'NIRSPEC'
+    model.meta.instrument.detector = 'NRS1'
+    model.meta.observation.date = '2023-07-22'
+    model.meta.observation.time = '06:24:45.569'
+    model.meta.instrument.fixed_slit = 'S200A1'
+    model.meta.exposure.type = 'NRS_FIXEDSLIT'
+    model.meta.subarray.name = 'ALLSLITS'
+
+    model.source_type = 'EXTENDED'
+    model.meta.wcsinfo.dispersion_direction = 1
+    model.meta.wcs = simple_wcs
+
+    model.data = np.arange(50 * 50, dtype=float).reshape((50, 50))
+    model.var_poisson = model.data * 0.02
+    model.var_rnoise = model.data * 0.02
+    model.var_flat = model.data * 0.05
+    yield model
+    model.close()
+
+
+@pytest.fixture()
+def mock_nirspec_mos(mock_nirspec_fs_one_slit):
+    model = dm.MultiSlitModel()
+    model.meta.instrument.name = 'NIRSPEC'
+    model.meta.instrument.detector = 'NRS1'
+    model.meta.observation.date = '2023-07-22'
+    model.meta.observation.time = '06:24:45.569'
+    model.meta.exposure.type = 'NRS_MSASPEC'
+
+    nslit = 3
+    for i in range(nslit):
+        slit = mock_nirspec_fs_one_slit.copy()
+        slit.name = str(i + 1)
+        model.slits.append(slit)
+
+    yield model
+    model.close()
+
+
+@pytest.mark.parametrize('slit_name', [None, 'S200A1', 'S1600A1'])
+def test_extract_nirspec_fs_slit(mock_nirspec_fs_one_slit, simple_wcs, slit_name):
+    mock_nirspec_fs_one_slit.name = slit_name
+    result = Extract1dStep.call(mock_nirspec_fs_one_slit)
+    assert result.meta.cal_step.extract_1d == 'COMPLETE'
+
+    if slit_name is None:
+        assert (result.spec[0].name
+                == mock_nirspec_fs_one_slit.meta.instrument.fixed_slit)
+    else:
+        assert result.spec[0].name == slit_name
+
+    # output wavelength is the same as input
+    _, _, expected_wave = simple_wcs(np.arange(50), np.arange(50))
+    assert np.allclose(result.spec[0].spec_table['WAVELENGTH'], expected_wave)
+
+    # output flux and errors are non-zero, exact values will depend
+    # on extraction parameters
+    assert np.all(result.spec[0].spec_table['FLUX'] > 0)
+    assert np.all(result.spec[0].spec_table['FLUX_ERROR'] > 0)
+    result.close()
+
+
+def test_extract_nirspec_mos_multi_slit(mock_nirspec_mos, simple_wcs):
+    result = Extract1dStep.call(mock_nirspec_mos)
+    assert result.meta.cal_step.extract_1d == 'COMPLETE'
+
+    for i, spec in enumerate(result.spec):
+        assert spec.name == str(i + 1)
+
+        # output wavelength is the same as input
+        _, _, expected_wave = simple_wcs(np.arange(50), np.arange(50))
+        assert np.allclose(spec.spec_table['WAVELENGTH'], expected_wave)
+
+        # output flux and errors are non-zero, exact values will depend
+        # on extraction parameters
+        assert np.all(spec.spec_table['FLUX'] > 0)
+        assert np.all(spec.spec_table['FLUX_ERROR'] > 0)
+
+    result.close()

jwst/outlier_detection/outlier_detection.py

@@ -3,6 +3,7 @@
 from functools import partial
 import logging
 import warnings
+import os
 
 import numpy as np
 from astropy.stats import sigma_clip
@@ -140,7 +141,10 @@ def do_detection(self):
         if pars['resample_data']:
             # Start by creating resampled/mosaic images for
             # each group of exposures
-            resamp = resample.ResampleData(self.input_models, single=True,
+            output_path = self.make_output_path(basepath=self.input_models[0].meta.filename,
+                            suffix='')
+            output_path = os.path.dirname(output_path)
+            resamp = resample.ResampleData(self.input_models, output=output_path, single=True,
                                            blendheaders=False, **pars)
             drizzled_models = resamp.do_drizzle()
 
@@ -171,6 +175,12 @@ def do_detection(self):
                 suffix='median')
             median_model.save(median_model_output_path)
             log.info(f"Saved model in {median_model_output_path}")
+        else:
+            # since we're not saving intermediate results if the drizzled models
+            # were written to disk, remove them
+            if not pars['in_memory']:
+                for fn in drizzled_models._models:
+                    _remove_file(fn)
 
         if pars['resample_data']:
             # Blot the median image back to recreate each input image specified
@@ -189,6 +199,9 @@ def do_detection(self):
 
         # clean-up (just to be explicit about being finished with
         # these results)
+        if not pars['save_intermediate_results'] and not pars['in_memory']:
+            for fn in blot_models._models:
+                _remove_file(fn)
         del median_model, blot_models
 
     def create_median(self, resampled_models):
@@ -335,6 +348,12 @@ def detect_outliers(self, blot_models):
                 self.inputs.dq[i, :, :] = self.input_models[i].dq
 
 
+def _remove_file(fn):
+    if isinstance(fn, str) and os.path.isfile(fn):
+        os.remove(fn)
+        log.debug(f"    {fn}")
+
+
 def flag_cr(sci_image, blot_image, snr="5.0 4.0", scale="1.2 0.7", backg=0,
             resample_data=True, **kwargs):
     """Masks outliers in science image by updating DQ in-place

jwst/outlier_detection/outlier_detection_spec.py

@@ -1,12 +1,10 @@
 """Class definition for performing outlier detection on spectra."""
-from functools import partial
-
 from stdatamodels.jwst import datamodels
 
 from jwst.datamodels import ModelContainer
 
 from ..resample import resample_spec, resample_utils
-from .outlier_detection import OutlierDetection
+from .outlier_detection import OutlierDetection, _remove_file
 
 import logging
 log = logging.getLogger(__name__)
@@ -58,6 +56,9 @@ def __init__(self, input_models, **pars):
         """
         OutlierDetection.__init__(self, input_models, **pars)
 
+        # Set up the list of all intermediate output files
+        self.output_list = []
+
     def do_detection(self):
         """Flag outlier pixels in DQ of input images."""
         self._convert_inputs()
@@ -103,16 +104,17 @@ def do_detection(self):
             log.info("Writing out MEDIAN image to: {}".format(
                      median_model.meta.filename))
             median_model.save(median_model.meta.filename)
+        else:
+            # since we're not saving intermediate results if the drizzled models
+            # were written to disk, remove them
+            if not pars['in_memory']:
+                for fn in drizzled_models._models:
+                    _remove_file(fn)
 
         if pars['resample_data'] is True:
             # Blot the median image back to recreate each input image specified
             # in the original input list/ASN/ModelContainer
             blot_models = self.blot_median(median_model)
-            if save_intermediate_results:
-                log.info("Writing out BLOT images...")
-                blot_models.save(
-                    partial(self.make_output_path, suffix='blot')
-                )
         else:
             # Median image will serve as blot image
             blot_models = ModelContainer()
@@ -125,4 +127,7 @@ def do_detection(self):
 
         # clean-up (just to be explicit about being finished
         #  with these results)
+        if not pars['save_intermediate_results']:
+            for fn in blot_models._models:
+                _remove_file(fn)
         del median_model, blot_models

jwst/outlier_detection/outlier_detection_step.py

@@ -1,6 +1,4 @@
 """Public common step definition for OutlierDetection processing."""
-import os
-
 from functools import partial
 
 from stdatamodels.jwst import datamodels
@@ -173,21 +171,6 @@ def process(self, input_data):
                     self.log.debug("The following files will be deleted since save_intermediate_results=False:")
                 for model in self.input_models:
                     model.meta.cal_step.outlier_detection = state
-                    if not self.save_intermediate_results:
-                        #  Remove unwanted files
-                        crf_path = self.make_output_path(basepath=model.meta.filename)
-                        if asn_id is None:
-                            suffix = model.meta.filename.split(sep='_')[-1]
-                            outlr_file = model.meta.filename.replace(suffix, 'outlier_i2d.fits')
-                        else:
-                            outlr_file = crf_path.replace('crf', 'outlier_i2d')
-                        blot_path = crf_path.replace('crf', 'blot')
-                        median_path = blot_path.replace('blot', 'median')
-
-                        for fle in [outlr_file, blot_path, median_path]:
-                            if os.path.isfile(fle):
-                                os.remove(fle)
-                                self.log.debug(f"    {fle}")
             else:
                 self.input_models.meta.cal_step.outlier_detection = state
             return self.input_models

jwst/regtest/test_niriss_image.py

@@ -6,7 +6,9 @@
 import pytest
 from astropy.io.fits.diff import FITSDiff
 
+from jwst import datamodels
 from jwst.stpipe import Step
+from jwst.tweakreg import TweakRegStep
 
 
 @pytest.fixture(scope="module")
@@ -44,6 +46,29 @@ def test_niriss_image_detector1(run_detector1, rtdata_module, fitsdiff_default_k
     _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix)
 
 
+@pytest.mark.bigdata
+def test_niriss_tweakreg_no_sources(rtdata, fitsdiff_default_kwargs):
+    """Make sure tweakreg is skipped when sources are not found.
+    """
+
+    rtdata.input = "niriss/imaging/jw01537-o003_20240406t164421_image3_00004_asn.json"
+    rtdata.get_asn("niriss/imaging/jw01537-o003_20240406t164421_image3_00004_asn.json")
+
+    args = ["jwst.tweakreg.TweakRegStep", rtdata.input, "--abs_refcat='GAIADR3'"]
+    result = Step.from_cmdline(args)
+    # Check that the step is skipped
+    assert result.skip
+
+    # Check the status of the step is set correctly in the files.
+    result = TweakRegStep.call(rtdata.input)
+
+    for fi in result._models:
+        with datamodels.open(fi) as model:
+            assert model.meta.cal_step.tweakreg == 'SKIPPED'
+
+    result.close()
+
+
 def _assert_is_same(rtdata_module, fitsdiff_default_kwargs, suffix):
     """Assertion helper for the above tests"""
     rtdata = rtdata_module

jwst/regtest/test_nirspec_mos_spec3.py

@@ -40,16 +40,24 @@ def test_nirspec_mos_spec3(run_pipeline, suffix, source_id, fitsdiff_default_kwa
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
     assert diff.identical, diff.report()
 
-    if "s2d" in output:
+    if suffix == "s2d":
         # Compare the calculated wavelengths
+        tolerance = 1e-03
         dmt = datamodels.open(rtdata.truth)
         dmr = datamodels.open(rtdata.output)
-        names = [s.name for s in dmt.slits]
-        for name in names:
-            st_idx = [(s.wcs, s.wavelength) for s in dmt.slits if s.name==name]
-            w = dmt.slits[st_idx].meta.wcs
+        if isinstance(dmt, datamodels.MultiSlitModel):
+            names = [s.name for s in dmt.slits]
+            for name in names:
+                st_idx = [(s.wcs, s.wavelength) for s in dmt.slits if s.name==name]
+                w = dmt.slits[st_idx].meta.wcs
+                x, y = wcstools.grid_from_bounding_box(w.bounding_box, step=(1, 1), center=True)
+                _, _, wave = w(x, y)
+                sr_idx = [(s.wcs, s.wavelength) for s in dmr.slits if s.name==name]
+                wlr = dmr.slits[sr_idx].wavelength
+                assert np.all(np.isclose(wave, wlr, atol=tolerance))
+        else:
+            w = dmt.meta.wcs
             x, y = wcstools.grid_from_bounding_box(w.bounding_box, step=(1, 1), center=True)
             _, _, wave = w(x, y)
-            sr_idx = [(s.wcs, s.wavelength) for s in dmr.slits if s.name==name]
-            wlr = dmr.slits[sr_idx].wavelength
-            assert np.all(np.isclose(wave, wlr, atol=1e-03))
+            wlr = dmr.wavelength
+            assert np.all(np.isclose(wave, wlr, atol=tolerance))