JP-3311: OutlierDetectionStep producing extraneous outlier_i2d files (#…

…8464)

Co-authored-by: Brett <brettgraham@gmail.com>

CHANGES.rst

@@ -95,6 +95,10 @@ outlier_detection
   ``OutlierDetectionStackStep``, ``outlierpars`` reference file handling,
   and ``scale_detection`` (an unused argument). [#8438]
 
+- Intermediate output files are now correctly removed after the step has
+  finished, unless save_intermediate_results is True. This PR also addressed
+  the _i2d files not being saved in the specified output directory. [#8464]
+
 photom
 ------
 

jwst/outlier_detection/outlier_detection.py

@@ -3,6 +3,7 @@
 from functools import partial
 import logging
 import warnings
+import os
 
 import numpy as np
 from astropy.stats import sigma_clip
@@ -140,7 +141,10 @@ def do_detection(self):
         if pars['resample_data']:
             # Start by creating resampled/mosaic images for
             # each group of exposures
-            resamp = resample.ResampleData(self.input_models, single=True,
+            output_path = self.make_output_path(basepath=self.input_models[0].meta.filename,
+                            suffix='')
+            output_path = os.path.dirname(output_path)
+            resamp = resample.ResampleData(self.input_models, output=output_path, single=True,
                                            blendheaders=False, **pars)
             drizzled_models = resamp.do_drizzle()
 
@@ -171,6 +175,12 @@ def do_detection(self):
                 suffix='median')
             median_model.save(median_model_output_path)
             log.info(f"Saved model in {median_model_output_path}")
+        else:
+            # since we're not saving intermediate results if the drizzled models
+            # were written to disk, remove them
+            if not pars['in_memory']:
+                for fn in drizzled_models._models:
+                    _remove_file(fn)
 
         if pars['resample_data']:
             # Blot the median image back to recreate each input image specified
@@ -189,6 +199,9 @@ def do_detection(self):
 
         # clean-up (just to be explicit about being finished with
         # these results)
+        if not pars['save_intermediate_results'] and not pars['in_memory']:
+            for fn in blot_models._models:
+                _remove_file(fn)
         del median_model, blot_models
 
     def create_median(self, resampled_models):
@@ -335,6 +348,12 @@ def detect_outliers(self, blot_models):
                 self.inputs.dq[i, :, :] = self.input_models[i].dq
 
 
+def _remove_file(fn):
+    if isinstance(fn, str) and os.path.isfile(fn):
+        os.remove(fn)
+        log.debug(f"    {fn}")
+
+
 def flag_cr(sci_image, blot_image, snr="5.0 4.0", scale="1.2 0.7", backg=0,
             resample_data=True, **kwargs):
     """Masks outliers in science image by updating DQ in-place

jwst/outlier_detection/outlier_detection_spec.py

@@ -1,12 +1,10 @@
 """Class definition for performing outlier detection on spectra."""
-from functools import partial
-
 from stdatamodels.jwst import datamodels
 
 from jwst.datamodels import ModelContainer
 
 from ..resample import resample_spec, resample_utils
-from .outlier_detection import OutlierDetection
+from .outlier_detection import OutlierDetection, _remove_file
 
 import logging
 log = logging.getLogger(__name__)
@@ -58,6 +56,9 @@ def __init__(self, input_models, **pars):
         """
         OutlierDetection.__init__(self, input_models, **pars)
 
+        # Set up the list of all intermediate output files
+        self.output_list = []
+
     def do_detection(self):
         """Flag outlier pixels in DQ of input images."""
         self._convert_inputs()
@@ -103,16 +104,17 @@ def do_detection(self):
             log.info("Writing out MEDIAN image to: {}".format(
                      median_model.meta.filename))
             median_model.save(median_model.meta.filename)
+        else:
+            # since we're not saving intermediate results if the drizzled models
+            # were written to disk, remove them
+            if not pars['in_memory']:
+                for fn in drizzled_models._models:
+                    _remove_file(fn)
 
         if pars['resample_data'] is True:
             # Blot the median image back to recreate each input image specified
             # in the original input list/ASN/ModelContainer
             blot_models = self.blot_median(median_model)
-            if save_intermediate_results:
-                log.info("Writing out BLOT images...")
-                blot_models.save(
-                    partial(self.make_output_path, suffix='blot')
-                )
         else:
             # Median image will serve as blot image
             blot_models = ModelContainer()
@@ -125,4 +127,7 @@ def do_detection(self):
 
         # clean-up (just to be explicit about being finished
         #  with these results)
+        if not pars['save_intermediate_results']:
+            for fn in blot_models._models:
+                _remove_file(fn)
         del median_model, blot_models

jwst/outlier_detection/outlier_detection_step.py

@@ -1,6 +1,4 @@
 """Public common step definition for OutlierDetection processing."""
-import os
-
 from functools import partial
 
 from stdatamodels.jwst import datamodels
@@ -173,21 +171,6 @@ def process(self, input_data):
                     self.log.debug("The following files will be deleted since save_intermediate_results=False:")
                 for model in self.input_models:
                     model.meta.cal_step.outlier_detection = state
-                    if not self.save_intermediate_results:
-                        #  Remove unwanted files
-                        crf_path = self.make_output_path(basepath=model.meta.filename)
-                        if asn_id is None:
-                            suffix = model.meta.filename.split(sep='_')[-1]
-                            outlr_file = model.meta.filename.replace(suffix, 'outlier_i2d.fits')
-                        else:
-                            outlr_file = crf_path.replace('crf', 'outlier_i2d')
-                        blot_path = crf_path.replace('crf', 'blot')
-                        median_path = blot_path.replace('blot', 'median')
-
-                        for fle in [outlr_file, blot_path, median_path]:
-                            if os.path.isfile(fle):
-                                os.remove(fle)
-                                self.log.debug(f"    {fle}")
             else:
                 self.input_models.meta.cal_step.outlier_detection = state
             return self.input_models

jwst/regtest/test_nirspec_mos_spec3.py

@@ -40,16 +40,24 @@ def test_nirspec_mos_spec3(run_pipeline, suffix, source_id, fitsdiff_default_kwa
     diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs)
     assert diff.identical, diff.report()
 
-    if "s2d" in output:
+    if suffix == "s2d":
         # Compare the calculated wavelengths
+        tolerance = 1e-03
         dmt = datamodels.open(rtdata.truth)
         dmr = datamodels.open(rtdata.output)
-        names = [s.name for s in dmt.slits]
-        for name in names:
-            st_idx = [(s.wcs, s.wavelength) for s in dmt.slits if s.name==name]
-            w = dmt.slits[st_idx].meta.wcs
+        if isinstance(dmt, datamodels.MultiSlitModel):
+            names = [s.name for s in dmt.slits]
+            for name in names:
+                st_idx = [(s.wcs, s.wavelength) for s in dmt.slits if s.name==name]
+                w = dmt.slits[st_idx].meta.wcs
+                x, y = wcstools.grid_from_bounding_box(w.bounding_box, step=(1, 1), center=True)
+                _, _, wave = w(x, y)
+                sr_idx = [(s.wcs, s.wavelength) for s in dmr.slits if s.name==name]
+                wlr = dmr.slits[sr_idx].wavelength
+                assert np.all(np.isclose(wave, wlr, atol=tolerance))
+        else:
+            w = dmt.meta.wcs
             x, y = wcstools.grid_from_bounding_box(w.bounding_box, step=(1, 1), center=True)
             _, _, wave = w(x, y)
-            sr_idx = [(s.wcs, s.wavelength) for s in dmr.slits if s.name==name]
-            wlr = dmr.slits[sr_idx].wavelength
-            assert np.all(np.isclose(wave, wlr, atol=1e-03))
+            wlr = dmr.wavelength
+            assert np.all(np.isclose(wave, wlr, atol=tolerance))

jwst/resample/resample.py

@@ -70,7 +70,12 @@ def __init__(self, input_models, output=None, single=False, blendheaders=True,
                 all products in memory.
         """
         self.input_models = input_models
+        self.output_dir = None
         self.output_filename = output
+        if output is not None and '.fits' not in output:
+            self.output_dir = output
+            self.output_filename = None
+
         self.pscale_ratio = pscale_ratio
         self.single = single
         self.blendheaders = blendheaders
@@ -296,6 +301,8 @@ def resample_many_to_many(self):
             if not self.in_memory:
                 # Write out model to disk, then return filename
                 output_name = output_model.meta.filename
+                if self.output_dir is not None:
+                    output_name = os.path.join(self.output_dir, output_name)
                 output_model.save(output_name)
                 log.info(f"Saved model in {output_name}")
                 self.output_models.append(output_name)

jwst/resample/resample_spec.py

@@ -63,6 +63,10 @@ def __init__(self, input_models, output=None, single=False, blendheaders=False,
         self.input_models = input_models
 
         self.output_filename = output
+        self.output_dir = None
+        if output is not None and '.fits' not in output:
+            self.output_dir = output
+            self.output_filename = None
         self.pscale_ratio = pscale_ratio
         self.pscale = pscale
         self.single = single