modifying record_step_status further, adding query_step_status

CHANGES.rst

@@ -73,6 +73,12 @@ combine_1d
 
 - Fix weights for combining errors from 1D spectra. [#8520]
 
+cube_build
+----------
+
+- Removed direct setting of the ``self.skip`` attribute from within the step
+  itself. [#8600]
+
 dark_current
 ------------
 
@@ -364,6 +370,13 @@ saturation
 
 - Adds a check for saturation bias in group 2 for IRS2 mode nframes > 1. [#8593]
 
+stpipe
+------
+
+- Removed setting of the `self.skip` attribute in the `record_step_status()` function,
+  added a `query_step_status()` function to use as an alternative to checking 
+  `self.skip`. [#8600]
+
 tweakreg
 --------
 
@@ -389,6 +402,9 @@ tweakreg
   message and skip ``tweakreg`` step when this condition is not satisfied and
   source confusion is possible during catalog matching. [#8476]
 
+- Removed direct setting of the ``self.skip`` attribute from within the step
+  itself. [#8600]
+
 wavecorr
 --------
 

jwst/cube_build/cube_build_step.py

@@ -3,7 +3,7 @@
 
 from jwst.datamodels import ModelContainer
 
-from ..stpipe import Step
+from ..stpipe import Step, record_step_status
 from . import cube_build
 from . import ifu_cube
 from . import data_types
@@ -396,9 +396,9 @@ def process(self, input):
                 if key in cube.meta.wcsinfo.instance:
                     del cube.meta.wcsinfo.instance[key]
         if status_cube == 1:
-            self.record_step_status(cube_container, "cube_build", success=False)
+            record_step_status(cube_container, "cube_build", success=False)
         else:
-            self.record_step_status(cube_container, "cube_build", success=True)
+            record_step_status(cube_container, "cube_build", success=True)
 
         t1 = time.time()
         self.log.debug(f'Time to build all cubes {t1-t0}')

jwst/master_background/master_background_mos_step.py

@@ -1,5 +1,6 @@
 """Master Background Pipeline for applying Master Background to NIRSpec MOS data"""
 from stpipe.step import preserve_step_pars
+from jwst.stpipe import record_step_status, query_step_status
 
 from stdatamodels.jwst import datamodels
 
@@ -106,9 +107,9 @@ def process(self, data):
             # If some type of background processing had already been done. Abort.
             # UNLESS forcing is enacted.
             if not self.force_subtract and \
-               'COMPLETE' in [data_model.meta.cal_step.back_sub, data_model.meta.cal_step.master_background]:
+               'COMPLETE' in [query_step_status(data_model, "back_sub"), query_step_status(data_model, "master_background")]:
                 self.log.info('Background subtraction has already occurred. Skipping.')
-                self.record_step_status(data, 'master_background', False)
+                record_step_status(data, 'master_background', success=False)
                 return data
 
             if self.user_background:
@@ -123,11 +124,11 @@ def process(self, data):
                 num_bkg, num_src = self._classify_slits(data_model)
                 if num_bkg == 0:
                     self.log.warning('No background slits available for creating master background. Skipping')
-                    self.record_step_status(data, 'master_background', False)
+                    record_step_status(data, 'master_background', False)
                     return data
                 elif num_src == 0:
                     self.log.warning('No source slits for applying master background. Skipping')
-                    self.record_step_status(data, 'master_background', False)
+                    record_step_status(data, 'master_background', False)
                     return data
 
                 self.log.info('Calculating master background')
@@ -136,14 +137,14 @@ def process(self, data):
             # Check that a master background was actually determined.
             if master_background is None:
                 self.log.info('No master background could be calculated. Skipping.')
-                self.record_step_status(data, 'master_background', False)
+                record_step_status(data, 'master_background', False)
                 return data
 
             # Now apply the de-calibrated background to the original science
             result = nirspec_utils.apply_master_background(data_model, mb_multislit, inverse=self.inverse)
 
             # Mark as completed and setup return data
-            self.record_step_status(result, 'master_background', True)
+            record_step_status(result, 'master_background', True)
             self.correction_pars = {
                 'masterbkg_1d': master_background,
                 'masterbkg_2d': mb_multislit

jwst/master_background/master_background_step.py

@@ -4,6 +4,7 @@
 from stdatamodels.jwst import datamodels
 
 from jwst.datamodels import ModelContainer
+from jwst.stpipe import record_step_status
 
 from ..stpipe import Step
 from ..combine_1d.combine1d import combine_1d_spectra
@@ -63,7 +64,7 @@ def process(self, input):
             # First check if we should even do the subtraction.  If not, bail.
             if not self._do_sub:
                 result = input_data.copy()
-                self.record_step_status(result, 'master_background', success=False)
+                record_step_status(result, 'master_background', success=False)
                 return result
 
             # Check that data is a supported datamodel. If not, bail.
@@ -78,7 +79,7 @@ def process(self, input):
                     "Input %s of type %s cannot be handled.  Step skipped.",
                     input, type(input)
                 )
-                self.record_step_status(result, 'master_background', success=False)
+                record_step_status(result, 'master_background', success=False)
                 return result
 
             # If user-supplied master background, subtract it
@@ -159,15 +160,15 @@ def process(self, input):
                         "Input %s of type %s cannot be handled without user-supplied background.  Step skipped.",
                         input, type(input)
                     )
-                    self.record_step_status(result, 'master_background', success=False)
+                    record_step_status(result, 'master_background', success=False)
                     return result
 
                 # Save the computed background if requested by user
                 if self.save_background:
                     self.save_model(master_background, suffix='masterbg1d', force=True, asn_id=asn_id)
                     self.save_model(background_2d_collection, suffix='masterbg2d', force=True, asn_id=asn_id)
 
-            self.record_step_status(result, 'master_background', success=True)
+            record_step_status(result, 'master_background', success=True)
 
         return result
 

jwst/pipeline/calwebb_spec2.py

@@ -5,6 +5,7 @@
 import numpy as np
 
 from stdatamodels.jwst import datamodels
+from jwst.stpipe import query_step_status
 
 from ..assign_wcs.util import NoDataOnDetectorError
 from ..lib.exposure_types import is_nrs_ifu_flatlamp, is_nrs_ifu_linelamp, is_nrs_slit_linelamp
@@ -332,7 +333,8 @@ def process_exposure_product(
             # as DO_NOT_USE or NON_SCIENCE.
             resampled = self.pixel_replace(resampled)
             resampled = self.cube_build(resampled)
-            if resampled[0].meta.cal_step.cube_build == 'COMPLETE':
+            print('TYPE resampled', type(resampled))
+            if query_step_status(resampled, "cube_build") == 'COMPLETE':
                 self.save_model(resampled[0], suffix='s3d')
         elif exp_type in ['MIR_LRS-SLITLESS']:
             resampled = calibrated.copy()
@@ -346,7 +348,7 @@ def process_exposure_product(
             # as DO_NOT_USE or NON_SCIENCE.
             resampled = self.pixel_replace(resampled)
         # Extract a 1D spectrum from the 2D/3D data
-        if exp_type in ['MIR_MRS', 'NRS_IFU'] and resampled[0].meta.cal_step.cube_build == 'SKIPPED':
+        if exp_type in ['MIR_MRS', 'NRS_IFU'] and query_step_status(resampled, "cube_build") == 'SKIPPED':
             # Skip extract_1d for IFU modes where no cube was built
             self.extract_1d.skip = True
 

jwst/pipeline/calwebb_spec3.py

@@ -6,6 +6,7 @@
 from stdatamodels.jwst import datamodels
 
 from jwst.datamodels import SourceModelContainer
+from jwst.stpipe import query_step_status
 
 from ..associations.lib.rules_level3_base import format_product
 from ..exp_to_source import multislit_to_container
@@ -148,7 +149,7 @@ def process(self, input):
 
             # If the step is skipped, do the container splitting that
             # would've been done in master_background
-            if source_models[0].meta.cal_step.master_background == 'SKIPPED':
+            if query_step_status(source_models, "master_background") == 'SKIPPED':
                 source_models, bkg_models = split_container(input_models)
                 # we don't need the background members
                 bkg_models.close()

jwst/pixel_replace/pixel_replace_step.py

@@ -2,6 +2,7 @@
 from functools import partial
 
 from ..stpipe import Step
+from jwst.stpipe import record_step_status
 from .. import datamodels
 from .pixel_replace import PixelReplacement
 
@@ -107,7 +108,7 @@ def process(self, input):
                         replacement = PixelReplacement(model, **pars)
                         replacement.replace()
                         output_model[i] = replacement.output
-                        self.record_step_status(output_model[i], 'pixel_replace', success=True)
+                        record_step_status(output_model[i], 'pixel_replace', success=True)
                 return output_model
             # ________________________________________
             # calewbb_spec2 case - single input model
@@ -117,6 +118,6 @@ def process(self, input):
                 result = input_model.copy()
                 replacement = PixelReplacement(result, **pars)
                 replacement.replace()
-                self.record_step_status(replacement.output, 'pixel_replace', success=True)
+                record_step_status(replacement.output, 'pixel_replace', success=True)
                 result = replacement.output
                 return result

jwst/regtest/test_nirspec_masterbackground.py

@@ -76,6 +76,7 @@ def test_masterbkg_corrpars(rtdata):
         mbs = MasterBackgroundMosStep()
         corrected = mbs.run(data)
 
+        print(corrected.meta.cal_step.__dict__)
         mbs.use_correction_pars = True
         corrected_corrpars = mbs.run(data)
 

jwst/stpipe/__init__.py

@@ -1,4 +1,5 @@
 from .core import JwstStep as Step, JwstPipeline as Pipeline
+from .core import record_step_status, query_step_status
 
 
-__all__ = ['Step', 'Pipeline']
+__all__ = ['Step', 'Pipeline', 'record_step_status', 'query_step_status']

jwst/stpipe/core.py

@@ -89,32 +89,6 @@ def finalize_result(self, result, reference_files_used):
                 if self.parent is None:
                     log.info(f"Results used CRDS context: {result.meta.ref_file.crds.context_used}")
 
-    def record_step_status(self, datamodel, cal_step, success=True):
-        """Record whether or not a step completed in meta.cal_step
-
-        Parameters
-        ----------
-        datamodel : `~jwst.datamodels.JwstDataModel` instance
-            This is the datamodel or container of datamodels to modify in place
-
-        cal_step : str
-            The attribute in meta.cal_step for recording the status of the step
-
-        success : bool
-            If True, then 'COMPLETE' is recorded.  If False, then 'SKIPPED'
-        """
-        if success:
-            status = 'COMPLETE'
-        else:
-            status = 'SKIPPED'
-
-        if isinstance(datamodel, Sequence):
-            for model in datamodel:
-                model.meta.cal_step._instance[cal_step] = status
-        else:
-            datamodel.meta.cal_step._instance[cal_step] = status
-
-        # TODO: standardize cal_step naming to point to the official step name
 
     def remove_suffix(self, name):
         return remove_suffix(name)
@@ -127,3 +101,59 @@ class JwstPipeline(Pipeline, JwstStep):
     def finalize_result(self, result, reference_files_used):
         if isinstance(result, JwstDataModel):
             log.info(f"Results used CRDS context: {crds_client.get_context_used(result.crds_observatory)}")
+
+
+def record_step_status(datamodel, cal_step, success=True):
+    """Record whether or not a step completed in meta.cal_step
+
+    Parameters
+    ----------
+    datamodel : `~jwst.datamodels.JwstDataModel` instance
+        This is the datamodel or container of datamodels to modify in place
+
+    cal_step : str
+        The attribute in meta.cal_step for recording the status of the step
+
+    success : bool
+        If True, then 'COMPLETE' is recorded.  If False, then 'SKIPPED'
+    """
+    if success:
+        status = 'COMPLETE'
+    else:
+        status = 'SKIPPED'
+
+    if isinstance(datamodel, Sequence):
+        for model in datamodel:
+            model.meta.cal_step._instance[cal_step] = status
+    else:
+        datamodel.meta.cal_step._instance[cal_step] = status
+
+    # TODO: standardize cal_step naming to point to the official step name
+
+
+def query_step_status(datamodel, cal_step):
+    """Query the status of a step in meta.cal_step
+    
+    Parameters
+    ----------
+    datamodel : `~jwst.datamodels.JwstDataModel` instance
+        This is the datamodel or container of datamodels to check
+        
+    cal_step : str
+        The attribute in meta.cal_step to check
+    
+    Returns
+    -------
+    status : str
+        The status of the step in meta.cal_step, typically 'COMPLETE' or 'SKIPPED'
+    """
+    if isinstance(datamodel, Sequence):
+        try:
+            return datamodel[0].meta.cal_step._instance[cal_step]
+        except KeyError:
+            return "NOT SET"
+    else:
+        try:
+            return datamodel.meta.cal_step._instance[cal_step]
+        except KeyError:
+            return "NOT SET"

jwst/tweakreg/tweakreg_step.py

@@ -16,6 +16,7 @@
 from tweakwcs.matchutils import XYXYMatch
 
 from jwst.datamodels import ModelContainer
+from jwst.stpipe import record_step_status
 
 # LOCAL
 from ..stpipe import Step
@@ -204,7 +205,7 @@ def process(self, input):
                 self.log.warning("At least two exposures are required for image "
                                  "alignment.")
                 self.log.warning("Nothing to do. Skipping 'TweakRegStep'...")
-                self.record_step_status(input, "tweakreg", success=False)
+                record_step_status(input, "tweakreg", success=False)
                 return input
 
         # === start processing images ===
@@ -309,7 +310,7 @@ def process(self, input):
                     self.log.warning("At least two exposures are required for "
                                      "image alignment.")
                     self.log.warning("Nothing to do. Skipping 'TweakRegStep'...")
-                    self.record_step_status(images, "tweakreg", success=False)
+                    record_step_status(images, "tweakreg", success=False)
                     if not align_to_abs_refcat:
                         return images
                     local_align_failed = True
@@ -325,7 +326,7 @@ def process(self, input):
                                    "matched to a single reference source. Try to "
                                    "adjust 'tolerance' and/or 'separation' parameters.")
                     self.log.warning("Skipping 'TweakRegStep'...")
-                    self.record_step_status(images, "tweakreg", success=False)
+                    record_step_status(images, "tweakreg", success=False)
                     return images
                 else:
                     raise e
@@ -335,7 +336,7 @@ def process(self, input):
                     self.log.warning("Skipping relative alignment (stage 1)...")
                 else:
                     self.log.warning("Skipping 'TweakRegStep'...")
-                    self.record_step_status(images, "tweakreg", success=False)
+                    record_step_status(images, "tweakreg", success=False)
                     return images
 
         if align_to_abs_refcat:
@@ -453,7 +454,7 @@ def process(self, input):
                         )
                         if local_align_failed or n_groups == 1:
                             self.log.warning("Nothing to do. Skipping 'TweakRegStep'...")
-                            self.record_step_status(images, "tweakreg", success=False)
+                            record_step_status(images, "tweakreg", success=False)
                             return images
                     else:
                         raise e
@@ -472,7 +473,7 @@ def process(self, input):
                         )
                         if local_align_failed or n_groups == 1:
                             self.log.warning("Skipping 'TweakRegStep'...")
-                            self.record_step_status(images, "tweakreg", success=False)
+                            record_step_status(images, "tweakreg", success=False)
                             return images
                     else:
                         raise e