merge my merge of release/3.0.x with Chris' merge of the calmgr refac…

…tor.

gemini_instruments/gemini/adclass.py

@@ -209,6 +209,11 @@ def _type_bad_pixel_mask(self):
                                           'GCAL_IR_ON', 'GCAL_IR_OFF', 'DARK',
                                            'NON_SIDEREAL', 'AZEL_TARGET'])
 
+    @astro_data_tag
+    def _type_mos_mask(self):
+        if self.phu.get('OBSTYPE', '').upper() == "MASK":
+            return TagSet(['MASK'], blocks=['IMAGE', 'SPECT'])
+
     @astro_data_tag
     def _status_raw(self):
         if 'GEM-TLM' not in self.phu:

gemini_instruments/gmos/adclass.py

@@ -124,10 +124,9 @@ def _tag_ifu(self):
             return TagSet(['IFU', mapping[mskn]])
 
     @astro_data_tag
-    def _tag_mask(self):
-        spg = self.phu.get
-        if spg('GRATING') == 'MIRROR' and spg('MASKTYP') != 0:
-            return TagSet(['MASK'])
+    def _tag_thruslit(self):
+        if self.phu.get('MASKTYP') != 0:
+            return TagSet(['THRUSLIT'], if_present=['IMAGE'])
 
     @astro_data_tag
     def _tag_image_or_spect(self):

gemini_instruments/gnirs/adclass.py

@@ -41,9 +41,9 @@ def _type_image(self):
             return TagSet(['IMAGE'])
 
     @astro_data_tag
-    def _type_mask(self):
+    def _type_thruslit(self):
         if 'Acq' not in self.phu.get('SLIT', ''):
-            return TagSet(['MASK'], if_present=['ACQUISITION'])
+            return TagSet(['THRUSLIT'], if_present=['IMAGE'])
 
     @astro_data_tag
     def _type_spect(self):

geminidr/__init__.py

@@ -5,10 +5,6 @@
 This module provides the caches library to primitives, but currently, only
 Bookkeeping uses the cache directly (addToList).
 
-This module now provides the Calibrations class, formerly part of cal_service.
-Calibrations() also uses the caches functions, which are now directly available
-here.
-
 E.g.,
 >>> from geminidr import PrimitivesBASE
 
@@ -37,23 +33,21 @@
 from .gemini.lookups import timestamp_keywords
 from .gemini.lookups.source_detection import sextractor_dict
 
-from recipe_system.cal_service import calurl_dict
+from recipe_system.cal_service import init_calibration_databases
 from recipe_system.utils.decorators import parameter_override
+from recipe_system.config import load_config
 
 import atexit
 # ------------------------------ caches ---------------------------------------
 # Formerly in cal_service/caches.py
 #
 # GLOBAL/CONSTANTS (could be exported to config file)
-CALS = "calibrations"
 
 # [caches]
 caches = {
     'reducecache': '.reducecache',
-    'calibrations': CALS
 }
 
-calindfile = os.path.join('.', caches['reducecache'], "calindex.pkl")
 stkindfile = os.path.join('.', caches['reducecache'], "stkindex.pkl")
 
 
@@ -80,52 +74,6 @@ def save_cache(obj, cachefile):
 # ------------------------- END caches-----------------------------------------
 
 
-class Calibrations:
-    def __init__(self, calindfile, user_cals={}, *args, **kwargs):
-        self._calindfile = calindfile
-        self._dict = {}
-        self._dict.update(load_cache(self._calindfile))
-        self._usercals = user_cals or {}  # Handle user_cals=None
-
-    def __getitem__(self, key):
-        return self._get_cal(*key)
-
-    def __setitem__(self, key, val):
-        self._add_cal(key, val)
-
-    def __delitem__(self, key):
-        # Cope with malformed keys
-        try:
-            self._dict.pop((key[0].calibration_key(), key[1]), None)
-        except (TypeError, IndexError):
-            pass
-
-    def _add_cal(self, key, val):
-        # Munge the key from (ad, caltype) to (ad.calibration_key, caltype)
-        key = (key[0].calibration_key(), key[1])
-        self._dict.update({key: val})
-        self.cache_to_disk()
-
-    def _get_cal(self, ad, caltype):
-        key = (ad.calibration_key(), caltype)
-        if key in self._usercals:
-            return self._usercals[key]
-        # If we've stacked images, the datalabel won't match so see if we
-        # can match the original datalabel (without "-STACK")
-        elif isinstance(key[0], str):
-            try:
-                return self._usercals[key[0].replace("-STACK", ""), key[1]]
-            except KeyError:
-                pass
-        calfile = self._dict.get(key)
-        return calfile
-
-    def cache_to_disk(self):
-        save_cache(self._dict, self._calindfile)
-
-# ------------------------------------------------------------------------------
-
-
 class dormantViewer:
     """
     An object that p.viewer can be assigned to, which only creates or connects
@@ -199,23 +147,33 @@ class PrimitivesBASE:
         A list of astrodata objects.
     mode : str
         Operational Mode, one of 'sq', 'qa', 'ql'.
+    ucals : dict
+        user-defined cals, e.g., {"processed_bias": "mybias.fits"}
+    uparms : dict
+        user-defined parameters, e.g., {"stackFrames:reject_method": "sigclip"}
     upload : list
         A list of products to upload to fitsstore.
         QA metrics uploaded if 'metrics' in upload.  E.g.::
 
             upload = ['metrics', ['calibs', ... ]]
 
+    config_file : str/None
+        name of DRAGONS configuration file (None => default)
     """
     tagset = None
 
-    def __init__(self, adinputs, mode='sq', ucals=None, uparms=None, upload=None):
+    def __init__(self, adinputs, mode='sq', ucals=None, uparms=None, upload=None,
+                 config_file=None):
+        # This is a general config file so we should load it now. Some of its
+        # information may be overridden by other parameters passed here.
+        load_config(config_file)
+
         self.streams          = {'main': adinputs}
         self.mode             = mode
         self.params           = {}
         self.log              = logutils.get_logger(__name__)
         self._upload          = upload
         self.user_params      = dict(uparms) if uparms else {}
-        self.calurl_dict      = calurl_dict.calurl_dict
         self.timestamp_keys   = timestamp_keywords.timestamp_keys
         self.keyword_comments = keyword_comments.keyword_comments
         self.sx_dict          = sextractor_dict.sx_dict.copy()
@@ -226,8 +184,10 @@ def __init__(self, adinputs, mode='sq', ucals=None, uparms=None, upload=None):
             for k, v in self.sx_dict.items()
         })
 
+        self.caldb            = init_calibration_databases(
+            getattr(self, "inst_lookups", None), ucals=ucals, upload=upload,
+            procmode=self.mode)
         self.cachedict        = set_caches()
-        self.calibrations     = Calibrations(calindfile, user_cals=ucals)
         self.stacks           = load_cache(stkindfile)
 
         # This lambda will return the name of the current caller.

geminidr/core/parameters_calibdb.py

@@ -5,9 +5,9 @@
 from gempy.library import config
 
 
-class addCalibrationConfig(config.Config):
+class setCalibrationConfig(config.Config):
     caltype = config.ChoiceField(
-        "Type of calibration required",
+        "Type of calibration assigned",
         str,
         allowed={"processed_arc": "processed ARC",
                  "processed_bias": "procsessed BIAS",
@@ -21,53 +21,32 @@ class addCalibrationConfig(config.Config):
     calfile = config.Field("Filename of calibration", str)
 
 
-class getCalibrationConfig(config.Config):
-    caltype = config.ChoiceField(
-        "Type of calibration required",
-        str,
-        allowed={"processed_arc": "processed ARC",
-                 "processed_bias": "procsessed BIAS",
-                 "processed_dark": "processed DARK",
-                 "processed_flat": "processed FLAT",
-                 "processed_fringe": "processed fringe",
-                 "processed_standard": "processed standard",
-                 "processed_slitillum": "processed slitillum",
-                 },
-        optional=False
-    )
-    procmode = config.Field("Processing mode", str, None, optional=True)
-    refresh = config.Field(
-        "Refresh existing calibration associations?", bool, True)
-    howmany = config.RangeField(
-        "Maximum number of calibrations to return", int, None, min=1, optional=True)
-
-
 class getProcessedArcConfig(config.Config):
-    refresh = config.Field("Refresh existing calibration associations?", bool, True)
+    pass
 
 
 class getProcessedBiasConfig(config.Config):
-    refresh = config.Field("Refresh existing calibration associations?", bool, True)
+    pass
 
 
 class getProcessedDarkConfig(config.Config):
-    refresh = config.Field("Refresh existing calibration associations?", bool, True)
+    pass
 
 
 class getProcessedFlatConfig(config.Config):
-    refresh = config.Field("Refresh existing calibration associations?", bool, True)
+    pass
 
 
 class getProcessedFringeConfig(config.Config):
-    refresh = config.Field("Refresh existing calibration associations?", bool, True)
+    pass
 
 
 class getProcessedStandardConfig(config.Config):
-    refresh = config.Field("Refresh existing calibration associations?", bool, True)
+    pass
 
 
 class getProcessedSlitIllumConfig(config.Config):
-    refresh = config.Field("Refresh existing calibration associations?", bool, True)
+    pass
 
 
 class getMDFConfig(config.Config):
@@ -83,9 +62,6 @@ class storeCalibrationConfig(config.Config):
                  "processed_flat": "processed FLAT",
                  "processed_fringe": "processed fringe",
                  "bpm": "bad pixel mask",
-                 "sq": "science quality",
-                 "qa": "QA",
-                 "ql": "quick look",
                  "processed_standard": "processed standard",
                  "processed_slitillum": "processed slitillum",
                  },