Merge branch 'feature/recipe-group'

igrins/libs/cal_db.py

@@ -97,7 +97,7 @@ def load_db(self, db_name):
     def db_query_basename(self, db_name, basename):
         band, master_obsid = self._get_band_masterobsid(basename)
         db = self.load_db(db_name)
-        resource_basename = db.query(band, master_obsid)
+        resource_basename = db.query(band, int(master_obsid))
         return resource_basename
 
 
@@ -109,7 +109,9 @@ def query_item_path(self, basename, item_type_or_desc,
 
         if not isinstance(basename, str):
             band, master_obsid = basename
-            basename = self.helper.get_basename(band, master_obsid)
+            # basename = self.helper.get_basename(band, master_obsid)
+            basename = self.helper.get_basename_with_groupname(band,
+                                                               master_obsid)
 
         if basename_postfix is not None:
             basename += basename_postfix
@@ -177,56 +179,78 @@ def _get_basename_old(self, band, master_obsid):
 
     def _get_basename(self, basename):
         if not isinstance(basename, str):
-            band, master_obsid = basename
-            basename = self.helper.get_basename(band, master_obsid)
+            band, groupname = basename
+            basename = self.helper.get_basename_with_groupname(band,
+                                                               groupname)
 
         return basename
 
     def _get_band_masterobsid(self, basename):
         if isinstance(basename, str):
-            sdc_, utdate, masterobsid_ = basename.split("_")
+            sdc_, utdate, masterobsid = basename.split("_")
             band = sdc_[-1]
-            masterobsid = int(masterobsid_)
+            # masterobsid = int(masterobsid_)
         else:
             band, masterobsid = basename
 
         return band, masterobsid
 
+    def _get_master_hdu(self, basename, master_hdu):
+        if master_hdu is None:
+            band, master_obsid = self._get_band_masterobsid(basename)
+            mastername = self.helper.get_filename(band, master_obsid)
+            master_hdu = self.helper.get_masterhdu(mastername)
+        elif isinstance(master_hdu, int):
+            band, master_obsid = self._get_band_masterobsid(basename)
+            # interprete master_hdu as master_obsid
+            master_obsid = master_hdu
+            master_hdu = self.helper.get_masterhdu(mastername)
+        else:
+            pass
+
+        return master_hdu
+
     def store_image(self, basename, item_type, data,
+                    master_hdu=None,
                     header=None, card_list=None):
-        band, master_obsid = self._get_band_masterobsid(basename)
-        basename = self._get_basename(basename)
+        from products import PipelineImageBase
 
         item_desc = self.DESC_DICT[item_type.upper()]
 
-        from products import PipelineImageBase
-        mastername = self.helper.get_filenames(band, [master_obsid])[0]
+        # band, master_obsid = self._get_band_masterobsid(basename)
+        # basename = self._get_basename(basename)
+
+        # mastername = self.helper.get_filenames(band, [master_obsid])[0]
+
+        # hdu = self.helper.get_masterhdu(mastername)
+
+        master_hdu = self._get_master_hdu(basename, master_hdu)
+
+        # if header is not None:
+        #     master_hdu.header = header
 
-        hdu = self.helper.get_masterhdu(mastername)
-        if header is not None:
-            hdu.header = header
         if card_list is not None:
-            hdu.header.extend(card_list)
+            master_hdu.header.extend(card_list)
 
         pipeline_image = PipelineImageBase([], data,
-                                           masterhdu=hdu)
+                                           masterhdu=master_hdu)
 
         self.helper.store_item(item_desc, basename,
                                pipeline_image)
 
-    def store_multi_image(self, basename, item_type, hdu_list,
+    def store_multi_image(self, basename,
+                          item_type, hdu_list,
+                          master_hdu=None,
                           basename_postfix=None):
-        band, master_obsid = self._get_band_masterobsid(basename)
-        basename = self._get_basename(basename)
+        # basename = self._get_basename(basename)
 
         item_desc = self.DESC_DICT[item_type.upper()]
 
-        from products import PipelineImages
-        mastername = self.helper.get_filenames(band, [master_obsid])[0]
-        hdu = self.helper.get_masterhdu(mastername)
+        master_hdu = self._get_master_hdu(basename, master_hdu)
 
+        from products import PipelineImages
         pipeline_image = PipelineImages(hdu_list,
-                                        masterhdu=hdu)
+                                        masterhdu=master_hdu)
 
         self.helper.store_item(item_desc, basename,
                                pipeline_image,

igrins/libs/obs_set.py

@@ -1,17 +1,23 @@
 class ObsSet(object):
-    def __init__(self, caldb, band, recipe_name, obsids, frametypes):
+    def __init__(self, caldb, band, recipe_name, obsids, frametypes,
+                 groupname=None):
         self.caldb = caldb
         self.recipe_name = recipe_name
         self.band = band
         self.obsids = obsids
         self.frametypes = frametypes
-        self.basename = self.caldb._get_basename((self.band, self.obsids[0]))
+        if groupname is None:
+            groupname = str(self.obsids[0])
+        self.basename = self.caldb._get_basename((self.band, groupname))
+        # this is for query
+        self.basename_for_query = self.caldb._get_basename((self.band, 
+                                                            obsids[0]))
 
     def get_config(self):
         return self.caldb.get_config()
 
     def get(self, name):
-        return self.caldb.get(self.basename, name)
+        return self.caldb.get(self.basename_for_query, name)
 
     def get_base_info(self):
         return self.caldb.get_base_info(self.band, self.obsids)
@@ -58,7 +64,8 @@ def load_db(self, db_name):
 
     def query_item_path(self, item_type_or_desc,
                         basename_postfix=None, subdir=None):
-        return self.caldb.query_item_path(self.basename, item_type_or_desc,
+        return self.caldb.query_item_path(self.basename_for_query, 
+                                          item_type_or_desc,
                                           basename_postfix=basename_postfix,
                                           subdir=subdir)
 
@@ -82,7 +89,7 @@ def load_image(self, item_type):
 
     def _load_item_from(self, item_type_or_desc,
                         basename_postfix=None):
-        return self.caldb.load_item_from(self, self.basename,
+        return self.caldb.load_item_from(self.basename,
                                          item_type_or_desc,
                                          basename_postfix=basename_postfix)
 
@@ -116,7 +123,8 @@ def store_multi_images(self, item_type, hdu_list,
 
     def load_resource_for(self, resource_type,
                           get_science_hdu=False):
-        return self.caldb.load_resource_for(self.basename, resource_type,
+        return self.caldb.load_resource_for(self.basename_for_query,
+                                            resource_type,
                                             get_science_hdu=get_science_hdu)
 
     # Ref data related

igrins/libs/path_info.py

@@ -3,10 +3,26 @@
 from os.path import join
 import astropy.io.fits as pyfits
 
+import re
+groupname_pattern = re.compile(r"(\d+)(\D.*)?")
+
 def ensure_dir(d):
     if not os.path.exists(d):
         os.makedirs(d)
 
+def get_zeropadded_groupname(groupname):
+    if isinstance(groupname, int):
+        groupname = "%04d" % groupname
+    else:
+        m = groupname_pattern.match(groupname)
+        if m:
+            m1, m2 = m.groups()
+            groupname = "%04d%s" % (int(m1), m2 if m2 else "")
+        else:
+            pass
+
+    return groupname
+
 class IGRINSPath(object):
     # IGRINS_CALIB_PATH="calib"
     # IGRINS_REDUCE_DATE=""
@@ -36,7 +52,9 @@ def __init__(self, config, utdate, ensure_dir=False):
 
         # filename pattern for input files
         self.fn_pattern = join(self.sections["INDATA_PATH"],
-                               "SDC%%s_%s_%%04d.fits" % (self.utdate,))
+                               "SDC%%s_%s_%%s.fits" % (self.utdate,))
+
+        self.basename_pattern = "SDC%%s_%s_%%s" % (self.utdate,)
 
         if ensure_dir:
             self.ensure_dir()
@@ -80,7 +98,13 @@ def get_filenames(self, band, runids):
         return [self.get_filename(band, i) for i in runids]
 
     def get_filename(self, band, runid):
-        return self.fn_pattern % (band, runid)
+        groupname = get_zeropadded_groupname(runid)
+        return self.fn_pattern % (band, groupname)
+
+    def get_basename(self, band, groupname):
+        groupname = get_zeropadded_groupname(groupname)
+        basename = self.basename_pattern % (band, groupname)
+        return basename
 
     def get_hdus(self, band, runids):
         fn_list = self.get_filenames(band, runids)

igrins/libs/recipe_base.py

@@ -1,5 +1,57 @@
 #from products import PipelineProducts
 
+def _parse_starting_obsids(starting_obsids):
+    if starting_obsids is not None:
+        starting_obsids = map(int, starting_obsids.split(","))
+        return starting_obsids
+    else:
+        return None
+
+def _parse_groups(groups):
+    if groups is not None:
+        groups = [s.strip() for s in groups.split(",")]
+        return groups
+    else:
+        return None
+
+def filter_a0v(a0v, a0v_obsid, group2):
+
+    # print master_obsid, a0v_obsid
+    if a0v is not None:
+        if a0v_obsid is not None:
+            raise ValueError("a0v-obsid option is not allowed "
+                             "if a0v opption is used")
+        elif str(a0v).upper() == "GROUP2":
+            a0v = group2
+    else:
+        if a0v_obsid is not None:
+            a0v = a0v_obsid
+        else:
+            # a0v, a0v_obsid is all None. Keep it as None
+            pass
+
+    return a0v
+
+def get_selected(recipes, recipe_name, starting_obsids, groups):
+    if starting_obsids is not None:
+        print ("'starting-obsids' option is deprecated, "
+               "please use 'groups' option.")
+        if groups is not None:
+            raise ValueError("'starting-obsids' option is not allowed"
+                             " when 'groups' option is used.")
+        else:
+            starting_obsids_parsed = _parse_starting_obsids(starting_obsids)
+
+            selected = recipes.select_fnmatch(recipe_name,
+                                              starting_obsids_parsed)
+    else:
+        groups_parsed = _parse_groups(groups)
+
+        selected = recipes.select_fnmatch_by_groups(recipe_name,
+                                                    groups_parsed)
+
+    return selected
+
 class RecipeBase(object):
     """ The derived mus define RECIPE_NAME attribute and must implement
         run_selected_bands method.
@@ -26,24 +78,21 @@ def get_recipes(self, utdate):
         from recipes import Recipes #load_recipe_list, make_recipe_dict
         return Recipes(fn)
 
-    def parse_starting_obsids(self, starting_obsids):
-        if starting_obsids is not None:
-            starting_obsids = map(int, starting_obsids.split(","))
-            return starting_obsids
-        else:
-            return None
-
     def run_selected_bands_with_recipe(self, utdate, selected, bands):
         # just ignore recipe
         selected2 = [_[1:] for _ in selected]
         self.run_selected_bands(utdate, selected2, bands)
 
     def __call__(self, utdate, bands="HK",
-                 starting_obsids=None, config_file="recipe.config"):
-        self.process(utdate, bands, starting_obsids, config_file)
+                 starting_obsids=None, groups=None,
+                 config_file="recipe.config"):
+        self.process(utdate, bands, starting_obsids, groups,
+                     config_file=config_file)
 
     def process(self, utdate, bands="HK",
-                starting_obsids=None, config_file="recipe.config",
+                starting_obsids=None, 
+                groups=None,
+                config_file="recipe.config",
                 **kwargs):
 
         from igrins_config import IGRINSConfig
@@ -55,10 +104,8 @@ def process(self, utdate, bands="HK",
 
         recipes = self.get_recipes(utdate)
 
-        starting_obsids_parsed = self.parse_starting_obsids(starting_obsids)
-
-        selected = recipes.select_fnmatch(self.RECIPE_NAME,
-                                          starting_obsids_parsed)
+        selected = get_selected(recipes, self.RECIPE_NAME,
+                                starting_obsids, groups)
 
         self.run_selected_bands_with_recipe(utdate, selected, bands,
                                             **kwargs)

igrins/libs/recipe_factory.py

@@ -14,8 +14,10 @@ def run_selected_bands_with_recipe(self, utdate, selected, bands,
                 obsids = s[1]
                 frame_types = s[2]
                 aux_infos = s[3]
+                groupname = aux_infos["GROUP1"]
 
-                self.process_band(utdate, recipe_name, band, 
+                self.process_band(utdate, recipe_name, band,
+                                  groupname,
                                   obsids, frame_types, aux_infos,
                                   self.config, **kwargs)
 
@@ -29,12 +31,15 @@ def new_recipe_func(function_name, recipe_cls):
 
     def _recipe_func(utdate, bands="HK",
                      starting_obsids=None,
+                     groups=None,
                      config_file="recipe.config",
                      **kwargs):
 
         _recipe_obj = recipe_cls()
         _recipe_obj.process(utdate, bands,
-                            starting_obsids, config_file,
+                            starting_obsids, 
+                            groups,
+                            config_file,
                             **kwargs)
 
     _recipe_func.__name__ = function_name.lower()

igrins/libs/recipe_helper.py

@@ -30,11 +30,19 @@ def get_caldb(self):
     def get_filenames(self, band, obsids):
         return self._igr_path.get_filenames(band, obsids)
 
+    def get_filename(self, band, obsid):
+        return self._igr_path.get_filename(band, obsid)
+
     def get_basename(self, band, master_obsid):
         filenames = self.get_filenames(band, [master_obsid])
         basename = os.path.splitext(os.path.basename(filenames[0]))[0]
         return basename
 
+    def get_basename_with_groupname(self, band, groupname):
+        if isinstance(groupname, int):
+            groupname = str(groupname)
+        return self._igr_path.get_basename(band, groupname)
+
     def get_base_info(self, band, obsids):
         filenames = self.get_filenames(band, obsids)
         basename = os.path.splitext(os.path.basename(filenames[0]))[0]