Merge pull request #266 from DavidT3/refine/dealWithPreProcessedProperly

Refine/deal with pre processed properly

daxa/exceptions.py

@@ -1,5 +1,5 @@
 #  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-#  Last modified by David J Turner (turne540@msu.edu) 08/04/2024, 20:44. Copyright (c) The Contributors
+#  Last modified by David J Turner (turne540@msu.edu) 16/04/2024, 19:47. Copyright (c) The Contributors
 
 
 class DAXAConfigError(Exception):
@@ -425,3 +425,45 @@ def __str__(self):
         else:
             return 'IncompatibleSaveError has been raised'
 
+
+class PreProcessedNotSupportedError(Exception):
+    def __init__(self, *args):
+        """
+        Exception raised if the user attempts to access pre-processed data for a mission class that does not support
+        it (usually because the data are not available in the archive).
+
+        :param expression:
+        :param message:
+        """
+        if args:
+            self.message = args[0]
+        else:
+            self.message = None
+
+    def __str__(self):
+        if self.message:
+            return '{0} '.format(self.message)
+        else:
+            return 'PreProcessedNotSupportedError has been raised'
+
+
+class PreProcessedNotAvailableError(Exception):
+    def __init__(self, *args):
+        """
+        Exception raised if the user attempts to access a pre-processed product that is not available for the
+        mission - e.g. if they try to access an image for an energy band not present in the archive.
+
+        :param expression:
+        :param message:
+        """
+        if args:
+            self.message = args[0]
+        else:
+            self.message = None
+
+    def __str__(self):
+        if self.message:
+            return '{0} '.format(self.message)
+        else:
+            return 'PreProcessedNotAvailableError has been raised'
+

daxa/files/erass_de_dr1_info.csv

@@ -2445,4 +2445,4 @@ ObsID,OWNER,RA_MIN,RA_MAX,DEC_MIN,DEC_MAX,RA,DEC,X_MIN,Y_MIN,N_NBRS,FIELD1,FIELD
 260177,2,240.0,280.0,-88.5,-85.5,260.0,-87.04518072076759,15.74947087361224,15.300393639120756,7,001180,236174,259174,281174,304174,220177,300177,000000,000000,2020-03-25T16:40:60,2020-04-05T12:43:45
 300177,2,280.0,320.0,-88.5,-85.5,300.0,-87.04518072076759,15.74947087361224,15.300393639120756,7,001180,259174,281174,304174,326174,260177,340177,000000,000000,2020-03-28T16:42:43,2020-04-09T04:42:26
 340177,2,320.0,360.0,-88.5,-85.5,340.0,-87.04518072076759,15.74947087361224,15.300393639120756,7,001180,304174,326174,349174,011174,300177,020177,000000,000000,2020-03-31T20:42:37,2020-04-12T00:44:26
-001180,2,0.0,360.0,-90.0,-88.5,0.0,-90.0,18.0,18.0,9,020177,060177,100177,140177,180177,220177,260177,300177,340177,2020-03-26T00:45:06,2020-04-06T12:45:53x
+001180,2,0.0,360.0,-90.0,-88.5,0.0,-90.0,18.0,18.0,9,020177,060177,100177,140177,180177,220177,260177,300177,340177,2020-03-26T00:45:06,2020-04-06T12:45:53

daxa/files/sas_warnings.csv

@@ -594,4 +594,4 @@ zeroChiSquared,bkgfit
 ZeroEffectiveArea,especget
 ZeroExposureLength,odfingest
 zeroRenorm,arfgen
-zeroSumDetmap,arfgen
+zeroSumDetmap,arfgen

daxa/mission/asca.py

@@ -1,5 +1,5 @@
 #  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-#  Last modified by David J Turner (turne540@msu.edu) 08/04/2024, 21:23. Copyright (c) The Contributors
+#  Last modified by David J Turner (turne540@msu.edu) 23/04/2024, 10:29. Copyright (c) The Contributors
 
 import gzip
 import io
@@ -20,7 +20,7 @@
 from tqdm import tqdm
 
 from daxa import NUM_CORES
-from daxa.exceptions import DAXADownloadError
+from daxa.exceptions import DAXADownloadError, PreProcessedNotSupportedError
 from daxa.mission.base import BaseMission
 
 # There are a lot of directories in the ASCA archives it would seem. There also isn't a great explanation of them
@@ -80,6 +80,40 @@ def __init__(self, insts: Union[List[str], str] = None, save_file_path: str = No
         #  to make sure the input instruments are allowed
         self.chosen_instruments = insts
 
+        # These are the 'translations' required between energy band and filename identifier for ROSAT images/expmaps -
+        #  it is organised so that top level keys are instruments, middle keys are lower energy bounds, and the lower
+        #  level keys are upper energy bounds, then the value is the filename identifier
+        # GIS all -> 0-1023 PI, LO -> 0-170 PI, HI -> 170-1024 PI
+        # SIS all -> 0-2047 PI, LO -> 0-547 PI, HI -> 547-2048 PI
+        # Apparently low is below 2 keV, high is above 2 keV, all is 'everything', might choose 0.4-10 keV for
+        #  SIS, and 0.7-10 keV for GIS
+        self._template_en_trans = {"SIS0": {Quantity(0.4, 'keV'): {Quantity(2, 'keV'): 'lo',
+                                                                   Quantity(10.0, 'keV'): 'all'},
+                                            Quantity(2, 'keV'): {Quantity(10.0, 'keV'): 'hi'}},
+                                   "SIS1": {Quantity(0.4, 'keV'): {Quantity(2, 'keV'): 'lo',
+                                                                   Quantity(10.0, 'keV'): 'all'},
+                                            Quantity(2, 'keV'): {Quantity(10.0, 'keV'): 'hi'}},
+                                   "GIS2": {Quantity(0.7, 'keV'): {Quantity(2, 'keV'): 'lo',
+                                                                   Quantity(10.0, 'keV'): 'all'},
+                                            Quantity(2, 'keV'): {Quantity(10.0, 'keV'): 'hi'}},
+                                   "GIS3": {Quantity(0.7, 'keV'): {Quantity(2, 'keV'): 'lo',
+                                                                   Quantity(10.0, 'keV'): 'all'},
+                                            Quantity(2, 'keV'): {Quantity(10.0, 'keV'): 'hi'}},
+                                   }
+        # This does strip the specific instrument number from the name, which doesn't work for the event lists
+        #  because they are provided individually, but it DOES work for the images and exposure maps as they exist
+        #  as combinations of the paired instruments. That means we only have to override the get_evt_list_path method
+        self._template_inst_trans = {'GIS2': 'gis', 'GIS3': 'gis', 'SIS0': 'sis', 'SIS1': 'sis'}
+
+        # We set up the ROSAT file name templates, so that the user (or other parts of DAXA) can retrieve paths
+        #  to the event lists, images, exposure maps, and background maps that can be downloaded
+        # TODO This isn't completely adequate - there can be different bit rates (h, l, m), AND INDEED MULTIPLE
+        #  sub-exposures of the same bit rate!
+        self._template_evt_name = "screened/ad{oi}{i}*h.evt"
+        self._template_img_name = "images/ad{oi}{i}*_{eb}.totsky"
+        self._template_exp_name = "images/ad{oi}{i}*.totexpo"
+        self._template_bck_name = None
+
         # Call the name property to set up the name and pretty name attributes
         self.name
 
@@ -381,7 +415,7 @@ def _download_call(observation_id: str, insts: List[str], raw_dir: str, download
 
                 # There are a few compressed fits files in each archive, but I think I'm only going to decompress the
                 #  event lists, as they're more likely to be used
-                if 'evt.gz' in down_file:
+                if 'evt.gz' in down_file or 'totsky.gz' in down_file or 'totexpo.gz' in down_file:
                     # Open and decompress the events file
                     with gzip.open(local_dir + down_file, 'rb') as compresso:
                         # Open a new file handler for the decompressed data, then funnel the decompressed events there
@@ -414,6 +448,12 @@ def download(self, num_cores: int = NUM_CORES, download_products: bool = False):
         # Grabs the raw data storage path
         stor_dir = self.raw_data_path
 
+        # We store the type of data that was downloaded
+        if download_products:
+            self._download_type = "raw+preprocessed"
+        else:
+            self._download_type = "raw"
+
         # A very unsophisticated way of checking whether raw data have been downloaded before (see issue #30)
         #  If not all data have been downloaded there are also secondary checks on an ObsID by ObsID basis in
         #  the _download_call method
@@ -491,6 +531,50 @@ def err_callback(err):
         else:
             warn("The raw data for this mission have already been downloaded.")
 
+    def get_evt_list_path(self, obs_id: str, inst: str = None) -> str:
+        """
+        A get method that provides the path to a downloaded pre-generated event list for the current mission (if
+        available). This method will not work if pre-processed data have not been downloaded.
+
+        :param str obs_id: The ObsID of the event list.
+        :param str inst: The instrument of the event list (if applicable).
+        :return: The requested event list path.
+        :rtype: str
+        """
+
+        inst, en_bnd_trans, file_inst, lo_en, hi_en = self._get_prod_path_checks(obs_id, inst)
+
+        # The reason we needed to override the base get method for event list path - the naming scheme for event lists
+        #  is different from images + exposure maps, as those are shipped with SIS0+1 and GIS2+3 added together.
+        if file_inst == 'sis':
+            file_inst = 's' + inst[-1]
+        elif file_inst[0] == 'gis':
+            file_inst = 'g' + inst[-1]
+
+        # The template path can take two forms, one is a straight string and can just be filled in, but the
+        #  other is a dictionary where the keys are instrument names and the values are the string file templates. We
+        #  need to check which is applicable to this mission and treat it accordingly
+        if isinstance(self._template_evt_name, str):
+            rel_pth = os.path.join(self.raw_data_path, obs_id, self._template_evt_name.format(oi=obs_id, i=file_inst))
+        # In some cases the instrument name will have to be supplied, otherwise we will not be able to
+        #  create a path
+        elif isinstance(self._template_evt_name, dict) and inst is None:
+            raise ValueError("The 'inst' argument cannot be None for this mission, as the different instruments have "
+                             "differently formatted pre-processed file names.")
+        # It is possible for only some instruments of a mission to have images, so we check
+        elif isinstance(self._template_evt_name, dict) and self._template_evt_name[inst] is None:
+            raise PreProcessedNotSupportedError("This mission ({m}) does not support the download of pre-processed "
+                                                "event lists for the {i} instrument, so a path cannot be "
+                                                "provided.".format(m=self.pretty_name, i=inst))
+        elif isinstance(self._template_evt_name, dict):
+            rel_pth = os.path.join(self.raw_data_path, obs_id, self._template_evt_name[inst].format(oi=obs_id,
+                                                                                                    i=file_inst))
+
+        # This performs certain checks to make sure the file exists, and fill in any wildcards
+        rel_pth = self._get_prod_path_post_checks(rel_pth, obs_id, inst, 'event list')
+
+        return rel_pth
+
     def assess_process_obs(self, obs_info: dict):
         """
         A slightly unusual method which will allow the ASCA mission to assess the information on a particular
@@ -523,6 +607,8 @@ def ident_to_obsid(self, ident: str):
 
         :param str ident: The unique identifier used in a particular processing step.
         """
-        raise NotImplementedError("The check_process_obs method has not yet been implemented for {n}, as it isn't yet"
-                                  "clear to me what form the unique identifiers will take once we start processing"
-                                  "{n} data ourselves.".format(n=self.pretty_name))
+        # raise NotImplementedError("The check_process_obs method has not yet been implemented for {n}, as it isn't yet"
+        #                           "clear to me what form the unique identifiers will take once we start processing"
+        #                           "{n} data ourselves.".format(n=self.pretty_name))
+        # All ASCA ObsIDs are 8 digits, so that is what we select
+        return ident[:8]