moved extras to self._extra_table

linetools/abund/ions.py

@@ -23,23 +23,6 @@
 from linetools.abund import roman
 
 ########################## ##########################
-########################## ##########################
-def ion_name(ion, flg=0, nspace=None):
-    """ Wrapper to ion_to_name for backwards compatability
-    WILL BE DEPRECATED
-    Parameters
-    ----------
-    ion
-    flg
-    nspace
-
-    Returns
-    -------
-
-    """
-    import pdb; pdb.set_trace()
-    warnings.warn("This method will be DEPRECATED.  Use ion_to_name")
-    return ion_to_name(ion, flg=flg, nspace=nspace)
 
 def ion_to_name(ion, flg=0, nspace=None):
     """ Convert ion tuple into a string

linetools/isgm/abscomponent.py

@@ -348,7 +348,7 @@ def add_abslines_from_linelist(self, llist='ISM', init_name=None, wvlim=None, mi
 
         """
         # get the transitions from LineList
-        llist = LineList(llist, extras=True)
+        llist = LineList(llist)
         if init_name is None:  # we have to guess it
             if (self.Zion) == (-1, -1):  # molecules
                 # init_name must be in self.attrib (this is a patch)

linetools/lists/linelist.py

@@ -14,7 +14,7 @@
 
 from astropy import units as u
 from astropy.units.quantity import Quantity
-from astropy.table import Table, vstack, Column, MaskedColumn
+from astropy.table import Table, hstack
 import warnings
 import pdb
 
@@ -67,7 +67,7 @@ class LineList(object):
     sort_by : str or list of str, optional
         Keys to sort the underlying data table by. Default is 'wrest'
 
-    redo_extra_columns : bool, optional
+    redo_extra: bool, optional
         Whether to recalculate extra columns for log(w*f), abundance, and ion_correction.
         Setting this to True is useful if a different abundance, or ionizatation_correction
         is used. Default is False.
@@ -76,8 +76,7 @@ class LineList(object):
 
     # Init
     def __init__(self, llst_key, verbose=False, closest=False, set_lines=True,
-                 use_ISM_table=True, use_cache=True, sort_by='wrest',
-                 redo_extra_columns=False, extras=False):
+                 use_cache=True, sort_by='wrest', redo_extra=False):
 
         # Error catching
         if not isinstance(llst_key, basestring):
@@ -90,18 +89,17 @@ def __init__(self, llst_key, verbose=False, closest=False, set_lines=True,
         self.closest = closest
         self.verbose = verbose
 
+        # Load Data
         self.load_data()
         '''
         if not use_ISM_table or llst_key not in ('ISM', 'HI', 'Strong'):
-            # Load Data
             self.load_data(use_cache=use_cache)
         '''
 
         if set_lines:
             # Set lines for use (from defined LineList)
             # This sets self._data
-            self.set_lines(use_ISM_table=use_ISM_table, verbose=verbose,
-                           use_cache=use_cache)
+            self.set_lines(verbose=verbose, use_cache=use_cache)
         else:
             self._data = None
 
@@ -110,10 +108,12 @@ def __init__(self, llst_key, verbose=False, closest=False, set_lines=True,
 
         # Sort
         self.sort_by = sort_by
+        # Extras
+        self._extra_table = Table(masked=True)
         if self._data is not None:
             # redo extra columns?
-            if extras:
-                self.set_extra_columns_to_datatable(redo=redo_extra_columns)
+            self._extra_table['Id_ex'] = self._data['Id']
+            self.make_extra_table(redo=redo_extra)
             # sort the LineList
             self.sortdata(sort_by)
 
@@ -267,6 +267,7 @@ def set_lines(self, verbose=True, use_cache=True, use_ISM_table=True):
         if use_cache and key in CACHE['data']:
             self._data = CACHE['data'][key]
             return
+        '''
         elif use_ISM_table and self.list in ('ISM', 'Strong', 'HI'):
             ism_file = resource_filename('linetools', 'data/lines/ISM_table.fits')
             data = Table.read(ism_file)
@@ -277,6 +278,7 @@ def set_lines(self, verbose=True, use_cache=True, use_ISM_table=True):
                 self._data = data
             CACHE['data'][key] = self._data
             return
+        '''
 
         indices = []
         set_flags = []
@@ -337,15 +339,17 @@ def set_lines(self, verbose=True, use_cache=True, use_ISM_table=True):
         # Parse and sort (consider masking instead)
         tmp_tab = self._fulltable[all_idx]
         tmp_tab.sort('wrest')
+        tmp_tab['Id'] = np.arange(len(tmp_tab)).astype(int)
 
-        #
+        # Finish
         self._data = tmp_tab
         CACHE['data'][key] = self._data
 
 
-    def set_extra_columns_to_datatable(self, abundance_type='solar', ion_correction='none',
+    def make_extra_table(self, abundance_type='solar', ion_correction='none',
                                        redo=False):
-        """Sets new convenient columns to the self._data Table. These will be useful
+        """Build an additional table that is parallel to self._data that
+        includes convenient columns. These will be useful
         for sorting the underlying data table in convenient ways, e.g. by expected
         relative strength, abundance, etc.
 
@@ -385,7 +389,7 @@ def set_extra_columns_to_datatable(self, abundance_type='solar', ion_correction=
 
         """
         # Avoid redo (especially for caching)
-        if ('ion_name' in self._data.keys()) and (redo is False):
+        if ('ion_name' in self._extra_table.keys()) and (redo is False):
             return
 
         if self.list not in ['HI', 'ISM', 'EUV', 'Strong', 'H2']:
@@ -400,7 +404,7 @@ def set_extra_columns_to_datatable(self, abundance_type='solar', ion_correction=
         elif self.list in ['H2']:
             for kk,name in enumerate(self.name):  # H2
                 ion_name[kk] = name.split('(')[0]
-        self._data['ion_name'] = ion_name
+        self._extra_table['ion_name'] = ion_name
 
         if self.list in ['H2']:
             # we want Jk to be 1 or 0 first
@@ -409,22 +413,22 @@ def set_extra_columns_to_datatable(self, abundance_type='solar', ion_correction=
             # second level: Jk = {2,3}
             cond = (self._data['Jk'] > 1) & (self._data['Jk'] <= 3)
             rel_strength = np.where(cond, 50, rel_strength)
-            self._data['rel_strength'] = rel_strength
+            self._extra_table['rel_strength'] = rel_strength
             return
 
         # Set QM strength as MaskedColumn (self._data['f'] is MaskedColumn)
         qm_strength = self._data['f'] * self._data['wrest']
         qm_strength.name = 'qm_strength'
-        self._data['log(w*f)'] = np.log10(qm_strength)
+        self._extra_table['log(w*f)'] = np.log10(qm_strength)
         # mask out potential nans
-        cond = np.isnan(self._data['log(w*f)'])
-        self._data['log(w*f)'].mask = np.where(cond, True, self._data['log(w*f)'].mask)
+        cond = np.isnan(self._extra_table['log(w*f)'])
+        self._extra_table['log(w*f)'].mask = np.where(cond, True, self._extra_table['log(w*f)'].mask)
 
         # Set Abundance
         if abundance_type not in ['none', 'solar']:
             raise ValueError('set_extra_columns_to_datatable: `abundance type` has to be either: `none` or `solar`')
         if abundance_type == 'none':
-            self._data['abundance'] = np.zeros(len(self._data))
+            self._extra_table['abundance'] = np.zeros(len(self._data))
         elif abundance_type == 'solar':
             from linetools.abund.solar import SolarAbund
             solar = SolarAbund()
@@ -434,21 +438,21 @@ def set_extra_columns_to_datatable(self, abundance_type='solar', ion_correction=
                 abund[Zmt] = row['Abund']
                 abund.mask[Zmt] = False
             # Deuterium is special
-            fchar = np.array([ion_name[0] for ion_name in self._data['ion_name'].data])
+            fchar = np.array([ion_name[0] for ion_name in self._extra_table['ion_name'].data])
             DI = fchar == 'D'
             abund[DI] = solar['D']
             # Finish
-            self._data['abundance'] = abund
+            self._extra_table['abundance'] = abund
 
         # Set ionization correction
         if ion_correction not in ['none']:
             raise ValueError('set_extra_columns_to_datatable: `ion_correction` '
                              'has to be `none`.')
         if ion_correction == 'none':
-            self._data['ion_correction'] = np.zeros(len(self._data))  # is in log10 scale, so 0 means no-correction
+            self._extra_table['ion_correction'] = np.zeros(len(self._data))  # is in log10 scale, so 0 means no-correction
 
         # Set relative strength in log10 scale
-        self._data['rel_strength'] = self._data['log(w*f)'] + self._data['abundance'] + self._data['ion_correction']
+        self._extra_table['rel_strength'] = self._extra_table['log(w*f)'] + self._extra_table['abundance'] + self._extra_table['ion_correction']
 
     def sortdata(self, keys, reverse=False):
         """Sort the LineList according to a given key or keys.
@@ -472,14 +476,31 @@ def sortdata(self, keys, reverse=False):
         if keys[0] == 'as_given':
             return
 
+        # hstack for convenience
+        if len(self._extra_table) == len(self._data):
+            flg_extra = True
+            dkeys = self._data.keys()
+            ekeys = self._extra_table.keys()
+            dtbl = hstack([self._data, self._extra_table], join_type='exact')
+        else:
+            flg_extra = False
+            dtbl = self._data
+
         # sort
-        self._data.sort(keys)
+        dtbl.sort(keys)
 
         # reverse?
         if reverse:
-            self._data.reverse()
+            dtbl.reverse()
         self.sort_by = keys
 
+        # Finish
+        if flg_extra:
+            self._data = dtbl[dkeys]
+            self._extra_table = dtbl[ekeys]
+        else:
+            self._data = dtbl
+
     def subset_lines(self, subset, reset_data=False, verbose=False, sort_by=['wrest']):
         """ Select a user-specific subset of the lines from the LineList
 
@@ -621,7 +642,7 @@ def all_transitions(self, line, debug=False):
         if line == 'unknown':
             return self.unknown_line()
         if self.list in ['H2']:
-            cond = self._data['ion_name'] == line.split('(')[0]
+            cond = self._extra_table['ion_name'] == line.split('(')[0]
             tbl = self._data[cond]  # cond is a masked boolean array
             if len(tbl) > 1:
                 return tbl
@@ -630,7 +651,7 @@ def all_transitions(self, line, debug=False):
 
         else:
             Z = None
-            indices = np.where(self._data['ion_name'] == line)[0]
+            indices = np.where(self._extra_table['ion_name'] == line)[0]
             if len(indices) == 0:
                 raise ValueError(
                     'Line {} not found in the LineList: {}'.format(line, self.list))
@@ -658,7 +679,7 @@ def all_transitions(self, line, debug=False):
                 return lilu.from_table_to_dict(tbl)
 
     def strongest_transitions(self, line, wvlims, n_max=3, verbose=False,
-                              data=None, debug=False):
+                              debug=False):
         """ Find the strongest transition for a single ion
         available_transitions() finds those for all ions in a wavelength interval
 
@@ -732,7 +753,8 @@ def strongest_transitions(self, line, wvlims, n_max=3, verbose=False,
             # remove transitions out of range
             data = data[cond]
             # sort by relative strength
-            sorted_inds = np.argsort(data['rel_strength'])
+            idx = data['Id'].data
+            sorted_inds = np.argsort(self._extra_table['rel_strength'][idx])
             # reverse sorted indices, so strongest get first
             sorted_inds = sorted_inds[::-1]
             # sort using sorted_inds
@@ -773,6 +795,7 @@ def available_transitions(self, wvlims, n_max_tuple=None, min_strength=0.):
         -------
         dict (if only 1 transition found) or Table (if > 1
         transitions are found) or None (if no transition is found)
+        This is an hstack of self._data and self._extra_table
         """
 
         if self.list not in ['HI', 'ISM', 'EUV', 'Strong']:
@@ -790,7 +813,9 @@ def available_transitions(self, wvlims, n_max_tuple=None, min_strength=0.):
         # Cut on wavelength first!
         gdwv = (self.wrest>wvlims[0]) & (self.wrest < wvlims[1])
         if np.any(gdwv):
-            data = self._data[gdwv]
+            tmp_data = self._data[gdwv]
+            extras = self._extra_table[gdwv]
+            data = hstack([tmp_data,extras])
         else:
             return None
 
@@ -802,7 +827,7 @@ def available_transitions(self, wvlims, n_max_tuple=None, min_strength=0.):
         # Identify unique ion_names (e.g. HI, CIV, CIII)
         # This is WRONG for FeII*
         unique_ion_names, idx = np.unique(data['ion_name'], return_index=True)
-        unique = data[idx]
+        unique= data[idx]
 
         '''
         # obtain the strongest available transition of a given unique ion species
@@ -828,7 +853,7 @@ def available_transitions(self, wvlims, n_max_tuple=None, min_strength=0.):
 
         # get rid of those below the min_strength threshold
         cond = unique['rel_strength'] >= min_strength
-        unique = unique[cond]
+        unique = unique[np.where(cond == True)[0]] # Deals with masking
         if len(unique) < 1:
             return None
 

linetools/lists/tests/test_init_linelist.py

@@ -32,11 +32,13 @@ def test_updgamma():
     ism = LineList('ISM')
     np.testing.assert_allclose(ism['HI 1215']['gamma'], 626500000.0/u.s)
 
+
 # Strong ISM LineList
 def test_strong():
     strng = LineList('Strong')
     assert len(strng._data) < 200
 
+
 # Strong ISM LineList
 def test_euv():
     euv = LineList('EUV')
@@ -97,9 +99,7 @@ def test_set_extra_columns_to_datatable():
     #    ism.set_extra_columns_to_datatable(ion_correction='incorrect_one', redo=True)
     # test expected strongest value
     ism = LineList('ISM')
-    ism.set_extra_columns_to_datatable(ion_correction='none', abundance_type='solar', redo=True)
-    np.testing.assert_allclose(ism['HI 1215']['rel_strength'], 14.704326420257642)
-    tab = ism._data
-
+    #np.testing.assert_allclose(ism['HI 1215']['rel_strength'], 14.704326420257642)  # THIS IS NO LONGER SUPPORTED
+    tab = ism._extra_table
     np.testing.assert_allclose(np.max(tab['rel_strength']), 14.704326420257642)
 

linetools/lists/tests/test_use_linelist.py

@@ -49,7 +49,7 @@ def test_closest():
 
 def test_all_transitions():
     error_msg = 'Something is wrong in all_transitions()'
-    ism = LineList('ISM', extras=True)
+    ism = LineList('ISM')
     #check simple case
     line = 'OVI'
     ovi_transitions = ism.all_transitions(line)
@@ -71,7 +71,7 @@ def test_all_transitions():
     out = ism.all_transitions(1215.6700*u.AA)
     assert len(out) == 30,"wrong line counts" # 30 Lyman series transitions
     #print('test_all_transitions() passed')
-    h2 = LineList('H2', extras=True)
+    h2 = LineList('H2')
     line = 'B19-0P(1)'
     out = h2.all_transitions(line)
     assert len(out) == 7
@@ -101,7 +101,7 @@ def test_strongest_transitions():
 
 def test_available_transitions():
     error_msg = 'Something is wrong in available_transitions()'
-    ism = LineList('ISM', extras=True)
+    ism = LineList('ISM')
     wvlims = (900,1800)*u.AA
     z = 0.1
 
@@ -131,10 +131,10 @@ def test_available_transitions():
     transitions = ism.available_transitions(wvlims/(1+z),n_max_tuple=2)
     assert isinstance(transitions,dict), error_msg
 
-'''
+
 def test_sortdata():  # With extras
     error_msg = 'Something is wrong in sortdata()'
-    ism = LineList('ISM', sort_by='name', extras=True)
+    ism = LineList('ISM', sort_by='name')
     assert ism.name[0] == 'AlII 1670', error_msg
     ism.sortdata('name', reverse=True)
     assert ism.name[0] == 'ZrIII 1798', error_msg
@@ -143,5 +143,4 @@ def test_sortdata():  # With extras
     ism.sortdata(['rel_strength'])
     assert ism.name[0] == 'CI** 1123b', error_msg
 
-'''