Merge pull request #190 from pysat/enh/187_clean

ENH: Use generalized clean function

CHANGELOG.md

@@ -41,6 +41,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
   * Added `var_translation` kwarg to `load_xarray` interface so that variables can
     be renamed before attempting to merge.
   * Improved usage of cdflib for users in xarray instruments
+  * Added a generalized `clean` routine to replace fill vals with NaNs
 * Deprecations
   * Deprecated jpl_gps instrtument module, moved roti instrument to igs_gps
 * Maintenance

pysatNASA/instruments/ace_epam_l2.py

@@ -65,8 +65,8 @@
 # Use standard init routine
 init = functools.partial(mm_nasa.init, module=mm_ace, name=name)
 
-# Use default ace clean
-clean = mm_ace.clean
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/ace_mag_l2.py

@@ -88,8 +88,8 @@ def preprocess(self):
     return
 
 
-# Use default ace clean
-clean = mm_ace.clean
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/ace_sis_l2.py

@@ -63,8 +63,8 @@
 # Use standard init routine
 init = functools.partial(mm_nasa.init, module=mm_ace, name=name)
 
-# Use default ace clean
-clean = mm_ace.clean
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/ace_swepam_l2.py

@@ -64,8 +64,8 @@
 # Use standard init routine
 init = functools.partial(mm_nasa.init, module=mm_ace, name=name)
 
-# Use default ace clean
-clean = mm_ace.clean
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/cnofs_plp.py

@@ -60,7 +60,6 @@
 
 import datetime as dt
 import functools
-import numpy as np
 
 from pysat.instruments.methods import general as mm_gen
 
@@ -88,21 +87,8 @@
 init = functools.partial(mm_nasa.init, module=mm_cnofs, name=name)
 
 
-def clean(self):
-    """Clean C/NOFS PLP data to the specified level.
-
-    Note
-    ----
-    Basic cleaning to find valid Epoch values
-
-    """
-
-    for key in self.data.columns:
-        if key != 'Epoch':
-            fill = self.meta[key, self.meta.labels.fill_val]
-            idx, = np.where(self[key] == fill)
-            self[idx, key] = np.nan
-    return
+# Use default clean
+clean = mm_nasa.clean
 
 
 # ----------------------------------------------------------------------------

pysatNASA/instruments/de2_fpi.py

@@ -81,8 +81,8 @@
 # Use standard init routine
 init = functools.partial(mm_nasa.init, module=mm_de2, name=name)
 
-# No cleaning, use standard warning function instead
-clean = mm_nasa.clean_warn
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/de2_lang.py

@@ -79,8 +79,8 @@
 # Use standard init routine
 init = functools.partial(mm_nasa.init, module=mm_de2, name=name)
 
-# No cleaning, use standard warning function instead
-clean = mm_nasa.clean_warn
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/de2_nacs.py

@@ -104,8 +104,8 @@
 # Use standard init routine
 init = functools.partial(mm_nasa.init, module=mm_de2, name=name)
 
-# No cleaning, use standard warning function instead
-clean = mm_nasa.clean_warn
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/de2_rpa.py

@@ -86,8 +86,8 @@
 # Use standard init routine
 init = functools.partial(mm_nasa.init, module=mm_de2, name=name)
 
-# No cleaning, use standard warning function instead
-clean = mm_nasa.clean_warn
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/de2_vefi.py

@@ -70,8 +70,8 @@
 # Use standard init routine
 init = functools.partial(mm_nasa.init, module=mm_de2, name=name)
 
-# No cleaning, use standard warning function instead
-clean = mm_nasa.clean_warn
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/de2_wats.py

@@ -101,8 +101,8 @@
 # Use standard init routine
 init = functools.partial(mm_nasa.init, module=mm_de2, name=name)
 
-# No cleaning, use standard warning function instead
-clean = mm_nasa.clean_warn
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/formosat1_ivm.py

@@ -74,8 +74,8 @@ def init(self):
     return
 
 
-# No cleaning, use standard warning function instead
-clean = mm_nasa.clean_warn
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/igs_gps.py

@@ -99,8 +99,8 @@
 init = functools.partial(mm_nasa.init, module=mm_igs, name=name)
 
 
-# No cleaning, use standard warning function instead
-clean = mm_nasa.clean_warn
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/iss_fpmu.py

@@ -26,12 +26,12 @@
 
 import datetime as dt
 import functools
-import numpy as np
 
 from pysat.instruments.methods import general as mm_gen
 from pysat import logger
 
 from pysatNASA.instruments.methods import cdaweb as cdw
+from pysatNASA.instruments.methods import general as mm_nasa
 
 # ----------------------------------------------------------------------------
 # Instrument attributes
@@ -79,25 +79,8 @@ def init(self):
     return
 
 
-def clean(self):
-    """Clean ISS FPMU data to the specified level.
-
-    Note
-    ----
-    'clean' - Replace Te and Ni fill values with NaN
-    'dusty' - Same as clean
-    'dirty' - Same as clean
-    'none'  - Not applied, default fill values are preserved
-
-    """
-
-    # Replace Te data fill
-    self.data.replace(-999., np.nan, inplace=True)
-
-    # Replace Ne data fill
-    self.data.replace(-9.9999998e+30, np.nan, inplace=True)
-
-    return
+# Use default clean
+clean = mm_nasa.clean
 
 
 # ----------------------------------------------------------------------------

pysatNASA/instruments/jpl_gps.py

@@ -98,8 +98,8 @@ def init(self):
     return
 
 
-# No cleaning, use standard warning function instead
-clean = mm_nasa.clean_warn
+# Use default clean
+clean = mm_nasa.clean
 
 # ----------------------------------------------------------------------------
 # Instrument functions

pysatNASA/instruments/methods/ace.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 """Provides non-instrument specific routines for ACE data."""
 
-import numpy as np
-
 from pysatNASA.instruments.methods import cdaweb as cdw
 
 ackn_str = ' '.join(("Please acknowledge the NASA National Space Science Data",
@@ -87,30 +85,3 @@ def load(fnames, tag='', inst_id='', to_pandas=False):
             data = data.to_dataframe()
 
     return data, meta
-
-
-def clean(self):
-    """Clean ACE data to the specified level.
-
-    Note
-    ----
-    Basic cleaning to replace fill values with NaN
-
-    """
-
-    # Get a list of coords for the data
-    if self.pandas_format:
-        coords = [self.data.index.name]
-    else:
-        coords = [key for key in self.data.coords.keys()]
-
-    for key in self.variables:
-        # Skip over the coordinates when cleaning
-        if key not in coords:
-            fill = self.meta[key, self.meta.labels.fill_val]
-
-            # Replace fill with nan
-            fill_mask = self[key] == fill
-            self[key] = self.data[key].where(~fill_mask)
-            self.meta[key] = {self.meta.labels.fill_val: np.nan}
-    return

pysatNASA/instruments/methods/general.py

@@ -1,5 +1,7 @@
 """General methods for NASA instruments."""
 
+import numpy as np
+
 import pysat
 
 
@@ -43,6 +45,33 @@ def init(self, module, name):
     return
 
 
+def clean(self):
+    """Clean data to the specified level.
+
+    Note
+    ----
+    Basic cleaning to replace fill values with NaN
+
+    """
+
+    # Get a list of coords for the data
+    if self.pandas_format:
+        coords = [self.data.index.name]
+    else:
+        coords = [key for key in self.data.coords.keys()]
+
+    for key in self.variables:
+        # Skip over the coordinates when cleaning
+        if key not in coords:
+            fill = self.meta[key, self.meta.labels.fill_val]
+
+            # Replace fill with nan
+            fill_mask = self[key] == fill
+            self[key] = self.data[key].where(~fill_mask)
+            self.meta[key] = {self.meta.labels.fill_val: np.nan}
+    return
+
+
 def clean_warn(self):
     """Warn user that cleaning not yet available for this data set.