Merge pull request #32 from dhhagan/dev

Dev

setup.py

@@ -5,7 +5,7 @@
 
 from smps import  __version__
 
-__version__ = '1.1.0'
+__version__ = '1.1.0-dev'
 
 setup(
     name='py-smps',

smps/__init__.py

@@ -3,4 +3,4 @@
 from .models import *
 from .rcmod import *
 
-__version__ = "1.1.0"
+__version__ = "1.1.0-dev"

smps/models.py

@@ -172,43 +172,48 @@ def stats(self, weight='number', dmin=0., dmax=1e3, rho=1.65, **kwargs):
         :param rho: particle density in units of g/cc3
         """
         # remove the weight from the kwargs
-
         assert(weight in ["number", "surface", "volume", "mass"])
 
         # initialize an empty dataframe to hold the results
-        res = pd.DataFrame()
+        res = pd.DataFrame(index=self.data.index)
 
         # subselect the dataframe to only include diameters of interest
         cpy = self.copy()
         cpy.data = cpy._subselect_frame(cpy.data, **kwargs)
 
+        # drop all rows that are completely NaN
+        cpy.data = cpy.data.dropna(how='all')
+
+        # make a shortcut for the index to inject
+        idx = cpy.data.index
+
         # calculate the total number of particles
-        res["number"] = cpy.dn.sum(axis=1)
-        res["surface_area"] = cpy.ds.sum(axis=1)
-        res["volume"] = cpy.dv.sum(axis=1)
-        res["mass"] = cpy.dv.mul(rho).sum(axis=1)
+        res.loc[idx, "number"] = cpy.dn.sum(axis=1)
+        res.loc[idx, "surface_area"] = cpy.ds.sum(axis=1)
+        res.loc[idx, "volume"] = cpy.dv.sum(axis=1)
+        res.loc[idx, "mass"] = cpy.dv.mul(rho).sum(axis=1)
 
         if weight == "number":
-            res["AM"] = 1e3*cpy.dn.mul(self.midpoints).sum(axis=1) / res["number"]
-            res["GM"] = 1e3*np.exp(cpy.dn.mul(np.log(self.midpoints), axis=1).sum(axis=1) / res["number"])
-            res["Mode"] = cpy.dn.apply(lambda x: 1e3*cpy.midpoints[cpy.dn.columns.get_loc(x.idxmax())], axis=1)
+            res.loc[idx, "AM"] = 1e3*cpy.dn.mul(self.midpoints).sum(axis=1) / res.loc[idx, "number"]
+            res.loc[idx, "GM"] = 1e3*np.exp(cpy.dn.mul(np.log(self.midpoints), axis=1).sum(axis=1) / res.loc[idx, "number"])
+            res.loc[idx, "Mode"] = cpy.dndlogdp.apply(lambda x: 1e3*cpy.midpoints[cpy.dndlogdp.columns.get_loc(x.idxmax())], axis=1)
 
-            tmp = cpy.dn.assign(GM=res['GM'].values)
+            tmp = cpy.dn.assign(GM=res.loc[idx, 'GM'].values)
         elif weight == "surface":
-            res["AM"] = 1e3 * cpy.ds.mul(self.midpoints).sum(axis=1) / res["surface_area"]
-            res["GM"] = 1e3 * np.exp(cpy.ds.mul(np.log(self.midpoints), axis=1).sum(axis=1) / res["surface_area"])
-            res["Mode"] = cpy.ds.apply(lambda x: 1e3*cpy.midpoints[cpy.ds.columns.get_loc(x.idxmax())], axis=1)
+            res.loc[idx, "AM"] = 1e3 * cpy.ds.mul(self.midpoints).sum(axis=1) / res.loc[idx, "surface_area"]
+            res.loc[idx, "GM"] = 1e3 * np.exp(cpy.ds.mul(np.log(self.midpoints), axis=1).sum(axis=1) / res.loc[idx, "surface_area"])
+            res.loc[idx, "Mode"] = cpy.dsdlogdp.apply(lambda x: 1e3*cpy.midpoints[cpy.dsdlogdp.columns.get_loc(x.idxmax())], axis=1)
 
-            tmp = cpy.ds.assign(GM=res['GM'].values)
+            tmp = cpy.ds.assign(GM=res.loc[idx, 'GM'].values)
         else:
-            res["AM"] = 1e3 * cpy.dv.mul(self.midpoints).sum(axis=1) / res["volume"]
-            res["GM"] = 1e3 * np.exp(cpy.dv.mul(np.log(self.midpoints), axis=1).sum(axis=1) / res["volume"])
-            res["Mode"] = cpy.dv.apply(lambda x: 1e3*cpy.midpoints[cpy.dv.columns.get_loc(x.idxmax())], axis=1)
+            res.loc[idx, "AM"] = 1e3 * cpy.dv.mul(self.midpoints).sum(axis=1) / res.loc[idx, "volume"]
+            res.loc[idx, "GM"] = 1e3 * np.exp(cpy.dv.mul(np.log(self.midpoints), axis=1).sum(axis=1) / res.loc[idx, "volume"])
+            res.loc[idx, "Mode"] = cpy.dvdlogdp.apply(lambda x: 1e3*cpy.midpoints[cpy.dvdlogdp.columns.get_loc(x.idxmax())], axis=1)
 
-            tmp = cpy.dv.assign(GM=res['GM'].values)
+            tmp = cpy.dv.assign(GM=res.loc[idx, 'GM'].values)
 
         # calculate the GSD
-        res["GSD"] = tmp.apply(self._gsd, axis=1)
+        res.loc[idx, "GSD"] = tmp.apply(self._gsd, axis=1)
 
         # delete the cpy to free up memory
         del cpy, tmp

smps/plots.py

@@ -26,8 +26,8 @@ def heatmap(X, Y, Z, ax=None, logy=True, cbar=True, hide_low=True,
     Z_plot = nan_to_num(Z_plot)
 
     # Set the colorbar min and max based on the min and max of the values
-    cbar_min = kwargs.pop('cbar_min', Z.min() if Z.min() > 0.0 else 1.)
-    cbar_max = kwargs.pop('cbar_max', Z.max())
+    cbar_min = kwargs.pop('cbar_min', Z_plot.min() if Z_plot.min() > 0.0 else 1.)
+    cbar_max = kwargs.pop('cbar_max', Z_plot.max())
 
     if hide_low:
         # Increase values below cbar_min to cbar_min
@@ -39,7 +39,7 @@ def heatmap(X, Y, Z, ax=None, logy=True, cbar=True, hide_low=True,
                         **plot_kws)
 
     # Set the figure keywords
-    fig_kws = dict(dict(figsize=(16,8)), **fig_kws)
+    fig_kws = dict(dict(figsize=(10,5)), **fig_kws)
 
     if ax is None:
         plt.figure(**fig_kws)

tests/io_tests.py

@@ -58,10 +58,12 @@ def test_smps_from_txt_columns(self):
         self.assertTrue(isinstance(res['bin_labels'], list))
         self.assertTrue(isinstance(res['bin_prefix'], str))
 
+    """
     def test_load_sample(self):
         m = smps.io.load_sample("boston")
 
         self.assertIsInstance(m, smps.models.SMPS)
+    """
 
     def test_models_generic_methods(self):
         res = smps.io.smps_from_txt(
@@ -160,6 +162,8 @@ def test_generic_calculations(self):
         cols_to_check.append(("Mode", "Mode(nm)"))
         cols_to_check.append(("GSD", "Geo. Std. Dev."))
 
+        print ("")
+
         for pkg_col, aim_col in cols_to_check:
             m, b, r, p, e = linregress(
                                 stats[pkg_col].values,
@@ -172,6 +176,8 @@ def test_generic_calculations(self):
             self.assertGreaterEqual(m, 0.99)
             self.assertLessEqual(m, 1.01)
 
+            print ("\t{} v {}: \n\t\tr2={:.3f}, m={:.2f}".format(pkg_col, aim_col, r**2, m))
+
         # check the surface-area-weighted stats
         stats = number.stats(weight='surface')
 
@@ -247,27 +253,26 @@ def test_models_alphasense_opcn2(self):
             print ("\t{} @ dmax={}".format(opc_col, dmax))
             print ("\t\tm={:.3f}, r2={:.3f}".format(m, r**2))
 
-            print (opcn2.scan_stats[opc_col].values[0:3])
-            print (opcn2.integrate(weight='mass', rho=1.65, dmax=dmax).values[0:3])
-
-            # make sure the correlation is above 0.99
-            #self.assertGreaterEqual(r**2, 0.99)
-
-            # make sure the slope is between 0.99 and 1.01
-            #self.assertGreaterEqual(m, 0.99)
-            #self.assertLessEqual(m, 1.01)
-
     def test_models_pops(self):
         pass
 
     def test_fit(self):
         from smps.fit import LogNormal
 
-        r = smps.io.load_sample("boston")
+        r = smps.io.smps_from_txt(
+                        os.path.join(datadir, "test_data_number.txt"),
+                        column=False, as_dict=False)
 
         # fit 1 mode in volume number space
         m = LogNormal()
 
         results = m.fit(X=r.midpoints, Y=r.dndlogdp.mean(), modes=1)
 
-        print (results.summary())
+    def test_calculations_with_nans(self):
+        number = smps.io.smps_from_txt(
+                        os.path.join(datadir, "test_data_number.txt"),
+                        column=False, as_dict=False)
+
+        number.resample("1min", inplace=True)
+
+        stats = number.stats(weight='number')