Merge pull request #51 from sciris/develop

Various updates

sciris/sc_math.py

@@ -243,7 +243,7 @@ def isprime(n, verbose=False):
 ### OTHER FUNCTIONS
 ##############################################################################
 
-__all__ += ['quantile', 'perturb', 'scaleratio', 'inclusiverange', 'smoothinterp']
+__all__ += ['quantile', 'perturb', 'scaleratio', 'normalize', 'inclusiverange', 'smooth', 'smoothinterp']
 
 
 def quantile(data, quantiles=[0.5, 0.25, 0.75]):
@@ -274,6 +274,7 @@ def perturb(n=1, span=0.5, randseed=None):
 
 def scaleratio(inarray, total=None):
     ''' Multiply a list or array by some factor so that its sum is equal to the total. '''
+    if total is None: total = 1.0
     origtotal = float(sum(inarray))
     ratio = total/origtotal
     outarray = np.array(inarray)*ratio
@@ -282,6 +283,17 @@ def scaleratio(inarray, total=None):
 
 
 
+def normalize(inarray, minval=0.0, maxval=1.0):
+    ''' Rescale an array between a minimum value and a maximum value '''
+    outarray = np.array(inarray)
+    outarray -= outarray.min()
+    outarray /= outarray.max()
+    outarray *= (maxval - minval)
+    outarray += minval
+    return outarray
+
+
+
 def inclusiverange(*args, **kwargs):
     '''
     Like arange/linspace, but includes the start and stop points. 
@@ -323,7 +335,26 @@ def inclusiverange(*args, **kwargs):
     x = np.linspace(start, stop, int(round((stop-start)/float(step))+1)) # Can't use arange since handles floating point arithmetic badly, e.g. compare arange(2000, 2020, 0.2) with arange(2000, 2020.2, 0.2)
 
     return x
-
+
+
+
+def smooth(data, repeats=None):
+    ''' Very crude function to smooth a 2D array -- very slow but simple and easy to use '''
+    if repeats is None:
+        repeats = int(np.floor(len(data)/5))
+    output = np.array(data)
+    kernel = np.array([0.25,0.5,0.25])
+    for r in range(repeats):
+        if output.ndim == 1:
+            np.convolve(data, kernel, mode='same')
+        elif output.ndim == 2:
+            for i in range(output.shape[0]): output[i,:] = np.convolve(output[i,:], kernel, mode='same')
+            for j in range(output.shape[1]): output[:,j] = np.convolve(output[:,j], kernel, mode='same')
+        else:
+            errormsg = 'Simple smooting only implemented for 1D and 2D arrays'
+            raise Exception(errormsg)
+    return output
+
 
 
 def smoothinterp(newx=None, origx=None, origy=None, smoothness=None, growth=None, ensurefinite=False, keepends=True, method='linear'):

sciris/sc_parallel.py

@@ -76,10 +76,10 @@ def parallelize(func=None, iterarg=None, iterkwargs=None, args=None, kwargs=None
     complex arguments.
     
     Either or both of iterarg or iterkwargs can be used. iterarg can be an iterable or an integer;
-    if the latter, it will run the function that number of times (which may be useful for
-    running "embarrassingly parallel" simulations). iterkwargs is a dict of iterables; each
-    iterable must be the same length (and the same length of iterarg, if it exists), and each 
-    dict key will be used as a kwarg to the called function.
+    if the latter, it will run the function that number of times and not pass the argument to the 
+    function (which may be useful for running "embarrassingly parallel" simulations). iterkwargs 
+    is a dict of iterables; each iterable must be the same length (and the same length of iterarg, 
+    if it exists), and each dict key will be used as a kwarg to the called function.
     
     This function can either use a fixed number of CPUs or allocate dynamically based
     on load. If ncpus is None and maxload is None, then it will use the number of CPUs
@@ -93,7 +93,7 @@ def f(x):
         results = sc.parallelize(f, [1,2,3])
     
     Example 2 -- simple usage for "embarrassingly parallel" processing:
-        def rnd(i):
+        def rnd():
             import pylab as pl
             return pl.rand()
         
@@ -139,13 +139,15 @@ def f(x,y):
 
     # Handle iterarg and iterkwargs
     niters = 0
+    embarrassing = False # Whether or not it's an embarrassingly parallel optimization
     if iterarg is not None and iterkwargs is not None:
         errormsg = 'You can only use one of iterarg or iterkwargs as your iterable, not both'
         raise Exception(errormsg)
     if iterarg is not None:
         if not(ut.isiterable(iterarg)):
             try:
                 iterarg = np.arange(iterarg)
+                embarrassing = True
             except Exception as E:
                 errormsg = 'Could not understand iterarg "%s": not iterable and not an integer: %s' % (iterarg, str(E))
                 raise Exception(errormsg)
@@ -191,7 +193,7 @@ def f(x,y):
             else: # Should be caught by previous checking, so shouldn't happen
                 errormsg = 'iterkwargs type not understood (%s)' % type(iterkwargs)
                 raise Exception(errormsg)
-        taskargs = TaskArgs(func, index, iterval, iterdict, args, kwargs, maxload, interval)
+        taskargs = TaskArgs(func, index, iterval, iterdict, args, kwargs, maxload, interval, embarrassing)
         argslist.append(taskargs)
 
     # Run simply using map -- no advantage here to using Process/Queue
@@ -256,15 +258,16 @@ def parallelcmd(cmd=None, parfor=None, returnval=None, maxload=None, interval=No
 
 class TaskArgs(ut.prettyobj):
         ''' A class to hold the arguments for the task -- must match both parallelize() and parallel_task() '''
-        def __init__(self, func, index, iterval, iterdict, args, kwargs, maxload, interval):
-            self.func     = func      # The function being called
-            self.index    = index     # The place in the queue
-            self.iterval  = iterval   # The value being iterated (may be None if iterdict is not None)
-            self.iterdict = iterdict  # A dictionary of values being iterated (may be None if iterval is not None)
-            self.args     = args      # Arguments passed directly to the function
-            self.kwargs   = kwargs    # Keyword arguments passed directly to the function
-            self.maxload  = maxload   # Maximum CPU load (ignored if ncpus is not None in parallelize()
-            self.interval = interval  # Interval to check load (only used with maxload)
+        def __init__(self, func, index, iterval, iterdict, args, kwargs, maxload, interval, embarrassing):
+            self.func         = func         # The function being called
+            self.index        = index        # The place in the queue
+            self.iterval      = iterval      # The value being iterated (may be None if iterdict is not None)
+            self.iterdict     = iterdict     # A dictionary of values being iterated (may be None if iterval is not None)
+            self.args         = args         # Arguments passed directly to the function
+            self.kwargs       = kwargs       # Keyword arguments passed directly to the function
+            self.maxload      = maxload      # Maximum CPU load (ignored if ncpus is not None in parallelize()
+            self.interval     = interval     # Interval to check load (only used with maxload)
+            self.embarrassing = embarrassing # Whether or not to pass the iterarg to the function (no if it's embarrassing)
             return None
 
 
@@ -273,16 +276,17 @@ def parallel_task(taskargs, outputqueue=None):
     ''' Task called by parallelize() -- not to be called directly '''
 
     # Handle inputs
-    func = taskargs.func
-    index = taskargs.index
-    args = taskargs.args
+    func   = taskargs.func
+    index  = taskargs.index
+    args   = taskargs.args
     kwargs = taskargs.kwargs
     if args   is None: args   = ()
     if kwargs is None: kwargs = {}
     if taskargs.iterval is not None:
         if not isinstance(taskargs.iterval, tuple): # Ensure it's a tuple
             taskargs.iterval = (taskargs.iterval,)
-        args = taskargs.iterval + args # If variable name is not supplied, prepend it to args
+        if not taskargs.embarrassing:
+            args = taskargs.iterval + args # If variable name is not supplied, prepend it to args
     if taskargs.iterdict is not None:
         for key,val in taskargs.iterdict.items():
             kwargs[key] = val # Otherwise, include it in kwargs

sciris/sc_plotting.py

@@ -233,8 +233,6 @@ def alpinecolormap(gap=0.1, mingreen=0.2, redbluemix=0.5, epsilon=0.01, demo=Fal
 
 
     def demoplot():
-        from mpl_toolkits.mplot3d import Axes3D # analysis:ignore
-
         maxheight = 3
         horizontalsize = 4;
         pl.seed(8)
@@ -249,8 +247,7 @@ def demoplot():
         data /= data.max()
         data *= maxheight
 
-        fig = pl.figure(figsize=(18,8))
-        ax = fig.gca(projection='3d')
+        fig,ax = ax3d(returnfig=True, figsize=(18,8))
         ax.view_init(elev=45, azim=30)
         X = np.linspace(0,horizontalsize,n)
         X, Y = np.meshgrid(X, X)
@@ -335,9 +332,6 @@ def vectocolor(vector, cmap=None, asarray=True, reverse=False):
 
 
 
-
-
-
 ## Create colormap
 def bicolormap(gap=0.1, mingreen=0.2, redbluemix=0.5, epsilon=0.01, demo=False):
     """
@@ -442,7 +436,81 @@ def rgb2hex(arr):
 ### PLOTTING FUNCTIONS
 ##############################################################################
 
-__all__ += ['boxoff', 'setylim', 'commaticks', 'SItickformatter', 'SIticks']
+__all__ += ['ax3d', 'surf3d', 'bar3d', 'boxoff', 'setylim', 'commaticks', 'SItickformatter', 'SIticks']
+
+
+def ax3d(fig=None, returnfig=False, silent=False, **kwargs):
+    ''' Create a 3D axis to plot in -- all arguments are passed to figure() '''
+    from mpl_toolkits.mplot3d import Axes3D # analysis:ignore
+    if fig is None: 
+        fig = pl.figure(**kwargs)
+    else:
+        silent = False # Never close an already open figure
+    ax = fig.gca(projection='3d')
+    if silent:
+        pl.close(fig)
+    if returnfig:
+        return fig,ax
+    else:
+        return ax
+
+
+
+def surf3d(data, fig=None, returnfig=False, plotkwargs=None, colorbar=True, **kwargs):
+    ''' Plot 2D data as a 3D surface '''
+
+    # Set default arguments
+    if plotkwargs is None: plotkwargs = {}
+    settings = {'rstride':1, 'cstride':1, 'linewidth':0, 'antialiased':False, 'cmap':'viridis'}
+    settings.update(plotkwargs)
+
+    # Create figure
+    fig,ax = ax3d(returnfig=True, fig=fig, **kwargs)
+    ax.view_init(elev=45, azim=30)
+    nx,ny = pl.array(data).shape
+    x = np.arange(nx)
+    y = np.arange(ny)
+    X, Y = np.meshgrid(x, y)
+    surf = ax.plot_surface(X, Y, data, **settings)
+    if colorbar:
+        fig.colorbar(surf)
+
+    if returnfig:
+        return fig,ax
+    else:
+        return ax
+
+
+
+def bar3d(data, fig=None, returnfig=False, plotkwargs=None, **kwargs):
+    ''' Plot 2D data as 3D bars '''
+
+    # Set default arguments
+    if plotkwargs is None: plotkwargs = {}
+    settings = {'width':0.8, 'depth':0.8, 'shade':True, 'cmap':'viridis'}
+    settings.update(plotkwargs)
+
+    # Create figure
+    fig,ax = ax3d(returnfig=True, fig=fig, **kwargs)
+
+    x, y, z = [], [], []
+    dx, dy, dz = [], [], []
+    color = vectocolor(data.flatten(), cmap=settings['cmap'])
+    for i in range(data.shape[0]):
+        for j in range(data.shape[1]):
+            x.append(i)
+            y.append(j)
+            z.append(0)
+            dx.append(settings['width'])
+            dy.append(settings['depth'])
+            dz.append(data[i,j])
+    ax.bar3d(x=x, y=y, z=z, dx=settings['width'], dy=settings['depth'], dz=dz, color=color, shade=settings['shade'])
+
+    if returnfig:
+        return fig,ax
+    else:
+        return ax
+
 
 
 def boxoff(ax=None, removeticks=True, flipticks=True):

sciris/sc_utils.py

@@ -12,12 +12,12 @@
 import hashlib
 import datetime
 import dateutil
+import subprocess
 import numbers
 import numpy as np
 import uuid as py_uuid
 from textwrap import fill
 from functools import reduce
-from subprocess import Popen, PIPE
 from collections import OrderedDict as OD
 from distutils.version import LooseVersion
 
@@ -1049,16 +1049,27 @@ def checkmem(origvariable, descend=False, order='n', plot=False, verbose=False):
 
 
 def runcommand(command, printinput=False, printoutput=False):
-   '''
-   Make it easier to run shell commands.
-   
-   Date: 2018mar28
-   '''
-   if printinput: print(command)
-   try:    output = Popen(command, shell=True, stdout=PIPE).communicate()[0].decode("utf-8")
-   except: output = 'Shell command failed'
-   if printoutput: print(output)
-   return output
+    '''
+    Make it easier to run shell commands.
+
+    Examples:
+        myfiles = sc.runcommand('ls').split('\n') # Get a list of files in the current folder
+        sc.runcommand('sshpass -f %s scp myfile.txt me@myserver:myfile.txt' % 'pa55w0rd', printinput=True, printoutput=True) # Copy a file remotely
+    
+    Date: 2019jan15
+    '''
+    if printinput:
+        print(command)
+    try:
+        p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        stderr = p.stdout.read().decode("utf-8") # Somewhat confusingly, send stderr to stdout
+        stdout = p.communicate()[0].decode("utf-8") # ...and then stdout to the pipe
+        output = stdout + '\n' + stderr if stderr else stdout # Only include the error if it was non-empty
+    except Exception as E:
+        output = 'runcommand(): shell command failed: %s' % str(E) # This is for a Python error, not a shell error -- those get passed to output
+    if printoutput: 
+        print(output)
+    return output
 
 
 

sciris/sc_version.py

@@ -1,5 +1,5 @@
 __all__ = ['__version__', '__versiondate__', '__license__']
 
-__version__      = '0.12.1'
-__versiondate__  = '2018-12-11'
+__version__      = '0.12.5'
+__versiondate__  = '2019-01-15'
 __license__      = 'Sciris %s (%s) -- (c) Sciris.org' % (__version__, __versiondate__)

setup.py

@@ -1,15 +1,13 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
-import sys
 from setuptools import setup, find_packages
+import sys
 
-# Optionally define extras
-if 'minimal' in sys.argv:
-    sys.argv.remove('minimal')
-    extras = []
-else:
-    extras = [
+# Define the requirements and extras
+requirements = [
+        'matplotlib>=1.4.2', # Plotting
+        'numpy>=1.10.1',     # Numerical functions
         'dill',              # File I/O
         'gitpython',         # Version information
         'openpyxl>=2.5',     # Spreadsheet functions
@@ -20,6 +18,15 @@
         'requests',          # HTTP methods
         ]
 
+# Optionally define extras
+if 'minimal' in sys.argv:
+    print('Performing minimal installation -- some file read/write functions will not work')
+    sys.argv.remove('minimal')
+    requirements = [
+        'matplotlib>=1.4.2', # Plotting
+        'numpy>=1.10.1',     # Numerical functions
+    ]
+
 # Get version information
 versionfile = 'sciris/sc_version.py'
 with open(versionfile, 'r') as f:
@@ -57,8 +64,5 @@
     classifiers=CLASSIFIERS,
     packages=find_packages(),
     include_package_data=True,
-    install_requires=[
-        'matplotlib>=1.4.2', # Plotting
-        'numpy>=1.10.1',     # Numerical functions
-    ] + extras,              # Optional dependencies
+    install_requires=requirements
 )