Make psutil dependency optional (#208)

MacOS users have trouble getting psutil to work, and it is only needed
to do memory profiling, which is not a necessity. This makes the psutil
package optional, and hobbles most of the MemoryProfiler functionality
if it isn't found. Removing the MemoryProfiler entirely in the absence
of psutil might be a better long-term goal, but it is rather intertwined
with reports and the standard operator and difficult to remove.

armi/bookkeeping/memoryProfiler.py

@@ -42,17 +42,29 @@
 https://pythonhosted.org/psutil/
 https://docs.python.org/2/library/gc.html#gc.garbage
 """
+import gc
 import logging
-import psutil
 import tabulate
-import gc
+from typing import Optional
 
 import armi
 from armi import interfaces
 from armi import mpiActions
 from armi import runLog
 from armi.reactor.composites import ArmiObject
 
+try:
+    import psutil
+
+    # psutil is an optional requirement, since it doesnt support MacOS very well
+    _havePsutil = True
+except ImportError:
+    runLog.warning(
+        "Failed to import psutil; MemoryProfiler will not provide meaningful data."
+    )
+    _havePsutil = False
+
+
 # disable the import warnings (Issue #88)
 logging.disable(logging.CRITICAL)
 from pympler.asizeof import asizeof
@@ -502,26 +514,34 @@ def invokeHook(self):
 class SystemAndProcessMemoryUsage(object):
     def __init__(self):
         self.nodeName = armi.MPI_NODENAME
-        self.percentNodeRamUsed = psutil.virtual_memory().percent
-        self.processMemoryInMB = psutil.Process().memory_info().rss / (1012.0 ** 2)
+        # no psutil, no memory diagnostics. TODO: Ideally, we could just cut
+        # MemoryProfiler out entirely, but it is referred to directly by the standard
+        # operator and reports, so easier said than done.
+        self.percentNodeRamUsed: Optional[float] = None
+        self.processMemoryInMB: Optional[float] = None
+        if _havePsutil:
+            self.percentNodeRamUsed = psutil.virtual_memory().percent
+            self.processMemoryInMB = psutil.Process().memory_info().rss / (1012.0 ** 2)
 
     def __isub__(self, other):
-        self.percentNodeRamUsed -= other.percentNodeRamUsed
-        self.processMemoryInMB -= other.processMemoryInMB
+        if self.percentNodeRamUsed is not None and other.percentNodeRamUsed is not None:
+            self.percentNodeRamUsed -= other.percentNodeRamUsed
+            self.processMemoryInMB -= other.processMemoryInMB
         return self
 
 
 class PrintSystemMemoryUsageAction(mpiActions.MpiAction):
     def __init__(self):
         mpiActions.MpiAction.__init__(self)
         self.usages = []
-        self.percentNodeRamUsed = 0.0
+        self.percentNodeRamUsed: Optional[float] = None
 
     def __iter__(self):
         return iter(self.usages)
 
     def __isub__(self, other):
-        self.percentNodeRamUsed -= other.percentNodeRamUsed
+        if self.percentNodeRamUsed is not None and other.percentNodeRamUsed is not None:
+            self.percentNodeRamUsed -= other.percentNodeRamUsed
         for mine, theirs in zip(self, other):
             mine -= theirs
         return self
@@ -530,13 +550,13 @@ def __isub__(self, other):
     def minProcessMemoryInMB(self):
         if len(self.usages) == 0:
             return 0.0
-        return min(mu.processMemoryInMB for mu in self)
+        return min(mu.processMemoryInMB or 0.0 for mu in self)
 
     @property
     def maxProcessMemoryInMB(self):
         if len(self.usages) == 0:
             return 0.0
-        return max(mu.processMemoryInMB for mu in self)
+        return max(mu.processMemoryInMB or 0.0 for mu in self)
 
     def invokeHook(self):
         spmu = SystemAndProcessMemoryUsage()
@@ -548,10 +568,10 @@ def printUsage(self, description=None):
 
         The printout looks something like:
 
-            SYS_MEM EDWARD102.hpcc.tp.int     14.4% RAM. Proc mem (MB):   491   472   471   471   471   470
-            SYS_MEM EDWARD103.hpcc.tp.int     13.9% RAM. Proc mem (MB):   474   473   472   471   460   461
-            SYS_MEM EDWARD104.hpcc.tp.int     ...
-            SYS_MEM EDWARD107.hpcc.tp.int     ...
+            SYS_MEM HOSTNAME     14.4% RAM. Proc mem (MB):   491   472   471   471   471   470
+            SYS_MEM HOSTNAME     13.9% RAM. Proc mem (MB):   474   473   472   471   460   461
+            SYS_MEM HOSTNAME     ...
+            SYS_MEM HOSTNAME     ...
 
         """
         printedNodes = set()
@@ -563,7 +583,7 @@ def printUsage(self, description=None):
                 continue
             printedNodes.add(memoryUsage.nodeName)
             nodeUsages = [mu for mu in self if mu.nodeName == memoryUsage.nodeName]
-            sysMemAvg = sum(mu.percentNodeRamUsed for mu in nodeUsages) / len(
+            sysMemAvg = sum(mu.percentNodeRamUsed or 0.0 for mu in nodeUsages) / len(
                 nodeUsages
             )
 
@@ -573,7 +593,8 @@ def printUsage(self, description=None):
                     "{:5.1f}%".format(sysMemAvg),
                     "{}".format(
                         " ".join(
-                            "{:5.0f}".format(mu.processMemoryInMB) for mu in nodeUsages
+                            "{:5.0f}".format(mu.processMemoryInMB or 0.0)
+                            for mu in nodeUsages
                         )
                     ),
                 )

requirements.txt

@@ -8,11 +8,12 @@ ordered-set
 pluggy
 pyyaml>=5.1
 pyevtk
-psutil
 xlrd
 pillow
 future
 configparser
 voluptuous
 pympler
 coverage
+
+psutil[memprof]

setup.py

@@ -61,7 +61,6 @@ def collectExtraFiles():
         "ordered-set",
         "pillow",
         "pluggy",
-        "psutil",
         "pyevtk",
         "pympler",
         "pyyaml>=5.1",
@@ -71,7 +70,7 @@ def collectExtraFiles():
         "xlrd",
         "yamlize",
     ],
-    extras_require={"mpi": ["mpi4py"], "grids": ["wxpython"]},
+    extras_require={"mpi": ["mpi4py"], "grids": ["wxpython"], "memprof": ["psutil"]},
     tests_require=["nbconvert", "jupyter_client", "ipykernel"],
     classifiers=[
         "Development Status :: 4 - Beta",