[4/N] Modernize FFX: reorganize core

.travis.yml

@@ -1,9 +1,13 @@
 language: python
-python:
-  - "2.7"
-  - "3.5"
-  - "3.6"
-  - "3.7"
+
+jobs:
+  include:
+    - python: "2.7"
+      env: NOVALIDATE=0
+    - python: "3.5"
+      env: NOVALIDATE=0
+    - python: "3.6"
+    - python: "3.7"
 
 install:
   - pip install -r dev-requirements.txt
@@ -13,6 +17,7 @@ before_script:
   - export PYTHONPATH=$(pwd):$PYTHONPATH;
 
 script:
+  - '[ -z "$NOVALIDATE" ] && make validate || echo "skipping validate"'
   - pytest --cov ffx
 
 after_success:

Makefile

@@ -5,7 +5,7 @@ black:
 	black ffx --line-length 100 --target-version py27 --target-version py35 --target-version py36 --target-version py37 --target-version py38 -S --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/"
 
 isort:
-	isort -rc
+	isort -rc -y
 
 validate: pylint isort black
 

ffx/__init__.py

@@ -1 +1,3 @@
-from .api import *
+from .api import FFXRegressor, run
+
+__all__ = ['run', 'FFXRegressor']

ffx/api.py

@@ -4,10 +4,10 @@
 
 from sklearn.base import BaseEstimator, RegressorMixin
 
-from . import core
-
 
 def run(train_X, train_y, test_X, test_y, varnames=None, verbose=False):
+    from ffx import core
+
     return core.MultiFFXModelFactory().build(train_X, train_y, test_X, test_y, varnames, verbose)
 
 

ffx/core.py → ffx/core/__init__.py

@@ -6,31 +6,26 @@
 
 # 3rd party dependencies
 import numpy
-import scipy
+import pandas as pd
 from six.moves import range, zip
 from sklearn.base import RegressorMixin
 from sklearn.linear_model import ElasticNet
 
-# user-changeable constants
-CONSIDER_INTER = True  # consider interactions?
-CONSIDER_DENOM = True  # consider denominator?
-CONSIDER_EXPON = True  # consider exponents?
-CONSIDER_NONLIN = True  # consider abs() and log()?
-CONSIDER_THRESH = True  # consider hinge functions?
-
-
-# Make dependency on pandas optional.
-try:
-    import pandas
-except ImportError:
-    pandas = None
-
-INF = float('Inf')
-# maximum time (s) for regularization update during pathwise learn.
-MAX_TIME_REGULARIZE_UPDATE = 5
-
-# GTH = Greater-Than Hinge function, LTH = Less-Than Hinge function
-OP_ABS, OP_MAX0, OP_MIN0, OP_LOG10, OP_GTH, OP_LTH = 1, 2, 3, 4, 5, 6
+from .constants import (
+    CONSIDER_DENOM,
+    CONSIDER_EXPON,
+    CONSIDER_INTER,
+    CONSIDER_NONLIN,
+    CONSIDER_THRESH,
+    INF,
+    MAX_TIME_REGULARIZE_UPDATE,
+    OP_ABS,
+    OP_GTH,
+    OP_LOG10,
+    OP_LTH,
+    OP_MAX0,
+    OP_MIN0,
+)
 
 
 def _approachStr(approach):
@@ -310,7 +305,7 @@ def simulate(self, X):
         elif op == OP_LOG10:
             # safeguard against: log() on values <= 0.0
             mn, mx = min(y_lin), max(y_lin)
-            if mn <= 0.0 or scipy.isnan(mn) or mx == INF or scipy.isnan(mx):
+            if mn <= 0.0 or numpy.isnan(mn) or mx == INF or numpy.isnan(mx):
                 ok = False
             else:
                 ya = numpy.log10(y_lin)
@@ -425,7 +420,7 @@ def simulate(self, X):
           y -- 1d array of [sample_i] : float
         """
         N = X.shape[0]
-        if scipy.isnan(self.constant):  # corner case
+        if numpy.isnan(self.constant):  # corner case
             yhat = numpy.array([INF] * N)
         else:  # typical case
             yhat = numpy.ones(N, dtype=float) * self.constant
@@ -466,7 +461,7 @@ def build(self, train_X, train_y, test_X, test_y, varnames=None, verbose=False):
           models -- list of FFXModel -- Pareto-optimal set of models
         """
 
-        if pandas is not None and isinstance(train_X, pandas.DataFrame):
+        if isinstance(train_X, pd.DataFrame):
             varnames = train_X.columns
             train_X = train_X.to_numpy()
             test_X = test_X.to_numpy()
@@ -605,9 +600,9 @@ def build(self, X, y, ss, varnames=None, verbose=False):
         @return
           models -- list of FFXModel -- Pareto-optimal set of models
         """
-        if pandas is not None and isinstance(X, pandas.DataFrame):
+        if isinstance(X, pd.DataFrame):
             varnames = X.columns
-            X = X.as_matrix()
+            X = X.to_numpy()
         if isinstance(X, numpy.ndarray) and varnames is None:
             raise Exception('varnames required for numpy.ndarray')
 
@@ -861,7 +856,7 @@ def _pathwiseLearn(
 
         n_samples = X_unbiased.shape[0]
         vals = numpy.dot(X_unbiased.T, y_unbiased)
-        vals = [val for val in vals if not scipy.isnan(val)]
+        vals = [val for val in vals if not numpy.isnan(val)]
         if vals:
             alpha_max = numpy.abs(max(vals) / (n_samples * ss.l1_ratio()))
         else:
@@ -948,7 +943,7 @@ def _pathwiseLearn(
                 )
 
             # maybe stop
-            if scipy.isinf(nmses[-1]):
+            if numpy.isinf(nmses[-1]):
                 if verbose:
                     print('    Pathwise learn: Early stop because nmse is inf')
                 return None
@@ -1176,7 +1171,7 @@ def nmse(yhat, y, min_y, max_y):
     y_range = float(max_y - min_y)
     try:
         result = math.sqrt(numpy.mean(((yhat_a - y_a) / y_range) ** 2))
-        if scipy.isnan(result):
+        if numpy.isnan(result):
             return INF
         return result
     except:  # pylint: disable=bare-except
@@ -1185,7 +1180,7 @@ def nmse(yhat, y, min_y, max_y):
 
 def yIsPoor(y):
     """Returns True if y is not usable"""
-    return max(scipy.isinf(y)) or max(scipy.isnan(y))
+    return max(numpy.isinf(y)) or max(numpy.isnan(y))
 
 
 def coefStr(x):

ffx/core/constants.py

@@ -0,0 +1,13 @@
+# user-changeable constants
+CONSIDER_INTER = True  # consider interactions?
+CONSIDER_DENOM = True  # consider denominator?
+CONSIDER_EXPON = True  # consider exponents?
+CONSIDER_NONLIN = True  # consider abs() and log()?
+CONSIDER_THRESH = True  # consider hinge functions?
+
+INF = float('Inf')
+# maximum time (s) for regularization update during pathwise learn.
+MAX_TIME_REGULARIZE_UPDATE = 5
+
+# GTH = Greater-Than Hinge function, LTH = Less-Than Hinge function
+OP_ABS, OP_MAX0, OP_MIN0, OP_LOG10, OP_GTH, OP_LTH = 1, 2, 3, 4, 5, 6

ffx_tests/tests.py

@@ -1,17 +1,17 @@
 import ffx
 import numpy as np
 from ffx.core import (
-    ConstantModel,
-    OperatorBase,
-    ProductBase,
-    SimpleBase,
     INF,
     OP_ABS,
     OP_GTH,
     OP_LOG10,
     OP_LTH,
     OP_MAX0,
     OP_MIN0,
+    ConstantModel,
+    OperatorBase,
+    ProductBase,
+    SimpleBase,
 )
 
 EPS = 0.001

setup.py

@@ -15,15 +15,7 @@
     url='https://github.com/natekupp/ffx',
     packages=['ffx'],
     entry_points={'console_scripts': ['ffx = ffx.cli:main']},
-    install_requires=[
-        'click>=5.0',
-        'contextlib2>=0.5.4',
-        'numpy',
-        'pandas',
-        'scipy',
-        'six',
-        'sklearn',
-    ],
+    install_requires=['click>=5.0', 'contextlib2>=0.5.4', 'numpy', 'pandas', 'six', 'sklearn',],
     classifiers=[
         'Development Status :: 4 - Beta',
         'Environment :: Console',