Refactoring OPVI to support Normalizing Flows

.travis.yml

@@ -19,13 +19,15 @@ install:
 env:
   - PYTHON_VERSION=2.7 FLOATX='float32' TESTCMD="--durations=10 --ignore=pymc3/tests/test_examples.py --cov-append --ignore=pymc3/tests/test_distributions_random.py --ignore=pymc3/tests/test_variational_inference.py --ignore=pymc3/tests/test_shared.py --ignore=pymc3/tests/test_smc.py --ignore=pymc3/tests/test_updates.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_sampling.py"
   - PYTHON_VERSION=2.7 FLOATX='float32' RUN_PYLINT="true" TESTCMD="--durations=10 --cov-append pymc3/tests/test_distributions_random.py pymc3/tests/test_shared.py pymc3/tests/test_smc.py pymc3/tests/test_sampling.py"
-  - PYTHON_VERSION=2.7 FLOATX='float32' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py pymc3/tests/test_posteriors.py"
+  - PYTHON_VERSION=2.7 FLOATX='float32' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_posteriors.py"
+  - PYTHON_VERSION=2.7 FLOATX='float32' TESTCMD="--durations=10 --cov-append pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py"
   - PYTHON_VERSION=2.7 FLOATX='float64' TESTCMD="--durations=10 --ignore=pymc3/tests/test_examples.py --cov-append --ignore=pymc3/tests/test_distributions_random.py --ignore=pymc3/tests/test_variational_inference.py --ignore=pymc3/tests/test_shared.py --ignore=pymc3/tests/test_smc.py --ignore=pymc3/tests/test_updates.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_sampling.py"
   - PYTHON_VERSION=2.7 FLOATX='float64' RUN_PYLINT="true" TESTCMD="--durations=10 --cov-append pymc3/tests/test_distributions_random.py pymc3/tests/test_shared.py pymc3/tests/test_smc.py pymc3/tests/test_sampling.py"
-  - PYTHON_VERSION=2.7 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py pymc3/tests/test_posteriors.py"
+  - PYTHON_VERSION=2.7 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_posteriors.py"
   - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append --ignore=pymc3/tests/test_examples.py --ignore=pymc3/tests/test_distributions_random.py --ignore=pymc3/tests/test_variational_inference.py --ignore=pymc3/tests/test_shared.py --ignore=pymc3/tests/test_smc.py --ignore=pymc3/tests/test_updates.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_sampling.py"
   - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_distributions_random.py pymc3/tests/test_shared.py pymc3/tests/test_smc.py pymc3/tests/test_sampling.py"
-  - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py pymc3/tests/test_posteriors.py"
+  - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_examples.py pymc3/tests/test_posteriors.py"
+  - PYTHON_VERSION=3.6 FLOATX='float64' TESTCMD="--durations=10 --cov-append pymc3/tests/test_variational_inference.py pymc3/tests/test_updates.py"
 script:
   - . ./scripts/test.sh $TESTCMD
 

pymc3/distributions/dist_math.py

@@ -11,7 +11,6 @@
 import theano
 
 from .special import gammaln
-from ..math import logdet as _logdet
 from pymc3.theanof import floatX
 
 f = floatX
@@ -164,64 +163,6 @@ def log_normal(x, mean, **kwargs):
     return f(c) - tt.log(tt.abs_(std)) - (x - mean) ** 2 / (2. * std ** 2)
 
 
-def log_normal_mv(x, mean, gpu_compat=False, **kwargs):
-    """
-    Calculate logarithm of normal distribution at point `x`
-    with given `mean` and `sigma` matrix
-
-    Parameters
-    ----------
-    x : Tensor
-        point of evaluation
-    mean : Tensor
-        mean of normal distribution
-    kwargs : one of parameters `{cov, tau, chol}`
-
-    Other Parameters
-    ----------------
-    gpu_compat : False, because LogDet is not GPU compatible yet.
-                 If this is set as true, the GPU compatible (but numerically unstable) log(det) is used.
-
-    Notes
-    -----
-    There are three variants for density parametrization.
-    They are:
-        1) covariance matrix - `cov`
-        2) precision matrix - `tau`,
-        3) cholesky decomposition matrix  - `chol`
-    ----
-    """
-    if gpu_compat:
-        def logdet(m):
-            return tt.log(tt.abs_(tt.nlinalg.det(m)))
-    else:
-        logdet = _logdet
-
-    T = kwargs.get('tau')
-    S = kwargs.get('cov')
-    L = kwargs.get('chol')
-    check = sum(map(lambda a: a is not None, [T, S, L]))
-    if check > 1:
-        raise ValueError('more than one required kwarg is passed')
-    if check == 0:
-        raise ValueError('none of required kwarg is passed')
-    # avoid unnecessary computations
-    if L is not None:
-        S = L.dot(L.T)
-        T = tt.nlinalg.matrix_inverse(S)
-        log_det = -logdet(S)
-    elif T is not None:
-        log_det = logdet(T)
-    else:
-        T = tt.nlinalg.matrix_inverse(S)
-        log_det = -logdet(S)
-    delta = x - mean
-    k = f(S.shape[0])
-    result = k * tt.log(2. * np.pi) - log_det
-    result += delta.dot(T).dot(delta)
-    return -.5 * result
-
-
 def MvNormalLogp():
     """Compute the log pdf of a multivariate normal distribution.
 

pymc3/model.py

@@ -676,7 +676,7 @@ def profile(self, outs, n=1000, point=None, profile=True, *args, **kwargs):
 
         return f.profile
 
-    def flatten(self, vars=None):
+    def flatten(self, vars=None, order=None, inputvar=None):
         """Flattens model's input and returns:
             FlatView with
             * input vector variable
@@ -687,16 +687,25 @@ def flatten(self, vars=None):
         ----------
         vars : list of variables or None
             if None, then all model.free_RVs are used for flattening input
+        order : ArrayOrdering
+            Optional, use predefined ordering
+        inputvar : tt.vector
+            Optional, use predefined inputvar
 
         Returns
         -------
         flat_view
         """
         if vars is None:
             vars = self.free_RVs
-        order = ArrayOrdering(vars)
-        inputvar = tt.vector('flat_view', dtype=theano.config.floatX)
-        inputvar.tag.test_value = flatten_list(vars).tag.test_value
+        if order is None:
+            order = ArrayOrdering(vars)
+        if inputvar is None:
+            inputvar = tt.vector('flat_view', dtype=theano.config.floatX)
+            if vars:
+                inputvar.tag.test_value = flatten_list(vars).tag.test_value
+            else:
+                inputvar.tag.test_value = np.asarray([], inputvar.dtype)
         replacements = {self.named_vars[name]: inputvar[slc].reshape(shape).astype(dtype)
                         for name, slc, shape, dtype in order.vmap}
         view = {vm.var: vm for vm in order.vmap}

pymc3/tests/conftest.py

@@ -1,50 +1,37 @@
-import theano
 import numpy as np
+import theano
+import pymc3 as pm
 import pytest
 
 
-class DataSampler(object):
-    """
-    Not for users
-    """
-    def __init__(self, data, batchsize=50, random_seed=42, dtype='floatX'):
-        self.dtype = theano.config.floatX if dtype == 'floatX' else dtype
-        self.rng = np.random.RandomState(random_seed)
-        self.data = data
-        self.n = batchsize
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        idx = (self.rng
-               .uniform(size=self.n,
-                        low=0.0,
-                        high=self.data.shape[0] - 1e-16)
-               .astype('int64'))
-        return np.asarray(self.data[idx], self.dtype)
-
-    next = __next__
-
-
-@pytest.fixture(scope="session", autouse=True)
+@pytest.fixture(scope="function", autouse=True)
 def theano_config():
     config = theano.configparser.change_flags(compute_test_value='raise')
     with config:
         yield
 
 
-@pytest.fixture(scope='function')
-def strict_float32():
+@pytest.fixture(scope='function', autouse=True)
+def exception_verbosity():
     config = theano.configparser.change_flags(
-        warn_float64='raise',
-        floatX='float32')
+        exception_verbosity='high')
     with config:
         yield
 
 
-@pytest.fixture('session', params=[
-    np.random.uniform(size=(1000, 10))
-])
-def datagen(request):
-    return DataSampler(request.param)
+@pytest.fixture(scope='function', autouse=False)
+def strict_float32():
+    if theano.config.floatX == 'float32':
+        config = theano.configparser.change_flags(
+            warn_float64='raise')
+        with config:
+            yield
+    else:
+        yield
+
+
+@pytest.fixture('function', autouse=False)
+def seeded_test():
+    # TODO: use this instead of SeededTest
+    np.random.seed(42)
+    pm.set_tt_rng(42)

pymc3/tests/test_minibatches.py

@@ -13,6 +13,35 @@
 from pymc3.theanof import GeneratorOp
 
 
+class _DataSampler(object):
+    """
+    Not for users
+    """
+    def __init__(self, data, batchsize=50, random_seed=42, dtype='floatX'):
+        self.dtype = theano.config.floatX if dtype == 'floatX' else dtype
+        self.rng = np.random.RandomState(random_seed)
+        self.data = data
+        self.n = batchsize
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        idx = (self.rng
+               .uniform(size=self.n,
+                        low=0.0,
+                        high=self.data.shape[0] - 1e-16)
+               .astype('int64'))
+        return np.asarray(self.data[idx], self.dtype)
+
+    next = __next__
+
+
+@pytest.fixture('module')
+def datagen():
+    return _DataSampler(np.random.uniform(size=(1000, 10)))
+
+
 def integers():
     i = 0
     while True: