Update build versions (#840)

* make tf code tf2 compatible * update build versions used * StanFit to StanMCMC * skip cmdstanpy tests with py35 * add missing sys import * Use temporarily StanFit object until pypi is updated * Keep still using StanFit * fix * update CmdStan fit creation * fix cmdstanpy fit creation, import * revert StanMCMC to StanFit
arviz-devs · Oct 8, 2019 · c84e0d4 · c84e0d4
1 parent a2ec64d
commit c84e0d4
Show file tree

Hide file tree

Showing 9 changed files with 56 additions and 19 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -10,22 +10,22 @@ matrix:
   include:
     - name: "Python 3.6 Unit Test"
       python: 3.6
-      env: PYTHON_VERSION=3.6 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=2 NAME="UNIT"
-    - name: "Python 3.6 Unit Test - PyStan=3 Pyro=0.2.1 Emcee=3"
+      env: PYTHON_VERSION=3.6 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=latest NAME="UNIT"
+    - name: "Python 3.6 Unit Test - PyStan=3 Pyro=0.2.1 Emcee=2 TF=1"
       python: 3.6
-      env: PYTHON_VERSION=3.6 PYSTAN_VERSION=preview PYRO_VERSION=0.2.1 PYTORCH_VERSION=0.4.1 EMCEE_VERSION=3 NAME="UNIT"
+      env: PYTHON_VERSION=3.6 PYSTAN_VERSION=preview PYRO_VERSION=0.2.1 PYTORCH_VERSION=0.4.1 EMCEE_VERSION=2 TF_VERSION=1 NAME="UNIT"
     - name: "Python 3.5 Unit Test"
       python: 3.5
-      env: PYTHON_VERSION=3.5 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=2 NAME="UNIT"
+      env: PYTHON_VERSION=3.5 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=latest NAME="UNIT"
     - name: "Python 3.7 Unit Test"
       python: 3.7
-      env: PYTHON_VERSION=3.7 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=2 NAME="UNIT"
+      env: PYTHON_VERSION=3.7 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=latest NAME="UNIT"
     - name: "Python 3.6 Lint"
       python: 3.6
-      env: PYTHON_VERSION=3.6 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=2 NAME="LINT"
+      env: PYTHON_VERSION=3.6 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=latest NAME="LINT"
     - name: "Python 3.6 Sphinx"
       python: 3.6
-      env: PYTHON_VERSION=3.6 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=2 NAME="SPHINX"
+      env: PYTHON_VERSION=3.6 PYSTAN_VERSION=latest PYRO_VERSION=latest EMCEE_VERSION=latest NAME="SPHINX"
 
 addons:
   apt:

diff --git a/arviz/data/io_tfp.py b/arviz/data/io_tfp.py
@@ -49,6 +49,12 @@ def __init__(
         self.tf = tf  # pylint: disable=invalid-name
         self.ed = ed  # pylint: disable=invalid-name
 
+        if int(self.tf.__version__[0]) > 1:
+            import tensorflow.compat.v1 as tf  # pylint: disable=import-error
+
+            tf.disable_v2_behavior()
+            self.tf = tf  # pylint: disable=invalid-name
+
     def posterior_to_xarray(self):
         """Convert the posterior to an xarray dataset."""
         data = {}

diff --git a/arviz/tests/helpers.py b/arviz/tests/helpers.py
@@ -346,6 +346,11 @@ def tfp_schools_model(num_schools, treatment_stddevs):
     import tensorflow_probability.python.edward2 as ed
     import tensorflow as tf
 
+    if int(tf.__version__[0]) > 1:
+        import tensorflow.compat.v1 as tf  # pylint: disable=import-error
+
+        tf.disable_v2_behavior()
+
     avg_effect = ed.Normal(loc=0.0, scale=10.0, name="avg_effect")  # `mu`
     avg_stddev = ed.Normal(loc=5.0, scale=1.0, name="avg_stddev")  # `log(tau)`
     school_effects_standard = ed.Normal(
@@ -364,6 +369,11 @@ def tfp_noncentered_schools(data, draws, chains):
     import tensorflow_probability.python.edward2 as ed
     import tensorflow as tf
 
+    if int(tf.__version__[0]) > 1:
+        import tensorflow.compat.v1 as tf  # pylint: disable=import-error
+
+        tf.disable_v2_behavior()
+
     del chains
 
     log_joint = ed.make_log_joint_fn(tfp_schools_model)

diff --git a/arviz/tests/test_data_cmdstanpy.py b/arviz/tests/test_data_cmdstanpy.py
@@ -1,9 +1,11 @@
 # pylint: disable=redefined-outer-name
 from glob import glob
 import os
+import sys
 import numpy as np
 import pytest
 
+
 from arviz import from_cmdstanpy
 from .helpers import (  # pylint: disable=unused-import
     chains,
@@ -15,6 +17,7 @@
 )
 
 
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="CmdStanPy is supported only Python 3.6+")
 class TestDataCmdStanPy:
     @pytest.fixture(scope="session")
     def data_directory(self):
@@ -30,14 +33,16 @@ def filepaths(self, data_directory):
     @pytest.fixture(scope="class")
     def data(self, filepaths):
         from cmdstanpy import StanFit
+        from cmdstanpy.stanfit import RunSet
         from cmdstanpy.model import CmdStanArgs, SamplerArgs
 
         class Data:
             args = CmdStanArgs(
                 "dummy.stan", "dummy.exe", list(range(1, 5)), method_args=SamplerArgs()
             )
-            obj = StanFit(args)
-            obj._csv_files = filepaths  # pylint: disable=protected-access
+            runset_obj = RunSet(args)
+            runset_obj._csv_files = filepaths  # pylint: disable=protected-access
+            obj = StanFit(runset_obj)
             obj._validate_csv_files()  # pylint: disable=protected-access
             obj._assemble_sample()  # pylint: disable=protected-access
 

diff --git a/arviz/tests/test_data_tfp.py b/arviz/tests/test_data_tfp.py
@@ -44,6 +44,11 @@ def get_inference_data3(self, data, eight_schools_params):
         """Read with observed Tensor var_names and dims."""
         import tensorflow as tf
 
+        if int(tf.__version__[0]) > 1:
+            import tensorflow.compat.v1 as tf  # pylint: disable=import-error
+
+            tf.disable_v2_behavior()
+
         inference_data = from_tfp(
             data.obj,
             var_names=["mu", "tau", "eta"],

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -12,32 +12,33 @@ jobs:
         PYTHON_VERSION: 3.6
         PYSTAN_VERSION: "latest"
         PYRO_VERSION: "latest"
-        EMCEE_VERSION: 2
+        EMCEE_VERSION: "latest"
         NAME: "UNIT"
-      Python_36_Unit_Test_PyStan_3_Pyro_0.2.1_Emcee_3:
+      Python_36_Unit_Test_PyStan_3_Pyro_0.2.1_Emcee_2_tf_1:
         PYTHON_VERSION: 3.6
         PYSTAN_VERSION: "preview"
         PYRO_VERSION: 0.2.1
         PYTORCH_VERSION: 0.4.1
-        EMCEE_VERSION: 3
+        EMCEE_VERSION: 2
+        TF_VERSION: 1
         NAME: "UNIT"
       Python_35_Unit_Test:
         PYTHON_VERSION: 3.5
         PYSTAN_VERSION: "latest"
         PYRO_VERSION: "latest"
-        EMCEE_VERSION: 2
+        EMCEE_VERSION: "latest"
         NAME: "UNIT"
       Python_37_Unit_Test:
         PYTHON_VERSION: 3.7
         PYSTAN_VERSION: "latest"
         PYRO_VERSION: "latest"
-        EMCEE_VERSION: 2
+        EMCEE_VERSION: "latest"
         NAME: "UNIT"
       Python_36_Lint:
         PYTHON_VERSION: 3.6
         PYSTAN_VERSION: "latest"
         PYRO_VERSION: "latest"
-        EMCEE_VERSION: 2
+        EMCEE_VERSION: "latest"
         NAME: "LINT"
 
   steps:

diff --git a/scripts/Dockerfile b/scripts/Dockerfile
@@ -8,12 +8,14 @@ ARG PYSTAN_VERSION
 ARG PYRO_VERSION
 ARG PYTORCH_VERSION
 ARG EMCEE_VERSION
+ARG TF_VERSION
 
 ENV PYTHON_VERSION=${PYTHON_VERSION}
 ENV PYSTAN_VERSION=${PYSTAN_VERSION}
 ENV PYRO_VERSION=${PYRO_VERSION}
 ENV PYTORCH_VERSION=${PYTORCH_VERSION}
 ENV EMCEE_VERSION=${EMCEE_VERSION}
+ENV TF_VERSION=${TF_VERSION}
 
 # Change behavior of create_test.sh script
 ENV DOCKER_BUILD=true

diff --git a/scripts/container.sh b/scripts/container.sh
@@ -13,6 +13,7 @@ if [[ $* == *--build* ]]; then
         --build-arg PYRO_VERSION=${PYRO_VERSION} \
         --build-arg PYTORCH_VERSION=${PYTORCH_VERSION} \
         --build-arg EMCEE_VERSION=${EMCEE_VERSION} \
+        --build-arg TF_VERSION=${TF_VERSION} \
         --rm
 fi
 

diff --git a/scripts/create_testenv.sh b/scripts/create_testenv.sh
@@ -12,11 +12,12 @@ PYTHON_VERSION=${PYTHON_VERSION:-${TRAVIS_PYTHON_VERSION:-3.6}}
 PYSTAN_VERSION=${PYSTAN_VERSION:-latest}
 PYTORCH_VERSION=${PYTORCH_VERSION:-latest}
 PYRO_VERSION=${PYRO_VERSION:-latest}
-EMCEE_VERSION=${EMCEE_VERSION:-2}
+EMCEE_VERSION=${EMCEE_VERSION:-latest}
+TF_VERSION=${TF_VERSION:-latest}
 
 
 if [[ $* != *--global* ]]; then
-    ENVNAME="testenv_${PYTHON_VERSION}_PYSTAN_${PYSTAN_VERSION}_PYRO_${PYRO_VERSION}_EMCEE_${EMCEE_VERSION}"
+    ENVNAME="testenv_${PYTHON_VERSION}_PYSTAN_${PYSTAN_VERSION}_PYRO_${PYRO_VERSION}_EMCEE_${EMCEE_VERSION}_TF_${TF_VERSION}"
 
     if conda env list | grep -q ${ENVNAME}
     then
@@ -70,10 +71,16 @@ else
   pip --no-cache-dir install pyro-ppl==${PYRO_VERSION}
 fi
 
-if [ "$EMCEE_VERSION" = "2" ]; then
+if [ "$EMCEE_VERSION" = "latest" ]; then
   pip --no-cache-dir install emcee
 else
-  pip --no-cache-dir install git+https://github.com/dfm/emcee.git
+  pip --no-cache-dir install "emcee<3"
+fi
+
+if [ "$TF_VERSION" = "latest" ]; then
+  pip --no-cache-dir install tensorflow
+else
+  pip --no-cache-dir install tensorflow==1.14 tensorflow_probability==0.7
 fi
 
 #  Install editable using the setup.py