Update Travis infrastructure to Python 3.6/3.7 and current numpy (The…

…ano#7)

* Fix subtensor numpy warning

* Fix simple typo: varible->size -> variable->size

Closes Theano#6734

* Counteract hypot redefinition for old Python versions

* fix doc warning

* Update extending_theano.txt

* OrderedDict cannot be imported from collections.abc, it is not abstract

See Theano#6664 (comment)

* Move more collections.abc imports to theano.compat

* Use math.gcd instead of fractions.gcd when possible

fractions.gcd() function has been removed from Python 3.9, it was
deprecated since Python 3.5.

https://docs.python.org/3.9/whatsnew/3.9.html#removed

* DOC: Allow building with Sphinx >= 2.0

* MAINT: Avoid SyntaxWarnings on import in Python 3.8

* flake8 code style fixes

* DOC: Retain compatibility with older Sphinx

* MAINT: Be compatible with numpy 1.17 and scipy 1.3

Fixes Theano#6715.  Includes and supersedes Theano#6721.

* flake8 code style fixes

* Remove doctest in python 3.4

* Do not install sphinx_rtd_theme for python 3.4 as the installation doesn't work anymore.

* fixing typo in GammaIncC Op

* re-add min > max clip test with fixed reference

* DOC: min > max clip may not match numpy

* updating travis build for Python 3

* order of stages

* conda activate

* installing conda properly

* conda install on travis

* conda activate issues on travis

* conda activate issues on travis

* doctest needs older version of numpy

* doctest

* reverting setup.cfg

* conda version conflicts

* need old sphinx version

* sphinx version

* don't bother building docs

* moved nosetester import

* also moving known failures plugin

Co-authored-by: Adrian Seyboldt <adrian.seyboldt@gmail.com>
Co-authored-by: Frédéric Bastien <frederic.bastien@gmail.com>
Co-authored-by: Tim Gates <tim.gates@iress.com>
Co-authored-by: Marcel Bargull <marcel.bargull@udo.edu>
Co-authored-by: Arnaud Bergeron <abergeron@gmail.com>
Co-authored-by: Miro Hrončok <miro@hroncok.cz>
Co-authored-by: Rebecca N. Palmer <rebecca_palmer@zoho.com>
Co-authored-by: Frederic Bastien <fbastien@nvidia.com>

.travis.yml

@@ -1,139 +1,90 @@
 # After changing this file, check it on:
 # http://lint.travis-ci.org/
-sudo: false
+os: linux
+dist: xenial
 cache:
   directories:
     - $HOME/.cache/pip
     - $HOME/.theano
-    - $HOME/download # Sufficient to add miniconda.sh to TRAVIS cache.
-    - $HOME/miniconda2 # Add the installation to TRAVIS cache.
-
+    - $HOME/download
+    - $HOME/miniconda
 
 language: python
-# For now, Python versions have to be listed in the "jobs" matrix
 
-# NB:
-# In before_install and install sections below,
-# some codes have been moved to separate files
-# to better handle if-else shell syntax
-# for multiple lines. New files are in
-# new folder ".travis".
+python:
+  - "3.7"
+  - "3.6"
 
-# command to install dependencies
-before_install:
-  - ./.travis/travis_before_install.sh
-  - export PATH=/home/travis/miniconda2/bin:$PATH
+stages:
+  - doc
+  - test
+
+env:
+  global:
+    - NUMPY_VERSION=1.18.1
+  jobs:
+    # - DOC=1 PART="theano/tests/test_flake8.py"
+    - PART="theano/compat theano/compile theano/d3viz theano/gof theano/misc theano/sandbox theano/scalar theano/scan_module theano/tests -e test_flake8.py theano/typed_list"
+    - PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
+    - PART="theano/tensor/tests/test_basic.py"
+    - PART="theano/tensor/tests/test_elemwise.py theano/tensor/tests/test_opt.py"
+    - PART="theano/tensor/nnet -e test_abstract_conv.py"
+    - PART="theano/tensor/nnet/tests/test_abstract_conv.py"
 
 addons:
   apt_packages:
-   - texlive-latex-recommended
-   - texlive-latex-extra
-   - texlive-fonts-recommended
-   - dvipng
+    - texlive-latex-recommended
+    - texlive-latex-extra
+    - texlive-fonts-recommended
+    - dvipng
+
+before_install:
+  - source .travis/travis_before_install.sh
 
 install:
-  - ./.travis/travis_install.sh
-  - source activate pyenv
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install pydot; else pip install pydot-ng; fi
-  - pip install . --no-deps --upgrade
-  - pip install flake8-future-import parameterized
-  - if [[ $TRAVIS_PYTHON_VERSION != '3.4' ]]; then pip install sphinx_rtd_theme; fi
-  # nose-exclude plugin allow us to tell nosetests to exclude folder with --exclude-dir=path/to/directory.
-  - pip install nose-exclude nose-timer
-  - if [[ $NUMPY_VERSION == '1.13.1' ]]; then conda install --yes -q scipy=0.19.1; else conda install --yes -q scipy=0.14; fi  # Try to reinstall it to fix the problem
+  - source .travis/travis_install.sh
 
 jobs:
   include:
-    # define prototype for doctest
     - &doctest
       stage: doc
-      python: "2.7"
-      env: NUMPY_VERSION=1.9.1 DOC=1 PART="theano/tests/test_flake8.py"
-    # re-use prototype, changing the Python version
-    - <<: *doctest
-      python: "2.7"
-      env: NUMPY_VERSION=1.13.1 DOC=1 PART="theano/tests/test_flake8.py"
-#    - <<: *doctest
-#      python: "3.4"
-#      env: NUMPY_VERSION=1.9.1 DOC=1 PART="theano/tests/test_flake8.py"
-    - <<: *doctest
-      python: "3.6"
-      env: NUMPY_VERSION=1.13.1 DOC=1 PART="theano/tests/test_flake8.py"
+      env: DOC=1 PART="theano/tests/test_flake8.py"
     - &normaltest
       stage: test
-      python: "2.7"
-      env: NUMPY_VERSION=1.9.1 PART="theano/compat theano/compile theano/d3viz theano/gof theano/misc theano/sandbox theano/scalar theano/scan_module theano/tests -e test_flake8.py theano/typed_list"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/compat theano/compile theano/d3viz theano/gof theano/misc theano/sandbox theano/scalar theano/scan_module theano/tests -e test_flake8.py theano/typed_list"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.13.1 PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
-    - <<: *normaltest
-      python: "3.6"
-      env: NUMPY_VERSION=1.13.1 PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/tests/test_basic.py"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/tests/test_basic.py"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/tests/test_elemwise.py theano/tensor/tests/test_opt.py"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/tests/test_elemwise.py theano/tensor/tests/test_opt.py"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/nnet -e test_abstract_conv.py"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/nnet -e test_abstract_conv.py"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/nnet/tests/test_abstract_conv.py"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/nnet/tests/test_abstract_conv.py"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 FLOAT32=1 PART="theano -e test_flake8.py --exclude-dir=theano/tensor/nnet --exclude-dir=theano/tensor/signal"
+      env: FAST_COMPILE=1 FLOAT32=1 PART="theano -e test_flake8.py --exclude-dir=theano/tensor/nnet --exclude-dir=theano/tensor/signal"
     - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 PART="theano -e test_flake8.py --exclude-dir=theano/tensor/nnet --exclude-dir=theano/tensor/signal"
+      env: FAST_COMPILE=1 PART="theano -e test_flake8.py --exclude-dir=theano/tensor/nnet --exclude-dir=theano/tensor/signal"
     - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 FLOAT32=1 PART="theano/tensor/nnet"
+      env: FAST_COMPILE=1 FLOAT32=1 PART="theano/tensor/nnet"
     - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 PART="theano/tensor/nnet"
+      env: FAST_COMPILE=1 PART="theano/tensor/nnet"
     - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 FLOAT32=1 PART="theano/tensor/signal"
+      env: FAST_COMPILE=1 FLOAT32=1 PART="theano/tensor/signal"
     - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 PART="theano/tensor/signal"
+      env: FAST_COMPILE=1 PART="theano/tensor/signal"
 
 script:
+  - conda activate pyenv
   - if [[ $FAST_COMPILE == "1" ]]; then export THEANO_FLAGS=$THEANO_FLAGS,mode=FAST_COMPILE; fi
   - if [[ $FLOAT32 == "1" ]]; then export THEANO_FLAGS=$THEANO_FLAGS,floatX=float32; fi
   - export THEANO_FLAGS=$THEANO_FLAGS,warn.ignore_bug_before=all,on_opt_error=raise,on_shape_error=raise,gcc.cxxflags=-pipe
   - export MKL_THREADING_LAYER=GNU
   - export MKL_NUM_THREADS=1
   - export OMP_NUM_THREADS=1
+  - which python
   - python --version
   - uname -a
   - free -m
   - df -h
   - ulimit -a
   - echo "$PART"
-# Print information to help debug problems
+  # Print information to help debug problems
   - python -c 'import numpy; print(numpy.__version__)'
   - python -c 'import theano; print(theano.__version__)'
   - python -c 'import theano; print(theano.config.__str__(print_doc=False))'
   - python -c 'import theano; assert(theano.config.blas.ldflags != "")'
-# Run tests for the given part
+  # Run tests for the given part
   - theano-nose -v --with-timer --timer-top-n 10 $PART
-  - if [[ $DOC == "1" ]]; then python doc/scripts/docgen.py --nopdf --check; fi
-  - if [[ $DOC == "1" ]]; then python doc/scripts/docgen.py --test --check; fi
 
 after_failure:
-  - cat /home/travis/.pip/pip.log
+  - cat $HOME/.pip/pip.log

.travis/travis_before_install.sh

@@ -1,13 +1,26 @@
 #!/usr/bin/env bash
+
+set -e
+
 # Install miniconda to avoid compiling scipy
-if test -e $HOME/miniconda2/bin ; then
+if test -e $HOME/miniconda/bin ; then
     echo "miniconda already installed."
 else
     echo "Installing miniconda."
-    rm -rf $HOME/miniconda2
+    rm -rf $HOME/miniconda
     mkdir -p $HOME/download
     if [[ -d $HOME/download/miniconda.sh ]] ; then rm -rf $HOME/download/miniconda.sh ; fi
-    wget -c https://repo.continuum.io/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O $HOME/download/miniconda.sh
-    chmod +x $HOME/download/miniconda.sh
-    $HOME/download/miniconda.sh -b
+    wget -c https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O $HOME/download/miniconda.sh
+
+    mkdir $HOME/.conda
+    bash $HOME/download/miniconda.sh -b -p $HOME/miniconda
 fi
+
+$HOME/miniconda/bin/conda init bash
+source ~/.bash_profile
+conda activate base
+
+hash -r
+conda config --set always_yes yes --set changeps1 no
+conda update -q conda
+conda info -a

.travis/travis_install.sh

@@ -1,17 +1,19 @@
 #!/usr/bin/env bash
-# In Python 3.4, we test the min version of NumPy and SciPy. In Python 2.7, we test more recent version.
-if test -e $HOME/miniconda2/envs/pyenv; then
-    echo "pyenv already exists."
+
+set -e
+
+if [[ $DOC == "1" ]]; then
+# this is a hack to deal with the fact that the docs and flake8 config are all set up
+# for old versions
+conda create --yes -q -n pyenv python=3.6 numpy=1.13.1
+conda activate pyenv
+conda install --yes -q mkl numpy=1.13.1 scipy=0.19.1 nose=1.3.7 pip flake8=3.5 six=1.11.0 pep8=1.7.1 pyflakes=1.6.0 mkl-service graphviz
+python -m pip install pydot-ng flake8-future-import parameterized nose-exclude nose-timer
 else
-    echo "Creating pyenv."
-    if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then conda create --yes -q -n pyenv python=2.7 ; fi
-    if [[ $TRAVIS_PYTHON_VERSION == '3.4' ]]; then conda create --yes -q -n pyenv python=3.4 ; fi
-    if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then conda create --yes -q -n pyenv python=3.6 ; fi
+conda create --yes -q -n pyenv python=$TRAVIS_PYTHON_VERSION
+conda activate pyenv
+conda install --yes -q mkl numpy scipy nose pip flake8 six pep8 pyflakes sphinx mkl-service graphviz  # libgfortran
+python -m pip install -q pydot-ng flake8-future-import parameterized sphinx_rtd_theme nose-exclude nose-timer
 fi
 
-source activate pyenv
-if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $NUMPY_VERSION == '1.9.1' ]]; then conda install --yes -q mkl numpy=1.9.1 scipy=0.14.0 nose=1.3.0 pip flake8=2.3 six=1.9.0 pep8=1.6.2 pyflakes=0.8.1 sphinx=1.5.1 mkl-service libgfortran=1 graphviz; fi
-if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $NUMPY_VERSION == '1.13.1' ]]; then conda install --yes -q mkl numpy=1.13.1 scipy=0.19.1 nose=1.3.0 pip flake8=2.3 six=1.9.0 pep8=1.6.2 pyflakes=0.8.1 sphinx=1.5.1 mkl-service libgfortran=3 graphviz; fi
-if [[ $TRAVIS_PYTHON_VERSION == '3.4' && $NUMPY_VERSION == '1.9.1' ]]; then conda install --yes -q mkl numpy=1.9.1 scipy=0.14.0 nose=1.3.4 pip flake8=2.3 six=1.9.0 pep8=1.6.2 pyflakes=0.8.1 sphinx=1.5.1 mkl-service libgfortran=1 graphviz pygments=2.1.3; fi
-if [[ $TRAVIS_PYTHON_VERSION == '3.6' && $NUMPY_VERSION == '1.13.1' ]]; then conda install --yes -q mkl numpy=1.13.1 scipy=0.19.1 nose=1.3.7 pip flake8=3.5 six=1.11.0 pep8=1.7.1 pyflakes=1.6.0 sphinx=1.5.1 mkl-service libgfortran=3 graphviz; fi
-source deactivate
+python -m pip install --no-deps --upgrade -e .

bin/theano_nose.py

@@ -101,11 +101,17 @@ def main_function():
     if '--without-knownfailure' not in sys.argv:
         try:
             from numpy.testing.noseclasses import KnownFailure
-            addplugins.append(KnownFailure())
         except ImportError:
-            _logger.warn(
-                'KnownFailure plugin from NumPy could not be imported. '
-                'Use --without-knownfailure to disable this warning.')
+            try:
+                from numpy.testing._private.noseclasses import KnownFailure
+                addplugins.append(KnownFailure())
+            except ImportError:
+                _logger.warn(
+                    'KnownFailure plugin from NumPy could not be imported. '
+                    'Use --without-knownfailure to disable this warning.')
+        else:
+            addplugins.append(KnownFailure())
+
     else:
         sys.argv.remove('--without-knownfailure')