Merge pull request #1270 from sunpy/conda_for_all_the_ci

Major CI improvements

.travis.yml

@@ -1,83 +1,78 @@
-language: python
+# We use C because conda manages our Python vesion
+# and running on OS/X dosen't work with Python.
+language: c
 
-python:
-  - 2.7
+os:
+    - linux
+    - osx
 
-virtualenv:
-     system_site_packages: false
+# These are currently not used, but we want to cache the packages dir
+# to reduce the number of packages to download.
+cache:
+    directories:
+        $HOME/miniconda/pkgs
+        /Users/travis/miniconda3/pkgs
 
-# Configure the build environment. Global varibles are defined for all
-# configurations. Each configuration in the matrix is run for each python version
-# above.
+# Configure the build environment. Global varibles are defined for all configurations.
 env:
     global:
-        - PIP_WHEEL_COMMAND="--find-links http://sunpy.cadair.com/wheelhouse/index.html --use-wheel --upgrade --trusted-host sunpy.cadair.com"
-        - TEST_MODE='offline'
+        # Modify these to represent the newest versions
         - LATEST_NUMPY=1.9.1
         - LATEST_PANDAS=0.15.2
+        # Fixed global vars
+        - PYTHON_VERSION=2.7
+        - TEST_MODE='offline'
         - NUMPY_VERSION=$LATEST_NUMPY
         - PANDAS_VERSION=$LATEST_PANDAS
+
 matrix:
-     include:
-        - python: 2.7
-          env: NUMPY_VERSION=1.8.1
-        - python: 2.7
-          env: PANDAS_VERSION=0.14.1
-        - python: 2.7
-          env: TEST_MODE='astropy-dev'
-        - python: 2.7
-          env: TEST_MODE='sphinx'
-        - python: 2.7
-          env: TEST_MODE='online'
+    include:
+         - os: linux
+           env: NUMPY_VERSION=1.8.1
+         - os: linux
+           env: PANDAS_VERSION=0.14.1
+         - os: linux
+           env: TEST_MODE='astropy-dev'
+         - os: linux
+           env: TEST_MODE='sphinx'
+         - os: linux
+           env: TEST_MODE='online'
 
+# If the online build fails, we don't want it to report the whole build broken.
+    allow_failures:
+        - env: TEST_MODE='online'
+
+# Install miniconda
 before_install:
-# Make sure the system is upto date and install scipy requirements
-   - sudo apt-get update -qq
-   - sudo apt-get install -qq libatlas-dev liblapack-dev gfortran
-   - if [[ $TEST_MODE == 'sphinx' ]]; then sudo apt-get install graphviz texlive-latex-extra dvipng; fi
-# Make sure pip is at the latest version (>1.4 for wheels)
-   - pip install pip --upgrade
-# Wheels also need a recent version of setuptools
-   - pip install setuptools>=0.8 --upgrade
-# Install more upto date openjpeg library.
-   - wget http://openjpeg.googlecode.com/files/openjpeg-1.5.0-Linux-x86_64.tar.gz
-   - sudo tar -xvf openjpeg-1.5.0-Linux-x86_64.tar.gz --strip-components=1 -C /
-# Install coveralls
-   - pip install $PIP_WHEEL_COMMAND coveralls pytest-cov pytest-xdist
+    - source continuous-integration/travis/install_$TRAVIS_OS_NAME.sh
 
-# Install all SunPy dependacies using wheels.
 install:
-# Install Numpy first so we force the version we are using
-   - pip install $PIP_WHEEL_COMMAND numpy==$NUMPY_VERSION
-   - pip install $PIP_WHEEL_COMMAND matplotlib
-   - pip install $PIP_WHEEL_COMMAND -vvv scipy
-   - pip install $PIP_WHEEL_COMMAND pandas==$PANDAS_VERSION
-   - pip install $PIP_WHEEL_COMMAND suds-jurko beautifulsoup4 requests
-   - pip install $PIP_WHEEL_COMMAND pytest pytest-xdist 
-   - pip install $PIP_WHEEL_COMMAND glymur
-   - pip install $PIP_WHEEL_COMMAND cython jinja2
-   - pip install $PIP_WHEEL_COMMAND scikit-image
-   - pip install $PIP_WHEEL_COMMAND sqlalchemy
-   - if [[ $TEST_MODE != 'astropy-dev' ]]; then pip install $PIP_WHEEL_COMMAND astropy>=0.4.0; fi 
-   - if [[ $TEST_MODE == 'astropy-dev' ]]; then pip install git+git://github.com/astropy/astropy.git; fi 
-# Install sphinx if needed
-   - if [[ $TEST_MODE == 'sphinx' ]]; then pip install $PIP_WHEEL_COMMAND sphinx==1.2.2; fi 
+    # Add a conda env and set up extra binstar channels
+    - conda create --yes -n test python=$PYTHON_VERSION
+    - conda config --add channels sunpy --add channels astropy-ci-extras --add channels https://conda.binstar.org/sunpy/channel/citesting
+    - source activate test
+    # Install default dependancies
+    - conda install --yes pip
+    - conda install -q --yes numpy=$NUMPY_VERSION pandas=$PANDAS_VERSION
+    - conda install -q --yes scipy matplotlib requests beautiful-soup sqlalchemy scikit-image pytest jinja2 cython
+    - pip install pytest-cov coverage coveralls
+    # Install sunpy conda packages
+    - conda install -q --yes suds-jurko glymur
+    # Install the correct version of astropy
+    - if [[ $TEST_MODE != 'astropy-dev' ]]; then conda install -q --yes astropy>=0.4.0; fi 
+    - if [[ $TEST_MODE == 'astropy-dev' ]]; then pip install git+git://github.com/astropy/astropy.git; fi 
+    # Install sphinx if needed
+    - if [[ $TEST_MODE == 'sphinx' ]]; then conda install -q --yes sphinx; fi 
 
-# Install SunPy and run tests.
 script:
-   - if [[ $TEST_MODE == 'sphinx' ]]; then python setup.py build_sphinx -w; fi
-   - if [[ $TEST_MODE == 'offline' || $TEST_MODE == 'astropy-dev' ]]; then python setup.py test --coverage --cov-report=html --offline-only; fi
-   - if [[ $TEST_MODE == 'online' ]]; then python setup.py test --coverage --cov-report=term-missing --parallel=8; fi
-
-after_success:
-   - if [[ $TEST_MODE == 'online' ]]; then coveralls; fi
+    - if [[ $TEST_MODE == 'sphinx' ]]; then python setup.py build_sphinx -w; fi
+    - if [[ $TEST_MODE == 'offline' || $TEST_MODE == 'astropy-dev' ]]; then python setup.py test --offline-only; fi
+    - if [[ $TEST_MODE == 'online' ]]; then python setup.py test --coverage --cov-report=html; fi
+    - if [[ $TEST_MODE == 'online' ]]; then coveralls --rcfile='./sunpy/tests/coveragerc'; fi
 
 # Notify the IRC channel of build status
 notifications:
   irc: "chat.freenode.net#SunPy"
   webhooks:
     urls:
       - https://webhooks.gitter.im/e/d1bf84e1bc1293e4dbc5
-    on_success: always  # options: [always|never|change] default: always
-    on_failure: always  # options: [always|never|change] default: always
-    on_start: false     # default: false

appveyor.yml

@@ -4,40 +4,43 @@
 environment:
 
   global:
+      # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+      # /E:ON and /V:ON options are not enabled in the batch script intepreter
+      # See: http://stackoverflow.com/a/13751649/163740
+      CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\continuous-integration\\appveyor\\run_with_env.cmd"
       PYTHON: "C:\\conda"
       MINICONDA_VERSION: "3.5.5"
 
   matrix:
       - PYTHON_VERSION: "2.7"
-        NUMPY_VERSION: "1.9.1"
 
 platform:
     - x64
 
 install:
     # Install miniconda using a powershell script.
-    - "powershell ./.install-miniconda.ps1"
+    - "powershell ./continuous-integration/appveyor/install-miniconda.ps1"
     - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
 
     # Check that we have the expected version and architecture for Python
-    - "python --version"
+    - "%CMD_IN_ENV% python --version"
 
     # Install the build and runtime dependencies of the project.
     - "conda update --yes conda"
     # Create a conda environment using the astropy bonus packages
     - "conda create -q --yes -n test -c astropy-ci-extras python=%PYTHON_VERSION%"
+    - "conda config --add channels https://conda.binstar.org/sunpy"
     - "activate test"
 
-    # Install specified version of numpy and dependencies
-    - "conda install -q --yes pip mingw libpython"
-    - "conda install -q --yes numpy=%NUMPY_VERSION% scipy astropy matplotlib pandas requests beautiful-soup sqlalchemy scikit-image pytest"
+    # Install default dependancies
+    - "conda install -q --yes numpy scipy astropy matplotlib pandas requests beautiful-soup sqlalchemy scikit-image pytest"
 
-    # Install non-conda packages
-    - "pip install suds"
+    # Install sunpy conda packages
+    - "conda install -q --yes suds-jurko glymur"
 
 # Not a .NET project, we build SunPy in the install step instead
 build: false
 
 test_script:
-    - "python setup.py test --offline-only"
+    - "%CMD_IN_ENV% python setup.py test --offline-only"
 

continuous-integration/appveyor/run_with_env.cmd

@@ -0,0 +1,47 @@
+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
+::
+:: To build extensions for 64 bit Python 2, we need to configure environment
+:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
+::
+:: 32 bit builds do not require specific environment configurations.
+::
+:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
+:: cmd interpreter, at least for (SDK v7.0)
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
+:: http://stackoverflow.com/a/13751649/163740
+::
+:: Author: Olivier Grisel
+:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+@ECHO OFF
+
+SET COMMAND_TO_RUN=%*
+SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
+
+SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%"
+IF %MAJOR_PYTHON_VERSION% == "2" (
+    SET WINDOWS_SDK_VERSION="v7.0"
+) ELSE IF %MAJOR_PYTHON_VERSION% == "3" (
+    SET WINDOWS_SDK_VERSION="v7.1"
+) ELSE (
+    ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
+    EXIT 1
+)
+
+IF "%PYTHON_ARCH%"=="64" (
+    ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
+    SET DISTUTILS_USE_SDK=1
+    SET MSSdk=1
+    "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
+    "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
+    ECHO Executing: %COMMAND_TO_RUN%
+    call %COMMAND_TO_RUN% || EXIT 1
+) ELSE (
+    ECHO Using default MSVC build environment for 32 bit architecture
+    ECHO Executing: %COMMAND_TO_RUN%
+    call %COMMAND_TO_RUN% || EXIT 1
+)

continuous-integration/travis/install_linux.sh

@@ -0,0 +1,23 @@
+###############################################################################
+# Install Linux Packages
+###############################################################################
+sudo apt-get upgrade
+sudo apt-get install -qq libatlas-dev liblapack-dev gfortran
+if [[ $TEST_MODE == 'sphinx' ]]; then sudo apt-get install graphviz texlive-latex-extra dvipng; fi
+
+# Install more upto date openjpeg library.
+wget http://openjpeg.googlecode.com/files/openjpeg-1.5.0-Linux-x86_64.tar.gz
+sudo tar -xvf openjpeg-1.5.0-Linux-x86_64.tar.gz --strip-components=1 -C /
+
+###############################################################################
+# Install miniconda
+###############################################################################
+MINICONDA_URL="http://repo.continuum.io/miniconda"
+MINICONDA_FILE="Miniconda-3.5.5-Linux-x86_64.sh"
+wget "${MINICONDA_URL}/${MINICONDA_FILE}"
+bash $MINICONDA_FILE -b
+
+export PATH=$HOME/miniconda/bin:$PATH
+
+conda update --yes conda
+conda install -q --yes binstar conda-build

continuous-integration/travis/install_osx.sh

@@ -0,0 +1,9 @@
+MINICONDA_URL="http://repo.continuum.io/miniconda"
+MINICONDA_FILE="Miniconda3-3.7.3-MacOSX-x86_64.sh"
+wget "${MINICONDA_URL}/${MINICONDA_FILE}"
+bash $MINICONDA_FILE -b
+
+export PATH=/Users/travis/miniconda3/bin:$PATH
+
+conda update --yes conda
+conda install -q --yes binstar conda-build

sunpy/database/tests/test_tables.py

@@ -17,7 +17,7 @@
 from sunpy.data.test import rootdir as testdir
 from sunpy.data.test.waveunit import waveunitdir, MQ_IMAGE
 from sunpy.data.sample import RHESSI_IMAGE, EIT_195_IMAGE
-
+from sunpy.tests.helpers import skip_windows
 import pytest
 
 
@@ -250,7 +250,6 @@ def test_entries_from_dir_recursively_true():
     assert len(entries) == 44
     # Older val = 31.
 
-
 def test_entries_from_dir_recursively_false():
     entries = list(
         entries_from_dir(testdir, False, default_waveunit='angstrom'))

sunpy/io/tests/test_filetools.py

@@ -6,7 +6,7 @@
 import sunpy.io
 import sunpy.data.test
 
-from sunpy.tests.helpers import skip_ana, skip_glymur
+from sunpy.tests.helpers import skip_ana, skip_glymur, skip_windows
 
 #==============================================================================
 # Test, read, get_header and write through the file independant layer

sunpy/tests/helpers.py

@@ -9,7 +9,7 @@
 import astropy.units as u
 import numpy as np
 
-__all__ = ['skip_glymur', 'skip_ana', 'warnings_as_errors']
+__all__ = ['skip_windows', 'skip_glymur', 'skip_ana', 'warnings_as_errors']
 
 # SunPy's JPEG2000 capabilities rely on the glymur library.  First we check to
 # make sure that glymur imports correctly before proceeding.
@@ -18,7 +18,11 @@
 except ImportError:
     SKIP_GLYMUR = True
 else:
-    SKIP_GLYMUR = False
+    # See if we have a C backend
+    if any((glymur.lib.openjp2.OPENJP2, glymur.lib.openjpeg.OPENJPEG)):
+        SKIP_GLYMUR = False
+    else:
+        SKIP_GLYMUR = True
 
 # Skip ana tests if we are on Windows or we can't import the c extension.
 import platform
@@ -34,6 +38,8 @@
 else:
     SKIP_ANA = SKIP_ANA or False
 
+skip_windows = pytest.mark.skipif(platform.system() == 'Windows', reason="Windows")
+
 skip_glymur = pytest.mark.skipif(SKIP_GLYMUR, reason="Glymur can not be imported")
 
 skip_ana = pytest.mark.skipif(SKIP_ANA, reason="ANA is not availible")