Merge pull request #170 from mcg1969/simplify-ci

CI simplification; use conda build
anaconda · Dec 19, 2018 · 0a6cea8 · 0a6cea8
2 parents b654efe + 5aa570c
commit 0a6cea8
Show file tree

Hide file tree

Showing 10 changed files with 129 additions and 222 deletions.
diff --git a/.binstar.yml b/.binstar.yml
diff --git a/.checkignore b/.checkignore
diff --git a/.pep257 b/.pep257
diff --git a/.travis.yml b/.travis.yml
@@ -1,89 +1,43 @@
-# https://travis-ci.org/Anaconda-Platform/anaconda-project/
-
-# language: python sets up virtualenv and pip that we don't need.
-# omitting language gives us ruby stuff. c seems likely to be a minimal setup.
+# Don't use language: python; this gives us an unnecessary virtualenv
 language: c
-# this is here to avoid a hang with python 2.7, suggested in
+# Seems to be needed to prevent hanging in Python 2.7
 # https://github.com/travis-ci/travis-ci/issues/6861
 sudo: required
 
 env:
-  # All supported python versions
-  - MINICONDA_VERSION=latest TRAVIS_PYTHON_VERSION=2.7 TEST_TARGET=tests CONDA_CANARY=false
-  - MINICONDA_VERSION=latest TRAVIS_PYTHON_VERSION=3.6 TEST_TARGET=tests CONDA_CANARY=false
-  - MINICONDA_VERSION=latest TRAVIS_PYTHON_VERSION=3.7 TEST_TARGET=tests CONDA_CANARY=false
-  # Can build the package
-  - MINICONDA_VERSION=latest TRAVIS_PYTHON_VERSION=3.7 TEST_TARGET=packaging CONDA_CANARY=false
-  # conda canary
-  - MINICONDA_VERSION=latest TRAVIS_PYTHON_VERSION=3.7 TEST_TARGET=tests CONDA_CANARY=true
+  # Build and test package on all supported python version
+  - BUILD_TARGET=3.7
+  - BUILD_TARGET=3.6 
+  - BUILD_TARGET=2.7
+  # Smoketest package on conda canary
+  - BUILD_TARGET=3.7 CONDA_CANARY="-c conda-canary"
 
 os:
   - linux
   - osx
 
 matrix:
-  # disable all but one job on OS X, we just want to smoke test OS X.
-  exclude:
-    - os: osx
-      env: MINICONDA_VERSION=latest TRAVIS_PYTHON_VERSION=2.7 TEST_TARGET=tests CONDA_CANARY=false
-    - os: osx
-      env: MINICONDA_VERSION=latest TRAVIS_PYTHON_VERSION=3.6 TEST_TARGET=tests CONDA_CANARY=false
-    - os: osx
-      env: MINICONDA_VERSION=latest TRAVIS_PYTHON_VERSION=3.7 TEST_TARGET=packaging CONDA_CANARY=false
+  # conda canary shouldn't block the build, just show us an FYI
   allow_failures:
-    # conda canary shouldn't block the build, just show us an FYI
-    - env: MINICONDA_VERSION=latest TRAVIS_PYTHON_VERSION=3.7 TEST_TARGET=tests CONDA_CANARY=true
+    - env: BUILD_TARGET=3.7 CONDA_CANARY="-c conda-canary"
 
 install:
-  - printenv | sort
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      MINICONDA_PYVERSION=2 ;
-    else
-      MINICONDA_PYVERSION=3 ;
-    fi ;
-    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      MINICONDA_OS=Linux ;
-    else
-      MINICONDA_OS=MacOSX ;
-    fi ;
-    echo "Fetching Python $MINICONDA_PYVERSION miniconda for $MINICONDA_OS" ;
-    wget https://repo.continuum.io/miniconda/Miniconda$MINICONDA_PYVERSION-$MINICONDA_VERSION-$MINICONDA_OS-x86_64.sh -O miniconda.sh
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then MINICONDA_OS=Linux; else MINICONDA_OS=MacOSX; fi
+  - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-$MINICONDA_OS-x86_64.sh -O miniconda.sh
   - bash miniconda.sh -b -p "$HOME"/miniconda
-  - source "$HOME"/miniconda/bin/activate root
-  - printenv | sort
+  - source "$HOME"/miniconda/etc/profile.d/conda.sh
   - conda config --set always_yes yes --set changeps1 no --set auto_update_conda false
-  # Use the latest conda
-  - conda update conda --no-pin -q
-  - if test x"$CONDA_CANARY" = xtrue ; then conda update -n root -c conda-canary conda; fi
-  - conda install -n root conda-build psutil
+  # yapf is pinned to help make sure we get the same results here as a user does locally.
+  - conda install -q $CONDA_CANARY -c defaults -c conda-forge conda conda-build conda-verify codecov flake8 pep257 yapf==0.25.0
   - conda info -a
-  # we don't use environment.yml in here because I'm not sure how to parameterize the
-  # Python version and OS version in it. If we switch to being recursive and use
-  # anaconda-project.yml instead of environment.yml there's a way to do that though.
-  # Adding conda-forge as a lower-priority channel to pick up a few packages
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      OS_PACKAGES=libffi ;
-    fi ;
-    conda create -q -n test-environment -c defaults -c conda-forge
-        python="$TRAVIS_PYTHON_VERSION" coverage tornado=4 pytest pytest-cov
-        pip redis notebook bokeh ruamel_yaml anaconda-client requests psutil
-        flake8 pep257 yapf keyring $OS_PACKAGES
-  - source activate test-environment
-  - unset CONDA_ENV_PATH # because the older conda in miniconda sets this, confusing some tests
-  - export PATH=`echo "$PATH" | sed -e s@"$HOME"/miniconda/bin:@@g`
+  - export LANG=en_US.UTF-8
+  - export COVERAGE_DIR=":$HOME/htmlcov"
   - printenv | sort
 
 script:
-  - if test "$TEST_TARGET" = tests; then
-      LANG=en_US.UTF-8 python scripts/run_tests.py --pytest-args ' -vv'; 
-    fi
-  - if test "$TRAVIS_PYTHON_VERSION" = "3.5" && test "$TEST_TARGET" = "packaging"; then
-      git fetch --unshallow ;
-      LANG=en_US.UTF-8 python scripts/create_conda_packages.py;
-    else
-      echo "Not building packages on $TRAVIS_PYTHON_VERSION target=$TEST_TARGET" ;
-    fi
+  - conda build conda.recipe --python=$BUILD_TARGET
+  - conda activate
+  - python scripts/run_tests.py --format-only
 
 after_success:
-  - conda install -q -n test-environment -c conda-forge codecov
   - codecov
diff --git a/anaconda_project/internal/test/test_streaming_popen.py b/anaconda_project/internal/test/test_streaming_popen.py
@@ -16,8 +16,16 @@
 from anaconda_project.internal.test.tmpfile_utils import tmp_script_commandline
 
 
-def add_lineseps(lines):
-    return list(map(lambda l: l + os.linesep, lines))
+def detect_linesep(lines):
+    # We discovered that for Windows / Py 2.7 / PYTHONUNBUFFERED=1
+    # the line separator is in fact "\n" for stdout and "\r\n" for stderr
+    # so we needed to be more intelligent about detecting this.
+    return '\r\n' if any('\r' in line for line in lines) else '\n'
+
+
+def add_lineseps(lines, sep=None):
+    sep = sep or os.linesep
+    return list(map(lambda l: l + sep, lines))
 
 
 def test_streaming():
@@ -73,17 +81,19 @@ def on_stderr(data):
         stderr_from_callback.append(data)
 
     (p, out_lines, err_lines) = streaming_popen.popen(print_stuff, on_stdout, on_stderr)
+    sep_out = detect_linesep(out_lines)
+    sep_err = detect_linesep(err_lines)
 
-    expected_out = add_lineseps([u'a', u'b', u'c', u'd', u'123456', u'💯 🌟', u'1', u'2', u'3', u'4', u'5'])
+    expected_out = add_lineseps([u'a', u'b', u'c', u'd', u'123456', u'💯 🌟', u'1', u'2', u'3', u'4', u'5'], sep_out)
     if platform.system() == 'Windows':
         # Windows can't output unicode
         if _PY2:
-            expected_out[5] = u'\U0001f4af \U0001f31f\r\n'
+            expected_out[5] = u'\U0001f4af \U0001f31f' + sep_out
         else:
-            expected_out[5] = u"Windows\r\n"
+            expected_out[5] = u"Windows" + sep_out
 
     expected_out.append(u'6')  # no newline after this one
-    expected_err = add_lineseps([u'x', u'y', u'z'])
+    expected_err = add_lineseps([u'x', u'y', u'z'], sep_err)
 
     assert expected_out == out_lines
     assert "".join(expected_out) == "".join(stdout_from_callback)
@@ -122,8 +132,9 @@ def on_stderr(data):
         stderr_from_callback.append(data)
 
     (p, out_lines, err_lines) = streaming_popen.popen(print_bad, on_stdout, on_stderr)
+    sep_out = detect_linesep(out_lines)
 
-    expected_out = add_lineseps([u'hello', u'B��\x00\x01�goodbye'])
+    expected_out = add_lineseps([u'hello', u'B��\x00\x01�goodbye'], sep_out)
     expected_err = []
 
     assert expected_out == out_lines
@@ -145,9 +156,11 @@ def test_callbacks_are_none():
 """)
 
     (p, out_lines, err_lines) = streaming_popen.popen(print_stuff, None, None)
+    sep_out = detect_linesep(out_lines)
+    sep_err = detect_linesep(err_lines)
 
-    expected_out = add_lineseps(['a'])
-    expected_err = add_lineseps(['b'])
+    expected_out = add_lineseps(['a'], sep_out)
+    expected_err = add_lineseps(['b'], sep_err)
 
     assert expected_out == out_lines
     assert expected_err == err_lines

diff --git a/anaconda_project/test/test_prepare.py b/anaconda_project/test/test_prepare.py
@@ -126,23 +126,23 @@ def prepare_system_environ(dirname):
         project = project_no_dedicated_env(dirname)
         os_environ_copy = deepcopy(os.environ)
         result = prepare_without_interaction(project)
+        assert result
+        assert result.errors == []
         assert project.directory_path == strip_environ(result.environ)['PROJECT_DIR']
         # os.environ wasn't modified
         assert os_environ_copy == os.environ
         # result.environ inherits everything in os.environ
-        for key in os_environ_copy:
-            if key == 'PATH' and platform.system() == 'Windows' and result.environ[key] != os.environ[key]:
-                print("prepare changed PATH on Windows and ideally it would not.")
-            else:
-                if key == 'PATH' and result.environ[key] != os.environ[key]:
-                    original = os.environ[key].split(os.pathsep)
-                    updated = result.environ[key].split(os.pathsep)
-                    print("ORIGINAL PATH: " + repr(original))
-                    print("UPDATED PATH: " + repr(updated))
-                    assert original == updated
-                assert result.errors == []
-                assert result
-                assert result.environ.get(key) == os.environ.get(key)
+        for key, original in os_environ_copy.items():
+            updated = result.environ.get(key)
+            if key == 'PATH' and updated != original:
+                if platform.system() == 'Windows':
+                    print("prepare changed PATH on Windows and ideally it would not.")
+                    continue
+                updated = updated.split(os.pathsep)
+                original = original.split(os.pathsep)
+                print("ORIGINAL PATH: " + repr(original))
+                print("UPDATED PATH: " + repr(updated))
+            assert updated == original
 
     with_directory_contents_completing_project_file({
         DEFAULT_PROJECT_FILENAME: """

diff --git a/appveyor.yml b/appveyor.yml
@@ -1,56 +1,37 @@
-# https://ci.appveyor.com/project/ContinuumAnalytics/anaconda-project-fqvvy/
+# Build only the master branch, tagged commits, and pull requests
+branches:
+  only:
+    - master
+    - /\d+\.\d+.*/
+
+# Don't run the (redundant) branch build with a pull request
+skip_branch_with_pr: true
 
 matrix:
   fast_finish: true
 
 environment:
-  global:
-    PYTHON: "C:\\conda"
-    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci-helpers\\appveyor\\windows_sdk.cmd"
-    PYTHON_ARCH: "64" # needs to be set for CMD_IN_ENV to succeed. If a mix
-                      # of 32 bit and 64 bit builds are needed, move this
-                      # to the matrix section.
+  PYTHONUNBUFFERED: 1
+  COVERAGE_DIR: ""
+  MINICONDA: C:\\Miniconda3-x64
   matrix:
-    - PYTHON_VERSION: "2.7"
-    - PYTHON_VERSION: "3.6"
-    - PYTHON_VERSION: "3.7"
-
-platform:
-  -x64
+    - PYTHON_VERSION: 3.7
+    - PYTHON_VERSION: 3.6
+    - PYTHON_VERSION: 2.7
 
 install:
-  - if "%PYTHON_VERSION%" == "3.6" set "BASE_PYTHON_VERSION=36"
-  - if "%PYTHON_VERSION%" == "3.7" set "BASE_PYTHON_VERSION=3"
-  - if "%PYTHON_ARCH%" == "64" set "ARCH_LABEL=-x64"
-  # These are already installed on appveyor.  Update them.
-  - set "CONDA_ROOT=C:\Miniconda%BASE_PYTHON_VERSION%%ARCH_LABEL%"
-  # we can't "activate root" with sufficiently-old conda which appveyor has (?)
-  # - "%CONDA_ROOT%\\Scripts\\activate root"
-  - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%"
-  - set PATH
-  - echo CONDA_PREFIX %CONDA_PREFIX%
-  - conda config --set always_yes yes
-  - conda config --add channels conda-forge --add channels defaults
-  # Use the latest conda
-  - conda install -n root -q conda --no-pin
-  # activate with newer conda, may set different variables
-  - "%CONDA_ROOT%\\Scripts\\activate root"
-  - echo CONDA_PREFIX %CONDA_PREFIX%
+  # conda 4.5.11 seems to expect that this directory exists already
+  - mkdir C:\Users\appveyor\.conda
+  - call %MINICONDA%\Scripts\activate.bat
+  # The safety checks are simply intended to ensure that there is enough disk space
+  # and the user has the necessary permissions to make environment changes. In a CI
+  # environment these are not necessary and slow things down noticeably on Windows.
+  - conda config --set always_yes yes --set changeps1 no --set auto_update_conda no --set safety_checks disabled
+  - conda install -q conda conda-build conda-verify
   - conda info -a
-  - conda create -q -n test-environment python=%PYTHON_VERSION% keyring pip notebook bokeh ruamel_yaml anaconda-client requests tornado coverage pytest pytest-cov pep257 flake8 yapf
 
 # Not a .NET project, we build in the install step instead
 build: false
 
 test_script:
-  - echo CONDA_PREFIX %CONDA_PREFIX%
-  - "%CONDA_ROOT%\\Scripts\\activate test-environment"
-  - echo CONDA_PREFIX %CONDA_PREFIX%
-  - echo PATH %PATH%
-  - conda list
-  - conda info -a
-  - python -c "import sys; print(sys.version)"
-  - python -c "import sys; print(sys.executable)"
-  - python -c "import sys; print(sys.prefix)"
-  - python --version
-  - python scripts/run_tests.py
+  - conda build conda.recipe --python=%PYTHON_VERSION%