Merge cb47efd into 0a01e24

quantopian · Jan 28, 2020 · 1cc175c · 1cc175c
2 parents 0a01e24 + cb47efd
commit 1cc175c
Show file tree

Hide file tree

Showing 22 changed files with 398 additions and 310 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -77,27 +77,28 @@ install:
   - sed -i "s/pandas==.*/pandas==%PANDAS_VERSION%/" etc/requirements.txt
   - sed -i "s/scipy==.*/scipy==%SCIPY_VERSION%/" etc/requirements.txt
   - IF NOT "%PANDAS_DATAREADER_VERSION%"=="" sed -i "s/pandas-datareader==.*/pandas-datareader==%PANDAS_DATAREADER_VERSION%/" etc/requirements.txt
-  - IF NOT "%DASK_VERSION%"=="" sed -i "s/dask\[dataframe\]==.*/dask\[dataframe\]==%DASK_VERSION%/" etc/requirements_blaze.txt
+  - IF NOT "%DASK_VERSION%"=="" sed -i "s/dask\[dataframe\]==.*/dask\[dataframe\]==%DASK_VERSION%/" etc/requirements.txt
   - cat etc/requirements.txt
   - conda info -a
   - conda install conda=4.3.30 conda-build=3.0.28 anaconda-client=1.6.3 --yes -q
   - conda list
   # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation
   - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64"
+  - "ECHO APPVEYOR_PULL_REQUEST_NUMBER is: %APPVEYOR_PULL_REQUEST_NUMBER%"
   - "%CMD_IN_ENV% python .\\ci\\make_conda_packages.py"
 
   # test that we can conda install zipline in a new env
   - conda create -n installenv --yes -q --use-local python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% zipline -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
 
-  - ps: $env:BCOLZ_VERSION=(sls "bcolz==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
-  - ps: $env:NUMEXPR_VERSION=(sls "numexpr==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
-  - ps: $env:PYTABLES_VERSION=(sls "tables==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
-  - ps: $env:H5PY_VERSION=(sls "h5py==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
-  - ps: $env:TALIB_VERSION=(sls "TA-Lib==(.*)" .\etc\requirements_talib.txt -ca).matches.groups[1].value
-  # We conda install certifi at the pinned exact version because it is a transitive dependency of zipline via requests and uses distutils for packaging. 
+  - ps: $env:BCOLZ_VERSION=(sls "bcolz==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:NUMEXPR_VERSION=(sls "numexpr==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:PYTABLES_VERSION=(sls "tables==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:H5PY_VERSION=(sls "h5py==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:TALIB_VERSION=(sls "ta-lib==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  # We conda install certifi at the pinned exact version because it is a transitive dependency of zipline via requests and uses distutils for packaging.
   # Since conda installs latest certifi by default, we would fail to uninstall that new version when trying to install the pinned version using pip later in the build:
   # "Cannot uninstall 'certifi'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall."
-  - ps: $env:CERTIFI_VERSION=(sls "certifi==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:CERTIFI_VERSION=(sls "certifi==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
   - conda create -n testenv --yes -q --use-local "pip<19" python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% pandas=%PANDAS_VERSION% scipy=%SCIPY_VERSION% ta-lib=%TALIB_VERSION% bcolz=%BCOLZ_VERSION% numexpr=%NUMEXPR_VERSION% pytables=%PYTABLES_VERSION% h5py=%H5PY_VERSION% certifi=%CERTIFI_VERSION% -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
   - activate testenv
   - bash etc/dev-install --cache-dir=%LOCALAPPDATA%\pip\Cache\pip_np%CONDA_NPY%py%CONDA_PY%

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -0,0 +1,108 @@
+name: Zipline CI
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build-and-test:
+    name: build (${{ matrix.os }}, ${{ matrix.python-version }}, ${{ matrix.pandas-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]  # TODO: Add windows-latest, macos-latest
+        python-version: [2.7, 3.5]
+        pandas-version: [0.18.1, 0.22.0]
+        include:  # TODO: Add 3.7, 3.8
+          - python-version: 2.7  # py2.7, old pandas
+            pandas-version: 0.18.1
+            numpy-version: 1.11.3
+            scipy-version: 0.17.1
+          - python-version: 3.5  # py3.5, new pandas
+            pandas-version: 0.18.1
+            numpy-version: 1.11.3
+            scipy-version: 0.17.1
+          - python-version: 3.5  # py3.5, new pandas
+            pandas-version: 0.22.0
+            numpy-version: 1.14.1
+            scipy-version: 1.0.0
+        exclude:
+          - python-version: 2.7
+            pandas-version: 0.22.0
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+      PANDAS_VERSION: ${{ matrix.pandas-version }}
+      NUMPY_VERSION: ${{ matrix.numpy-version }}
+      SCIPY_VERSION: ${{ matrix.scipy-version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Pip cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/.pip
+          key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.pandas-version }}-${{ hashFiles('**/requirements.txt') }}-pip-cache
+      - name: Install dependencies
+        run: |
+          if [ "$PANDAS_VERSION" = 0.22.0 ]; then
+            PANDAS_DATAREADER_VERSION=0.4.0 DASK_VERSION=0.17.1
+          fi
+          source ./ci/actions/overwrite_requirements.sh
+          cat etc/requirements.txt
+
+          etc/dev-install --user --cache-dir="$HOME/.cache/.pip/pip_np$CONDA_NPY"
+          pip freeze | sort
+
+          echo "::set-env name=PATH::$PATH"
+      - name: Lint
+        run: |
+          source ./testenv/bin/activate
+          flake8 zipline tests
+      - name: Tests
+        run: |
+          source ./testenv/bin/activate
+          nosetests --with-coverage
+
+  conda-build:
+    name: conda build (${{ matrix.os }}, ${{ matrix.python-version }}, ${{ matrix.pandas-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [2.7, 3.5]  # TODO: Add 3.7, 3.8
+        include:
+          - python-version: 2.7  # py2.7, old pandas
+            pandas-version: 0.18.1
+            numpy-version: 1.11.3
+            scipy-version: 0.17.1
+          - python-version: 3.5  # py3.5, new pandas
+            pandas-version: 0.18.1
+            numpy-version: 1.11.3
+            scipy-version: 0.17.1
+          - python-version: 3.5  # py3.5, new pandas
+            pandas-version: 0.22.0
+            numpy-version: 1.14.1
+            scipy-version: 1.0.0
+            dask-version: 0.17.1
+            pandas-datareader-version: 0.4.0
+    env:
+      # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets#creating-encrypted-secrets
+      ANACONDA_TOKEN: ${{ secrets.AnacondaToken }}
+      CONDA_ROOT_PYTHON_VERSION: "2.7"
+    steps:
+          - name: Checkout repository
+            uses: actions/checkout@v2
+          - name: Unshallow clone  # Unshallow the clone so the conda build can clone it.
+            run: git fetch --unshallow --tags
+          - name: Install miniconda
+            run: |
+              source ./ci/actions/install_miniconda.sh
+
+              conda info -a
+              conda install conda=4.3.30 conda-build=3.0.28 anaconda-client=1.6.3 --yes -q
+              conda list
diff --git a/.travis.yml b/.travis.yml
@@ -55,8 +55,8 @@ install:
   - conda install conda=4.3.30 conda-build=3.0.28 anaconda-client=1.6.3 --yes -q
   - conda list
 
-  - TALIB_VERSION=$(cat ./etc/requirements_talib.txt | sed "s/TA-Lib==\(.*\)/\1/")
-  - CERTIFI_VERSION=$(cat ./etc/requirements.txt | grep "certifi" | sed "s/certifi==\(.*\)/\1/")
+  - TALIB_VERSION=$(cat ./etc/requirements.txt | grep "ta-lib" | sed "s/ta-lib==\([^ ]*\) *.*/\1/")
+  - CERTIFI_VERSION=$(cat ./etc/requirements.txt | grep "certifi" | sed "s/certifi==\([^ ]*\) *.*/\1/")
   - IFS='.' read -r -a NPY_VERSION_ARR <<< "$NUMPY_VERSION"
   - CONDA_NPY=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]}
   - CONDA_PY=$TRAVIS_PYTHON_VERSION
@@ -73,7 +73,7 @@ install:
   # Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads.
   # See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details.
   - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
-  # We conda install certifi at the pinned exact version because it is a transitive dependency of zipline via requests and uses distutils for packaging. 
+  # We conda install certifi at the pinned exact version because it is a transitive dependency of zipline via requests and uses distutils for packaging.
   # Since conda installs latest certifi by default, we would fail to uninstall that new version when trying to install the pinned version using pip later in the build:
   # "Cannot uninstall 'certifi'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall."
   - conda create -n testenv --use-local --yes -c quantopian -c quantopian/label/ci pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION pandas=$PANDAS_VERSION scipy=$SCIPY_VERSION ta-lib=$TALIB_VERSION libgfortran=3.0 certifi=$CERTIFI_VERSION

diff --git a/ci/actions/install_miniconda.sh b/ci/actions/install_miniconda.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ "$RUNNER_OS" == "macOS" ]]; then
+    MINICONDA_OS=MacOSX
+else
+    MINICONDA_OS=Linux
+fi
+
+wget "https://repo.continuum.io/miniconda/Miniconda${CONDA_ROOT_PYTHON_VERSION:0:1}-4.3.30-$MINICONDA_OS-x86_64.sh" -O miniconda.sh
+chmod +x miniconda.sh
+./miniconda.sh -b -p $HOME/miniconda
+export PATH="$HOME/miniconda/bin:$PATH"
diff --git a/ci/actions/overwrite_requirements.sh b/ci/actions/overwrite_requirements.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [[ "$RUNNER_OS" == "macOS" ]]; then
+    # On OSX, sed refuses to edit in place, so give it an empty extension for the rename.
+    function sed_inplace() {
+        sed -i '' "$@"
+    }
+else
+    function sed_inplace() {
+        sed -i "$@"
+    }
+fi
+
+sed_inplace "s/numpy==.*/numpy==$NUMPY_VERSION/" etc/requirements.txt
+sed_inplace "s/pandas==.*/pandas==$PANDAS_VERSION/" etc/requirements.txt
+sed_inplace "s/scipy==.*/scipy==$SCIPY_VERSION/" etc/requirements.txt
+if [ -n "$PANDAS_DATAREADER_VERSION" ]; then
+    sed_inplace "s/pandas-datareader==.*/pandas-datareader==$PANDAS_DATAREADER_VERSION/" etc/requirements.txt
+fi
+if [ -n "$DASK_VERSION" ]; then
+    sed_inplace "s/dask\[dataframe\]==.*/dask\[dataframe\]==$DASK_VERSION/" etc/requirements.txt
+fi
diff --git a/ci/make_conda_packages.py b/ci/make_conda_packages.py
@@ -64,6 +64,6 @@ def main(env, do_upload):
 if __name__ == '__main__':
     env = os.environ.copy()
     main(env,
-         do_upload=((env.get('ANACONDA_TOKEN')
-                     and env.get('APPVEYOR_REPO_BRANCH') == 'master')
-                    and 'APPVEYOR_PULL_REQUEST_NUMBER' not in env))
+         do_upload=(env.get('ANACONDA_TOKEN')
+                    and env.get('APPVEYOR_REPO_BRANCH') == 'master'
+                    and not env.get('APPVEYOR_PULL_REQUEST_NUMBER')))
diff --git a/ci/travis/overwrite_requirements.sh b/ci/travis/overwrite_requirements.sh
@@ -18,5 +18,5 @@ if [ -n "$PANDAS_DATAREADER_VERSION" ]; then
     sed_inplace "s/pandas-datareader==.*/pandas-datareader==$PANDAS_DATAREADER_VERSION/" etc/requirements.txt
 fi
 if [ -n "$DASK_VERSION" ]; then
-    sed_inplace "s/dask\[dataframe\]==.*/dask\[dataframe\]==$DASK_VERSION/" etc/requirements_blaze.txt
+    sed_inplace "s/dask\[dataframe\]==.*/dask\[dataframe\]==$DASK_VERSION/" etc/requirements.txt
 fi
diff --git a/conda/README.md b/conda/README.md
@@ -12,7 +12,6 @@ conda-build you should be able to:
 ```bash
 conda build ta-lib
 conda build logbook
-conda build cyordereddict
 conda build zipline
 ```
 

diff --git a/conda/cyordereddict/meta.yaml b/conda/cyordereddict/meta.yaml
diff --git a/docs/source/development-guidelines.rst b/docs/source/development-guidelines.rst
@@ -20,7 +20,7 @@ First, you'll need to clone Zipline by running:
 Then check out to a new branch where you can make your changes:
 
 .. code-block:: bash
-		
+
    $ git checkout -b some-short-descriptive-name
 
 If you don't already have them, you'll need some C library dependencies. You can follow the `install guide`__ to get the appropriate dependencies.
@@ -117,12 +117,12 @@ Then run ``pip install`` TA-lib:
 
 .. code-block:: bash
 
-   $ pip install -r ./etc/requirements_talib.txt
+   $ pip install -r ./etc/requirements_talib.in -c ./etc/requirements.txt
 
 You should now be free to run tests:
 
 .. code-block:: bash
-		
+
    $ nosetests
 
 
@@ -141,10 +141,45 @@ __ https://ci.appveyor.com/project/quantopian/zipline
 
 Packaging
 ---------
+
 To learn about how we build Zipline conda packages, you can read `this`__ section in our release process notes.
 
 __ release-process.html#uploading-conda-packages
-
+
+
+Updating dependencies
+---------------------
+
+If you update the zipline codebase so that it now depends on a new version of a library,
+then you should update the lower bound on that dependency in ``etc/requirements.in``
+(or ``etc/requirements_dev.in`` as appropriate).
+We use `pip-compile`__ to find mutually compatible versions of dependencies for the
+``etc/requirements.txt`` lockfile used in our CI environments.
+
+__ https://github.com/jazzband/pip-tools/
+
+When you update a dependency in an ``.in`` file,
+you need to re-run the ``pip-compile`` command included in the header of `the lockfile`__;
+otherwise the lockfile will not meet the constraints specified to pip by zipline
+at install time (via ``etc/requirements.in`` via ``setup.py``).
+
+__ https://github.com/quantopian/zipline/tree/master/etc/requirements.txt
+
+If the zipline codebase can still support an old version of a dependency, but you want
+to update to a newer version of that library in our CI environments, then only the
+lockfile needs updating. To update the lockfile without bumping the lower bound,
+re-run the ``pip-compile`` command included in the header of the lockfile with the
+addition of the ``--upgrade-package`` or ``-P`` `flag`__, e.g.
+
+__ https://github.com/jazzband/pip-tools/#updating-requirements
+
+.. code-block:: bash
+
+   $ pip-compile --output-file=etc/reqs.txt etc/reqs.in ... -P six==1.13.0 -P "click>4.0.0"
+
+As you can see above, you can include multiple such constraints in a single invocation of ``pip-compile``.
+
+
 Contributing to the Docs
 ------------------------
 
@@ -209,7 +244,7 @@ __ https://git-scm.com/book/en/v2/Distributed-Git-Contributing-to-a-Project
 
    At least one of those calculations, max_leverage, was causing a
    divide by zero error.
-   
+
    Instead of papering over that error, the entire calculation was
    a bit suspect so removing, with possibility of adding it back in
    later with handling the case (or raising appropriate errors) when