Merge c6b44e8 into 74010a8

.appveyor.yml

@@ -84,27 +84,28 @@ install:
   - sed -i "s/pandas==.*/pandas==%PANDAS_VERSION%/" etc/requirements.txt
   - sed -i "s/scipy==.*/scipy==%SCIPY_VERSION%/" etc/requirements.txt
   - IF NOT "%PANDAS_DATAREADER_VERSION%"=="" sed -i "s/pandas-datareader==.*/pandas-datareader==%PANDAS_DATAREADER_VERSION%/" etc/requirements.txt
-  - IF NOT "%DASK_VERSION%"=="" sed -i "s/dask\[dataframe\]==.*/dask\[dataframe\]==%DASK_VERSION%/" etc/requirements_blaze.txt
+  - IF NOT "%DASK_VERSION%"=="" sed -i "s/dask\[dataframe\]==.*/dask\[dataframe\]==%DASK_VERSION%/" etc/requirements.txt
   - cat etc/requirements.txt
   - conda info -a
   - conda install conda=4.3.30 conda-build=3.0.28 anaconda-client=1.6.3 --yes -q
   - conda list
   # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation
   - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64"
+  - "ECHO APPVEYOR_PULL_REQUEST_NUMBER is: %APPVEYOR_PULL_REQUEST_NUMBER%"
   - "%CMD_IN_ENV% python .\\ci\\make_conda_packages.py"
 
   # test that we can conda install zipline in a new env
   - conda create -n installenv --yes -q --use-local python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% zipline -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
 
-  - ps: $env:BCOLZ_VERSION=(sls "bcolz==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
-  - ps: $env:NUMEXPR_VERSION=(sls "numexpr==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
-  - ps: $env:PYTABLES_VERSION=(sls "tables==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
-  - ps: $env:H5PY_VERSION=(sls "h5py==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
-  - ps: $env:TALIB_VERSION=(sls "TA-Lib==(.*)" .\etc\requirements_talib.txt -ca).matches.groups[1].value
+  - ps: $env:BCOLZ_VERSION=(sls "bcolz==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:NUMEXPR_VERSION=(sls "numexpr==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:PYTABLES_VERSION=(sls "tables==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:H5PY_VERSION=(sls "h5py==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:TALIB_VERSION=(sls "ta-lib==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
   # We conda install certifi at the pinned exact version because it is a transitive dependency of zipline via requests and uses distutils for packaging.
   # Since conda installs latest certifi by default, we would fail to uninstall that new version when trying to install the pinned version using pip later in the build:
   # "Cannot uninstall 'certifi'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall."
-  - ps: $env:CERTIFI_VERSION=(sls "certifi==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
+  - ps: $env:CERTIFI_VERSION=(sls "certifi==([^ ]*)" .\etc\requirements.txt -ca).matches.groups[1].value
   - conda create -n testenv --yes -q --use-local "pip<19" python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% pandas=%PANDAS_VERSION% scipy=%SCIPY_VERSION% ta-lib=%TALIB_VERSION% bcolz=%BCOLZ_VERSION% numexpr=%NUMEXPR_VERSION% pytables=%PYTABLES_VERSION% h5py=%H5PY_VERSION% certifi=%CERTIFI_VERSION% -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
   - activate testenv
   - bash etc/dev-install --cache-dir=%LOCALAPPDATA%\pip\Cache\pip_np%CONDA_NPY%py%CONDA_PY%

.travis.yml

@@ -58,8 +58,8 @@ install:
   - conda install conda=4.3.30 conda-build=3.0.28 anaconda-client=1.6.3 --yes -q
   - conda list
 
-  - TALIB_VERSION=$(cat ./etc/requirements_talib.txt | sed "s/TA-Lib==\(.*\)/\1/")
-  - CERTIFI_VERSION=$(cat ./etc/requirements.txt | grep "certifi" | sed "s/certifi==\(.*\)/\1/")
+  - TALIB_VERSION=$(cat ./etc/requirements.txt | grep "ta-lib" | sed "s/ta-lib==\([^ ]*\) *.*/\1/")
+  - CERTIFI_VERSION=$(cat ./etc/requirements.txt | grep "certifi" | sed "s/certifi==\([^ ]*\) *.*/\1/")
   - IFS='.' read -r -a NPY_VERSION_ARR <<< "$NUMPY_VERSION"
   - CONDA_NPY=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]}
   - CONDA_PY=$TRAVIS_PYTHON_VERSION

ci/make_conda_packages.py

@@ -64,6 +64,6 @@ def main(env, do_upload):
 if __name__ == '__main__':
     env = os.environ.copy()
     main(env,
-         do_upload=((env.get('ANACONDA_TOKEN')
-                     and env.get('APPVEYOR_REPO_BRANCH') == 'master')
-                    and 'APPVEYOR_PULL_REQUEST_NUMBER' not in env))
+         do_upload=(env.get('ANACONDA_TOKEN')
+                    and env.get('APPVEYOR_REPO_BRANCH') == 'master'
+                    and not env.get('APPVEYOR_PULL_REQUEST_NUMBER')))

ci/travis/overwrite_requirements.sh

@@ -18,5 +18,5 @@ if [ -n "$PANDAS_DATAREADER_VERSION" ]; then
     sed_inplace "s/pandas-datareader==.*/pandas-datareader==$PANDAS_DATAREADER_VERSION/" etc/requirements.txt
 fi
 if [ -n "$DASK_VERSION" ]; then
-    sed_inplace "s/dask\[dataframe\]==.*/dask\[dataframe\]==$DASK_VERSION/" etc/requirements_blaze.txt
+    sed_inplace "s/dask\[dataframe\]==.*/dask\[dataframe\]==$DASK_VERSION/" etc/requirements.txt
 fi

conda/README.md

@@ -12,7 +12,6 @@ conda-build you should be able to:
 ```bash
 conda build ta-lib
 conda build logbook
-conda build cyordereddict
 conda build zipline
 ```
 

docs/source/development-guidelines.rst

@@ -117,7 +117,7 @@ Then run ``pip install`` TA-lib:
 
 .. code-block:: bash
 
-   $ pip install -r ./etc/requirements_talib.txt
+   $ pip install -r ./etc/requirements_talib.in -c ./etc/requirements.txt
 
 You should now be free to run tests:
 
@@ -141,10 +141,45 @@ __ https://ci.appveyor.com/project/quantopian/zipline
 
 Packaging
 ---------
+
 To learn about how we build Zipline conda packages, you can read `this`__ section in our release process notes.
 
 __ release-process.html#uploading-conda-packages
 
+
+Updating dependencies
+---------------------
+
+If you update the zipline codebase so that it now depends on a new version of a library,
+then you should update the lower bound on that dependency in ``etc/requirements.in``
+(or ``etc/requirements_dev.in`` as appropriate).
+We use `pip-compile`__ to find mutually compatible versions of dependencies for the
+``etc/requirements.txt`` lockfile used in our CI environments.
+
+__ https://github.com/jazzband/pip-tools/
+
+When you update a dependency in an ``.in`` file,
+you need to re-run the ``pip-compile`` command included in the header of `the lockfile`__;
+otherwise the lockfile will not meet the constraints specified to pip by zipline
+at install time (via ``etc/requirements.in`` via ``setup.py``).
+
+__ https://github.com/quantopian/zipline/tree/master/etc/requirements.txt
+
+If the zipline codebase can still support an old version of a dependency, but you want
+to update to a newer version of that library in our CI environments, then only the
+lockfile needs updating. To update the lockfile without bumping the lower bound,
+re-run the ``pip-compile`` command included in the header of the lockfile with the
+addition of the ``--upgrade-package`` or ``-P`` `flag`__, e.g.
+
+__ https://github.com/jazzband/pip-tools/#updating-requirements
+
+.. code-block:: bash
+
+   $ pip-compile --output-file=etc/reqs.txt etc/reqs.in ... -P six==1.13.0 -P "click>4.0.0"
+
+As you can see above, you can include multiple such constraints in a single invocation of ``pip-compile``.
+
+
 Contributing to the Docs
 ------------------------
 

etc/dev-install

@@ -34,22 +34,16 @@ fi
 
 # These have to be installed first so that the other requirements can be
 # compiled against the specific versions we use.
-python -m pip install numpy Cython -c etc/requirements.txt $@
+python -m pip install -r etc/requirements_build.in -c etc/requirements.txt $@
 
 # XXX: bcolz has to be compiled against our specific version of numpy:
 # by default, it uses an incompatible pre-compiled binary.
-python -m pip install --no-binary=bcolz -r etc/requirements.txt -c etc/requirements.txt $@
+python -m pip install --no-binary=bcolz -e .[all] -r etc/requirements_blaze.in -c etc/requirements.txt $@
 
 # TODO: resolve these error messages:
 # flake8 3.6.0 has requirement setuptools>=30, but you'll have setuptools 28.8.0 which is incompatible.
 # blaze keepalive-30.g31060532 has requirement odo>=0.5.0, but you'll have odo 0.3.2+729.gda7f26d which is incompatible.
 
-python -m pip install -r etc/requirements_dev.txt -c etc/requirements.txt $@
-python -m pip install -r etc/requirements_blaze.txt -c etc/requirements.txt $@
-
-# All requirements should already be satisfied by this point.
-python -m pip install -e .[all] -c etc/requirements.txt $@
-
 echo
 echo "Installation complete! Try running 'zipline --help'."
 echo

etc/requirements.in

@@ -0,0 +1,63 @@
+# Incompatible with earlier PIP versions
+pip>=7.1.0
+# bcolz fails to install if this is not in the build_requires.
+setuptools>18.0
+
+Logbook>=0.12.5
+
+# Command line interface helper
+click>=4.0.0
+
+# Language utilities
+six>=1.10.0
+contextlib2>=0.4.0
+python-interface>=1.5.3
+multipledispatch>=0.6.0
+# FUNctional programming utilities
+toolz>=0.8.2
+
+# Scientific Libraries
+numpy>=1.11.3
+pandas>=0.18.1,<=0.22
+pandas-datareader>=0.2.1
+scipy>=0.17.1
+# Needed for parts of pandas.stats
+patsy>=0.4.0
+statsmodels>=0.6.1
+
+# For financial risk calculations
+empyrical>=0.5.0
+
+# Dates/times/calendars
+pytz>=2018.5
+# Country Codes
+iso3166>=0.9
+trading-calendars>=1.6.1
+
+# For fetching remote data
+requests>=2.9.1
+
+# Asset writer and finder
+sqlalchemy>=1.0.8
+# For asset db management
+alembic>=0.7.7
+
+# On disk storage format for pipeline data.
+bcolz>=0.12.1,<1
+# On disk storage format for pricing data.
+h5py>=2.7.1
+tables>=3.4.3
+
+# Performance
+lru-dict>=1.1.4
+intervaltree>=2.1.0
+# faster array ops.
+bottleneck>=1.0.0
+
+# Graph algorithms used by zipline.pipeline
+networkx>=1.9.1,<2.0
+# NumericalExpression pipeline terms.
+numexpr>=2.6.1
+
+# Currency Codes
+iso4217>=1.6.20180829