Merge e56443e into 323e558

.travis.yml

@@ -80,23 +80,16 @@ install:
   - conda create -n testenv --use-local --yes -c quantopian -c quantopian/label/ci pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION pandas=$PANDAS_VERSION scipy=$SCIPY_VERSION ta-lib=$TALIB_VERSION libgfortran=3.0 certifi=$CERTIFI_VERSION
   - source activate testenv
 
-  - CACHE_DIR="$HOME/.cache/.pip/pip_np""$CONDA_NPY"
-  - python -m pip install --upgrade pip==18.1 coveralls --cache-dir=$CACHE_DIR
-  # With this env var and CI both set, pip installing bcolz tries to compile it with coverage on py2, which fails to link against gcov on OSX.
+  # XXX: With TRAVIS and CI both set, pip installing bcolz tries to compile it with coverage on py2, which fails to link against gcov on OSX.
   # https://github.com/Blosc/bcolz/blob/8234a7505da5188dbaf415b7e36d4609d2c8c2f1/setup.py#L134-L136
-  - unset TRAVIS
-  - python -m pip install -r etc/requirements.txt --cache-dir=$CACHE_DIR
-  - TRAVIS=true  # restore value
-  - python -m pip install -r etc/requirements_dev.txt --cache-dir=$CACHE_DIR
-  - python -m pip install -r etc/requirements_blaze.txt --cache-dir=$CACHE_DIR  # this uses git requirements right now
-  - python -m pip install -r etc/requirements_talib.txt --cache-dir=$CACHE_DIR
-  - python -m pip install -e .[all] --cache-dir=$CACHE_DIR
+  - TRAVIS='' EXTERNAL_REQUIREMENTS='coveralls' etc/dev-install --cache-dir="$HOME/.cache/.pip/pip_np$CONDA_NPY"
+
 before_script:
   - pip freeze | sort
 script:
   - flake8 zipline tests
   - nosetests --with-coverage
-  # deactive env to get access to anaconda command
+  # deactivate env to get access to anaconda command
   - source deactivate
 
   # unshallow the clone so the conda build can clone it.

docs/source/development-guidelines.rst

@@ -27,26 +27,29 @@ If you don't already have them, you'll need some C library dependencies. You can
 
 __ install.html
 
-The following section assumes you already have virtualenvwrapper and pip installed on your system. Suggested installation of Python library dependencies used for development:
+Once you've created and activated a `virtual environment`__, run the ``etc/dev-install`` script to install all development dependencies in their required order:
+
+__ https://docs.python.org/3/library/venv.html
 
 .. code-block:: bash
 
-   $ mkvirtualenv zipline
-   $ ./etc/ordered_pip.sh ./etc/requirements.txt
-   $ pip install -r ./etc/requirements_dev.txt
-   $ pip install -r ./etc/requirements_blaze.txt 
+   $ python3 -m venv venv
+   $ source venv/bin/activate
+   $ etc/dev-install
+
+Or, using `virtualenvwrapper`__:
 
-You can build the C extensions by running:
+__ https://virtualenvwrapper.readthedocs.io/en/latest/
 
 .. code-block:: bash
 
-   $ python setup.py build_ext --inplace
+   $ mkvirtualenv zipline
+   $ etc/dev-install
 
-In case you want to install the package locally and use ``zipline`` wrapper script:
+After installation, you should be able to use the ``zipline`` command line interface from your virtualenv:
 
 .. code-block:: bash
 
-   $ pip install -e .
    $ zipline --help
 
 To finish, make sure `tests`__ pass.
@@ -61,6 +64,12 @@ If you get an error running nosetests after setting up a fresh virtualenv, pleas
    $ deactivate zipline
    $ workon zipline
 
+During development, you can rebuild the C extensions by running:
+
+.. code-block:: bash
+
+   $ python setup.py build_ext --inplace
+
 
 Development with Docker
 -----------------------

etc/dev-install

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+set -euo pipefail  # Safety mode.
+# (Travis' env command doesn't permit options in the shebang line.)
+
+# Consolidated installation script for use by both Travis and humans.
+#
+# First installs a known-good version of pip, then any requirements
+# specified in the EXTERNAL_REQUIREMENTS environment variable (e.g.,
+# coveralls); then installs the project requirements, constrained by
+# etc/requirements.txt; then editably installs zipline itself.
+#
+# Forwards positional arguments to all invocations of pip install.
+
+echo
+echo "Installing zipline using $(which python)"
+echo
+
+# New releases of pip have frequently caused strange issues. Make sure
+# we know exactly which version we're working with.
+python -m pip install pip==19.2.2 $@
+
+# Install external requirements first: if they share any of our
+# transitive dependencies, we want our pinned versions to win.
+if [ "${EXTERNAL_REQUIREMENTS:-}" ]; then
+    # Note: If EXTERNAL_REQUIREMENTS is unset, the expression in the
+    # above test expands to the empty string, which fails the test.
+    # (Simply expanding $EXTERNAL_REQUIREMENTS causes an error with the
+    # -u option, which helps prevent many other kinds of errors.)
+    echo "Installing additional packages: $EXTERNAL_REQUIREMENTS"
+    python -m pip install "$EXTERNAL_REQUIREMENTS" $@
+fi
+
+# These have to be installed first so that the other requirements can be
+# compiled against the specific versions we use.
+python -m pip install numpy Cython -c etc/requirements.txt $@
+
+# bcolz has to be compiled against our specific version of numpy:
+# by default, it uses an incompatible pre-compiled binary.
+python -m pip install --no-binary=bcolz -r etc/requirements.txt -c etc/requirements.txt $@
+
+# TODO: resolve these error messages:
+# flake8 3.6.0 has requirement setuptools>=30, but you'll have setuptools 28.8.0 which is incompatible.
+# Running setup.py (path:[venv]/src/blaze-dev/setup.py) egg_info for package blaze-dev produced metadata for project name blaze. Fix your #egg=blaze-dev fragments.
+# blaze keepalive-30.g31060532 has requirement odo>=0.5.0, but you'll have odo 0.3.2+729.gda7f26d which is incompatible.
+
+python -m pip install -r etc/requirements_dev.txt -c etc/requirements.txt $@
+python -m pip install -r etc/requirements_blaze.txt -c etc/requirements.txt $@
+
+# All requirements should already be satisfied by this point.
+python -m pip install -e .[all] -c etc/requirements.txt $@
+
+echo
+echo "Installation complete! Try running 'zipline --help'."
+echo

etc/requirements.txt

@@ -1,6 +1,6 @@
 # Incompatible with earlier PIP versions
 pip>=7.1.0
-# bcolz fails to install if this is not in the build_requires.
+# XXX: bcolz fails to install if this is not in the build_requires.
 setuptools>18.0
 
 # Logging

etc/requirements_dev.txt

@@ -49,15 +49,18 @@ matplotlib==1.5.3
 Markdown==2.6.2
 
 # Checking for old PIP packages
-futures==3.0.5
+futures==3.0.5;python_version<'3.0'
 requests-futures==0.9.7
 piprot==0.9.6
 
 # For mocking out requests fetches
 responses==0.9.0
 
 # Debugging
-pdbpp==0.9.1
+# Note: pdbpp is frequently very useful for development and debugging;
+# but not everyone likes it, and it applies nightmare-inducing sys.path
+# hacks during the site-packages discovery process, so it is no longer
+# included as a requirement. Install and use at your own risk.
 pyrepl==0.8.4;python_version<'3.0'
 fancycompleter==0.7
 wmctrl==0.3