BLD: Improve dev-install and CI config

- Upgrades known-good pip version
- Upgrades to latest setuptools (which resolves an error message)
- Uses pip env vars instead of options
- Removes the need to forward positional args
- Reduces the number of 'pip install' invocations

.appveyor.yml

@@ -100,7 +100,7 @@ install:
   - ps: $env:CERTIFI_VERSION=(sls "certifi==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value
   - conda create -n testenv --yes -q --use-local "pip<19" python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% pandas=%PANDAS_VERSION% scipy=%SCIPY_VERSION% ta-lib=%TALIB_VERSION% bcolz=%BCOLZ_VERSION% numexpr=%NUMEXPR_VERSION% pytables=%PYTABLES_VERSION% h5py=%H5PY_VERSION% certifi=%CERTIFI_VERSION% -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
   - activate testenv
-  - bash etc/dev-install --cache-dir=%LOCALAPPDATA%\pip\Cache\pip_np%CONDA_NPY%py%CONDA_PY%
+  - bash PIP_CACHE_DIR=%LOCALAPPDATA%\pip\Cache\pip_np%CONDA_NPY%py%CONDA_PY% etc/dev-install
   - python -m pip freeze | sort
 
 test_script:

.travis.yml

@@ -81,7 +81,7 @@ install:
 
   # XXX: With TRAVIS and CI both set, pip installing bcolz tries to compile it with coverage on py2, which fails to link against gcov on OSX.
   # https://github.com/Blosc/bcolz/blob/8234a7505da5188dbaf415b7e36d4609d2c8c2f1/setup.py#L134-L136
-  - TRAVIS='' EXTERNAL_REQUIREMENTS='coveralls' etc/dev-install --cache-dir="$HOME/.cache/.pip/pip_np$CONDA_NPY"
+  - TRAVIS='' EXTERNAL_REQUIREMENTS='coveralls' PIP_CACHE_DIR="$HOME/.cache/.pip/pip_np$CONDA_NPY" etc/dev-install
 
 before_script:
   - pip freeze | sort

etc/dev-install

@@ -1,25 +1,39 @@
 #!/usr/bin/env bash
+set -euo pipefail
 
 # Consolidated installation script for use by both Travis and humans.
 #
 # First installs a known-good version of pip, then any requirements
 # specified in the EXTERNAL_REQUIREMENTS environment variable (e.g.,
 # coveralls); then installs the project requirements, constrained by
 # etc/requirements.txt; then editably installs zipline itself.
-#
-# Forwards positional arguments to all invocations of pip install.
-
-# Travis' env command doesn't permit options in the shebang line, so
-# set them here.
-set -euvxo pipefail
 
 echo
 echo "Installing zipline using $(which python)"
 echo
 
+set -x
+
+python --version
+python -m pip --version
+
 # New releases of pip have frequently caused strange issues. Make sure
 # we know exactly which version we're working with.
-python -m pip install pip==19.2.2 $@
+python -m pip install pip==19.3.1
+# Python 3.5's default setuptools is 28.8.0; flake8 requires >=30.
+# The newest version of this one is probably fine.
+python -m pip install --upgrade setuptools
+
+# Pip's command line options can also be set with environment variables:
+# all subsequent invocations of pip will use these options.
+#
+# XXX: Pip works fine with relative paths if they are provided via the
+# -c/--constraint option, but not via the PIP_CONSTRAINT environment
+# variable: use `realpath` to make the path absolute.
+export PIP_CONSTRAINT="$(realpath etc/requirements.txt)"
+# XXX: bcolz has to be compiled against our specific version of numpy:
+# by default, it uses an incompatible pre-compiled wheel.
+export PIP_NO_BINARY=bcolz
 
 # Install external requirements first: if they share any of our
 # transitive dependencies, we want our pinned versions to win.
@@ -29,27 +43,31 @@ if [ "${EXTERNAL_REQUIREMENTS:-}" ]; then
     # (Simply expanding $EXTERNAL_REQUIREMENTS causes an error with the
     # -u option, which helps prevent many other kinds of errors.)
     echo "Installing additional packages: $EXTERNAL_REQUIREMENTS"
-    python -m pip install "$EXTERNAL_REQUIREMENTS" $@
+    python -m pip install "$EXTERNAL_REQUIREMENTS"
 fi
 
 # These have to be installed first so that the other requirements can be
 # compiled against the specific versions we use.
-python -m pip install numpy Cython -c etc/requirements.txt $@
+python -m pip install numpy Cython
 
-# XXX: bcolz has to be compiled against our specific version of numpy:
-# by default, it uses an incompatible pre-compiled binary.
-python -m pip install --no-binary=bcolz -r etc/requirements.txt -c etc/requirements.txt $@
+python -m pip install \
+    -r etc/requirements.txt \
+    -r etc/requirements_dev.txt \
+    -r etc/requirements_blaze.txt
 
-# TODO: resolve these error messages:
-# flake8 3.6.0 has requirement setuptools>=30, but you'll have setuptools 28.8.0 which is incompatible.
+# TODO: resolve this error message:
 # blaze keepalive-30.g31060532 has requirement odo>=0.5.0, but you'll have odo 0.3.2+729.gda7f26d which is incompatible.
 
-python -m pip install -r etc/requirements_dev.txt -c etc/requirements.txt $@
-python -m pip install -r etc/requirements_blaze.txt -c etc/requirements.txt $@
-
 # All requirements should already be satisfied by this point.
-python -m pip install -e .[all] -c etc/requirements.txt $@
+python -m pip install -e .[all]
+
+# Make sure it's installed and working.
+nosetests
+which zipline
+zipline --help
+
+set +x
 
 echo
-echo "Installation complete! Try running 'zipline --help'."
+echo "Installation complete!"
 echo