Switching from virtualenv to pyenv for travis

Virtualenv has some sort of bug that prevents us from setting up a py3
virtualenv.

.travis.yml

@@ -34,6 +34,8 @@ matrix:
       env: USE_CLANG_COMPLETER=true YCMD_PYTHON_VERSION=2.7 COVERAGE=true
     - os: osx
       env: USE_CLANG_COMPLETER=false YCMD_PYTHON_VERSION=2.6
+    - os: osx
+      env: USE_CLANG_COMPLETER=true YCMD_PYTHON_VERSION=2.6
     - os: osx
       env: USE_CLANG_COMPLETER=true YCMD_PYTHON_VERSION=3.3
     - os: linux
@@ -53,20 +55,33 @@ addons:
      - cmake
      - clang-3.7
      - ninja-build
-     - python2.6
-     - python2.6-dev
-     - python2.7
-     - python2.7-dev
+     # The confusing part is that on Travis Linux with YCMD_PYTHON_VERSION=3.3,
+     # we build the C++ parts against the below system python3.3, but run
+     # against the pyenv python3.3. This is because stupid cmake 2.8.11 has a
+     # bug preventing it from finding the pyenv pythons (ostensibly; I haven't
+     # checked, but online reports say the issue is gone with cmake 3.4).
+     # Everything still works though, it's just weird.
      - python3.3
      - python3.3-dev
-     - python-virtualenv
-     - python-setuptools
+     # Everything below is a Python build dep (though it depends on Python
+     # version). We need them because pyenv builds Python.
+     - libssl-dev
+     - zlib1g-dev
+     - libbz2-dev
+     - libreadline-dev
+     - libsqlite3-dev
+     - wget
+     - curl
+     - llvm
+     - libncurses5-dev
+     - libncursesw5-dev
 cache:
   directories:
     - $HOME/.cache/pip  # Python packages from pip
     - $HOME/.dnx/packages  # .Net packages?
     - $HOME/.multirust  # what multirust downloads
     - $HOME/.cargo  # cargo package deps
+    - $HOME/.pyenv  # pyenv
     - $TRAVIS_BUILD_DIR/clang_archives  # clang downloads
     # dependency compilation output
     - $TRAVIS_BUILD_DIR/third_party/racerd/target

ci/travis/travis_install.linux.sh

@@ -1,5 +1,34 @@
 # Linux-specific installation
 
+#############
+# pyenv setup
+#############
+
+# DON'T exit if error
+set +e
+git clone https://github.com/yyuu/pyenv.git ~/.pyenv
+git fetch --tags
+git checkout v20160202
+# Exit if error
+set -e
+
+export PYENV_ROOT="$HOME/.pyenv"
+export PATH="$PYENV_ROOT/bin:$PATH"
+
+eval "$(pyenv init -)"
+
+if [ "${YCMD_PYTHON_VERSION}" == "2.6" ]; then
+  PYENV_VERSION="2.6.6"
+elif [ "${YCMD_PYTHON_VERSION}" == "2.7" ]; then
+  PYENV_VERSION="2.7.6"
+else
+  PYENV_VERSION="3.3.0"
+fi
+
+pyenv install --skip-existing ${PYENV_VERSION}
+pyenv rehash
+pyenv global ${PYENV_VERSION}
+
 # We can't use sudo, so we have to approximate the behaviour of the following:
 # $ sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.7 100
 
@@ -17,4 +46,3 @@ ln -s /usr/bin/clang-3.7 ${HOME}/bin/gcc
 
 export PATH=${HOME}/bin:${PATH}
 
-virtualenv -p python${YCMD_PYTHON_VERSION} ${YCMD_VENV_DIR}

ci/travis/travis_install.osx.sh

@@ -7,6 +7,13 @@ brew install node.js || brew outdated node.js || brew upgrade node.js
 brew install go || brew outdated go || brew upgrade go
 brew install ninja
 
+# TODO: In theory, we should be able to just use the pyenv python setup from
+# travis_install.linux.sh for both Linux and OS X. In practice, we get:
+#   Fatal Python error: PyThreadState_Get: no current thread
+# on OS X. So we do something special for OS X. We shouldn't have to though.
+
+YCMD_VENV_DIR=${HOME}/venvs/ycmd_test
+
 # OS X comes with 2 versions of python by default, and a neat system
 # (versioner) to switch between them:
 #   /usr/bin/python2.7 - python 2.7
@@ -24,3 +31,13 @@ VENV_VERSION=13.1.2
 curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${VENV_VERSION}.tar.gz
 tar xvfz virtualenv-${VENV_VERSION}.tar.gz
 python virtualenv-${VENV_VERSION}/virtualenv.py -p python${YCMD_PYTHON_VERSION} ${YCMD_VENV_DIR}
+
+# virtualenv doesn't copy python-config https://github.com/pypa/virtualenv/issues/169
+# but our build system uses it
+cp /usr/bin/python${YCMD_PYTHON_VERSION}-config ${YCMD_VENV_DIR}/bin/python-config
+
+# virtualenv script is noisy, so don't print every command
+set +v
+source ${YCMD_VENV_DIR}/bin/activate
+set -v
+

ci/travis/travis_install.sh

@@ -2,31 +2,28 @@
 
 set -ev
 
-YCMD_VENV_DIR=${HOME}/venvs/ycmd_test
+####################
+# OS-specific setup
+####################
 
 # Requirements of OS-specific install:
 #  - install any software which is not installed by Travis configuration
-#  - create (but don't activate) a virtualenv for the python version
-#    ${YCMD_PYTHON_VERSION} in the directory ${YCMD_VENV_DIR}, e.g.
-#    virtualenv -p python${YCMD_PYTHON_VERSION} ${YCMD_VENV_DIR}
+#  - setup the correct python for $YCMD_PYTHON_VERSION
 source ci/travis/travis_install.${TRAVIS_OS_NAME}.sh
 
-# virtualenv doesn't copy python-config https://github.com/pypa/virtualenv/issues/169
-# but our build system uses it
-cp /usr/bin/python${YCMD_PYTHON_VERSION}-config ${YCMD_VENV_DIR}/bin/python-config
-
-# virtualenv script is noisy, so don't print every command
-set +v
-source ${YCMD_VENV_DIR}/bin/activate
-set -v
 
 # It is quite easy to get the above series of steps wrong. Verify that the
 # version of python actually in the path and used is the version that was
 # requested, and fail the build if we broke the travis setup
-python_version=$(python -c 'import sys; print "{0}.{1}".format( sys.version_info[0], sys.version_info[1] )')
+python_version=$(python -c 'import sys; print( "{0}.{1}".format( sys.version_info[0], sys.version_info[1] ) )')
 echo "Checking python version (actual ${python_version} vs expected ${YCMD_PYTHON_VERSION})"
 test ${python_version} == ${YCMD_PYTHON_VERSION}
 
+
+############
+# pip setup
+############
+
 pip install -U pip wheel setuptools
 pip install -r test_requirements.txt
 npm install -g typescript
@@ -36,6 +33,11 @@ if [ x"${COVERAGE}" = x"true" ]; then
   pip install coveralls
 fi
 
+
+############
+# rust setup
+############
+
 # Need rust available, but travis doesn't give it to you without language: rust
 pushd ${HOME}
 git clone --recursive https://github.com/brson/multirust