Merge pull request #363 from puremourning/osx-more-py3

[READY] Test python 3.3 on OS X

.travis.yml

@@ -36,8 +36,6 @@ matrix:
       env: USE_CLANG_COMPLETER=false YCMD_PYTHON_VERSION=2.6
     - os: osx
       env: USE_CLANG_COMPLETER=true YCMD_PYTHON_VERSION=2.6
-    - os: osx
-      env: USE_CLANG_COMPLETER=true YCMD_PYTHON_VERSION=3.3
     - os: linux
       env: USE_CLANG_COMPLETER=true YCMD_PYTHON_VERSION=2.7
 addons:

build.py

@@ -148,17 +148,42 @@ def CustomPythonCmakeArgs():
     print( 'Searching for python with prefix: {0} and lib {1}:'.format(
       python_prefix, which_python ) )
 
-    if p.isfile( '{0}.a'.format( lib_python ) ):
-      python_library = '{0}.a'.format( lib_python )
-    # This check is for CYGWIN
-    elif p.isfile( '{0}.dll.a'.format( lib_python ) ):
-      python_library = '{0}.dll.a'.format( lib_python )
-    elif p.isfile( '{0}.dylib'.format( lib_python ) ):
+    # On MacOS, ycmd does not work with statically linked python library.
+    # It typically manifests with the following error
+    # when there is a self-compiled python without --enable-framework (or,
+    # technically --enable-shared):
+    #   Fatal Python error: PyThreadState_Get: no current thread
+    #
+    # The most likely explanation for this is that both the ycm_core.so and the
+    # python binary include copies of libpython.a (or whatever included
+    # objects). When python executable starts it initializes only the globals
+    # within its copy, so when ycm_core.so's copy starts executing, it points at
+    # its own copy which is uninitialized.
+    #
+    # Some platforms' dynamic linkers (ld.so) are able to resolve this when
+    # loading shared libraries at runtime[citation needed], but OSX seemingly
+    # cannot.
+    #
+    # So we do 2 things special on OS X:
+    #  - look for a .dylib first
+    #  - if we find a .a, raise an error.
+    #
+    if p.isfile( '{0}.dylib'.format( lib_python ) ):
       python_library = '{0}.dylib'.format( lib_python )
     elif p.isfile( '/usr/lib/lib{0}.dylib'.format( which_python ) ):
       # For no clear reason, python2.6 only exists in /usr/lib on OS X and
       # not in the python prefix location
       python_library = '/usr/lib/lib{0}.dylib'.format( which_python )
+    elif p.isfile( '{0}.a'.format( lib_python ) ):
+      if OnMac():
+        sys.exit( 'ERROR: You must use a python compiled with '
+                  '--enable-shared or --enable-framework (and thus a {0}.dylib '
+                  'library) on OS X'.format( lib_python ) )
+
+      python_library = '{0}.a'.format( lib_python )
+    # This check is for CYGWIN
+    elif p.isfile( '{0}.dll.a'.format( lib_python ) ):
+      python_library = '{0}.dll.a'.format( lib_python )
     else:
       sys.exit( 'ERROR: Unable to find an appropriate python library' )
 

ci/travis/travis_install.linux.sh

@@ -1,34 +1,5 @@
 # Linux-specific installation
 
-#############
-# pyenv setup
-#############
-
-# DON'T exit if error
-set +e
-git clone https://github.com/yyuu/pyenv.git ~/.pyenv
-git fetch --tags
-git checkout v20160202
-# Exit if error
-set -e
-
-export PYENV_ROOT="$HOME/.pyenv"
-export PATH="$PYENV_ROOT/bin:$PATH"
-
-eval "$(pyenv init -)"
-
-if [ "${YCMD_PYTHON_VERSION}" == "2.6" ]; then
-  PYENV_VERSION="2.6.6"
-elif [ "${YCMD_PYTHON_VERSION}" == "2.7" ]; then
-  PYENV_VERSION="2.7.6"
-else
-  PYENV_VERSION="3.3.0"
-fi
-
-pyenv install --skip-existing ${PYENV_VERSION}
-pyenv rehash
-pyenv global ${PYENV_VERSION}
-
 # We can't use sudo, so we have to approximate the behaviour of the following:
 # $ sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.7 100
 

ci/travis/travis_install.osx.sh

@@ -3,41 +3,25 @@
 # There's a homebrew bug which causes brew update to fail the first time. Run
 # it twice to workaround. https://github.com/Homebrew/homebrew/issues/42553
 brew update || brew update
-brew install node.js || brew outdated node.js || brew upgrade node.js
-brew install go || brew outdated go || brew upgrade go
-brew install ninja
-
-# TODO: In theory, we should be able to just use the pyenv python setup from
-# travis_install.linux.sh for both Linux and OS X. In practice, we get:
-#   Fatal Python error: PyThreadState_Get: no current thread
-# on OS X. So we do something special for OS X. We shouldn't have to though.
-
-YCMD_VENV_DIR=${HOME}/venvs/ycmd_test
-
-# OS X comes with 2 versions of python by default, and a neat system
-# (versioner) to switch between them:
-#   /usr/bin/python2.7 - python 2.7
-#   /usr/bin/python2.6 - python 2.6
-#
-# We just set the system default to match it
-# http://stackoverflow.com/q/6998545
-defaults write com.apple.versioner.python Version ${YCMD_PYTHON_VERSION}
-
-# virtualenv is not installed by default on OS X under python2.6, and we don't
-# have sudo, so we install it manually. There is no "latest" link, so we have
-# to install a specific version.
-VENV_VERSION=13.1.2
-
-curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${VENV_VERSION}.tar.gz
-tar xvfz virtualenv-${VENV_VERSION}.tar.gz
-python virtualenv-${VENV_VERSION}/virtualenv.py -p python${YCMD_PYTHON_VERSION} ${YCMD_VENV_DIR}
-
-# virtualenv doesn't copy python-config https://github.com/pypa/virtualenv/issues/169
-# but our build system uses it
-cp /usr/bin/python${YCMD_PYTHON_VERSION}-config ${YCMD_VENV_DIR}/bin/python-config
-
-# virtualenv script is noisy, so don't print every command
-set +v
-source ${YCMD_VENV_DIR}/bin/activate
-set -v
 
+# List of homebrew formulae to install in the order they appear.
+# We require node, go and ninja for our build and tests, and all the others are
+# dependencies of pyenv.
+REQUIREMENTS="node.js
+              go
+              ninja
+              readline
+              autoconf
+              pkg-config
+              openssl"
+
+# Install node, go, ninja, pyenv and dependencies
+for pkg in $REQUIREMENTS; do
+  # Install package, or upgrade it if it is already installed
+  brew install $pkg || brew outdated $pkg || brew upgrade $pkg
+done
+
+# In order to work with ycmd, python *must* be built as a shared library. The
+# most compatible way to do this on OS X is with --enable-framework. This is
+# set via the PYTHON_CONFIGURE_OPTS option
+export PYTHON_CONFIGURE_OPTS="--enable-framework"

ci/travis/travis_install.sh

@@ -11,6 +11,34 @@ set -ev
 #  - setup the correct python for $YCMD_PYTHON_VERSION
 source ci/travis/travis_install.${TRAVIS_OS_NAME}.sh
 
+#############
+# pyenv setup
+#############
+
+# DON'T exit if error
+set +e
+git clone https://github.com/yyuu/pyenv.git ~/.pyenv
+git fetch --tags
+git checkout v20160202
+# Exit if error
+set -e
+
+export PYENV_ROOT="$HOME/.pyenv"
+export PATH="$PYENV_ROOT/bin:$PATH"
+
+eval "$(pyenv init -)"
+
+if [ "${YCMD_PYTHON_VERSION}" == "2.6" ]; then
+  PYENV_VERSION="2.6.6"
+elif [ "${YCMD_PYTHON_VERSION}" == "2.7" ]; then
+  PYENV_VERSION="2.7.6"
+else
+  PYENV_VERSION="3.3.0"
+fi
+
+pyenv install --skip-existing ${PYENV_VERSION}
+pyenv rehash
+pyenv global ${PYENV_VERSION}
 
 # It is quite easy to get the above series of steps wrong. Verify that the
 # version of python actually in the path and used is the version that was