From b612b328ffae8b9edb5883f30fea1fb2118e343c Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Thu, 11 Feb 2016 20:49:56 +0000 Subject: [PATCH 1/5] Force use of shared library python on OS X --- build.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/build.py b/build.py index da8e63f873..f3e83d9676 100755 --- a/build.py +++ b/build.py @@ -148,17 +148,42 @@ def CustomPythonCmakeArgs(): print( 'Searching for python with prefix: {0} and lib {1}:'.format( python_prefix, which_python ) ) - if p.isfile( '{0}.a'.format( lib_python ) ): - python_library = '{0}.a'.format( lib_python ) - # This check is for CYGWIN - elif p.isfile( '{0}.dll.a'.format( lib_python ) ): - python_library = '{0}.dll.a'.format( lib_python ) - elif p.isfile( '{0}.dylib'.format( lib_python ) ): + # On MacOS, ycmd does not work with statically linked python library. + # It typically manifests with the following error + # when there is a self-compiled python without --enable-framework (or, + # technically --enable-shared): + # Fatal Python error: PyThreadState_Get: no current thread + # + # The most likely explanation for this is that both the ycm_core.so and the + # python binary include copies of libpython.a (or whatever included + # objects). When python executable starts it initiliases only the globals + # within its copy, so when ycm_core.so's copy starts executing, it points at + # its own copy which is uninitialised. + # + # Some platforms' dynamic linkers (ld.so) are able to resolve this when + # loading shared libraries at runtime[citation needed], but OSX seemingly + # cannot. + # + # So we do 2 things special on OS X: + # - look for a .dylib first + # - if we find a .a, raise an error. + # + if p.isfile( '{0}.dylib'.format( lib_python ) ): python_library = '{0}.dylib'.format( lib_python ) elif p.isfile( '/usr/lib/lib{0}.dylib'.format( which_python ) ): # For no clear reason, python2.6 only exists in /usr/lib on OS X and # not in the python prefix location python_library = '/usr/lib/lib{0}.dylib'.format( which_python ) + elif p.isfile( '{0}.a'.format( lib_python ) ): + if OnMac(): + sys.exit( 'ERROR: You must use a python compiled with ' + '--enable-shared or --enable-framework (and thus a {0}.dylib ' + 'library) on OS X'.format( lib_python ) ) + + python_library = '{0}.a'.format( lib_python ) + # This check is for CYGWIN + elif p.isfile( '{0}.dll.a'.format( lib_python ) ): + python_library = '{0}.dll.a'.format( lib_python ) else: sys.exit( 'ERROR: Unable to find an appropriate python library' ) From 35b990dcd51cce0795a2e4adc561cfa1ecd8b746 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Thu, 11 Feb 2016 20:52:18 +0000 Subject: [PATCH 2/5] Use pyenv on OS X and support running tests on python 3. This requires ensuring that python is built with a shared library --- .travis.yml | 2 -- ci/travis/travis_install.osx.sh | 45 ++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c9612a446..2808db51a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,8 +36,6 @@ matrix: env: USE_CLANG_COMPLETER=false YCMD_PYTHON_VERSION=2.6 - os: osx env: USE_CLANG_COMPLETER=true YCMD_PYTHON_VERSION=2.6 - - os: osx - env: USE_CLANG_COMPLETER=true YCMD_PYTHON_VERSION=3.3 - os: linux env: USE_CLANG_COMPLETER=true YCMD_PYTHON_VERSION=2.7 addons: diff --git a/ci/travis/travis_install.osx.sh b/ci/travis/travis_install.osx.sh index 3877d484a8..ca00c77f08 100644 --- a/ci/travis/travis_install.osx.sh +++ b/ci/travis/travis_install.osx.sh @@ -3,39 +3,42 @@ # There's a homebrew bug which causes brew update to fail the first time. Run # it twice to workaround. https://github.com/Homebrew/homebrew/issues/42553 brew update || brew update -brew install node.js || brew outdated node.js || brew upgrade node.js -brew install go || brew outdated go || brew upgrade go -brew install ninja -# TODO: In theory, we should be able to just use the pyenv python setup from -# travis_install.linux.sh for both Linux and OS X. In practice, we get: -# Fatal Python error: PyThreadState_Get: no current thread -# on OS X. So we do something special for OS X. We shouldn't have to though. +# install node, go, ninja, pyenv and dependencies +for pkg in node.js go ninja readline autoconf pkg-config openssl pyenv; do + # install package, or upgrade it if it is already installed + brew install $pkg || brew outdated $pkg || brew upgrade $pkg +done + +# We use homebrew's defaults for PYENV_ROOT etc. +eval "$(pyenv init -)" + +if [ "${YCMD_PYTHON_VERSION}" == "2.6" ]; then + PYENV_VERSION="2.6.6" +elif [ "${YCMD_PYTHON_VERSION}" == "2.7" ]; then + PYENV_VERSION="2.7.6" +else + PYENV_VERSION="3.3.0" +fi + +# In order to work with ycmd, python *must* be built as a shared library. The +# most compatible way to do this on OS X is with --enable-framework. This is +# set via the PYTHON_CONFIGURE_OPTS option +PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing ${PYENV_VERSION} +pyenv rehash +pyenv global ${PYENV_VERSION} YCMD_VENV_DIR=${HOME}/venvs/ycmd_test -# OS X comes with 2 versions of python by default, and a neat system -# (versioner) to switch between them: -# /usr/bin/python2.7 - python 2.7 -# /usr/bin/python2.6 - python 2.6 -# -# We just set the system default to match it -# http://stackoverflow.com/q/6998545 -defaults write com.apple.versioner.python Version ${YCMD_PYTHON_VERSION} - # virtualenv is not installed by default on OS X under python2.6, and we don't # have sudo, so we install it manually. There is no "latest" link, so we have # to install a specific version. -VENV_VERSION=13.1.2 +VENV_VERSION=14.0.6 curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${VENV_VERSION}.tar.gz tar xvfz virtualenv-${VENV_VERSION}.tar.gz python virtualenv-${VENV_VERSION}/virtualenv.py -p python${YCMD_PYTHON_VERSION} ${YCMD_VENV_DIR} -# virtualenv doesn't copy python-config https://github.com/pypa/virtualenv/issues/169 -# but our build system uses it -cp /usr/bin/python${YCMD_PYTHON_VERSION}-config ${YCMD_VENV_DIR}/bin/python-config - # virtualenv script is noisy, so don't print every command set +v source ${YCMD_VENV_DIR}/bin/activate From 7789695f6057bd185959355cc03e60f661c29172 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Thu, 11 Feb 2016 21:54:26 +0000 Subject: [PATCH 3/5] Fix UK spelling --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index f3e83d9676..ec10bbff68 100755 --- a/build.py +++ b/build.py @@ -156,9 +156,9 @@ def CustomPythonCmakeArgs(): # # The most likely explanation for this is that both the ycm_core.so and the # python binary include copies of libpython.a (or whatever included - # objects). When python executable starts it initiliases only the globals + # objects). When python executable starts it initializes only the globals # within its copy, so when ycm_core.so's copy starts executing, it points at - # its own copy which is uninitialised. + # its own copy which is uninitialized. # # Some platforms' dynamic linkers (ld.so) are able to resolve this when # loading shared libraries at runtime[citation needed], but OSX seemingly From 3fd75e7a7a16e909c3cba4da840459da39861b4a Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Thu, 11 Feb 2016 21:55:39 +0000 Subject: [PATCH 4/5] Tidy up homebrew requirements --- ci/travis/travis_install.osx.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ci/travis/travis_install.osx.sh b/ci/travis/travis_install.osx.sh index ca00c77f08..d50ebb0993 100644 --- a/ci/travis/travis_install.osx.sh +++ b/ci/travis/travis_install.osx.sh @@ -4,9 +4,20 @@ # it twice to workaround. https://github.com/Homebrew/homebrew/issues/42553 brew update || brew update -# install node, go, ninja, pyenv and dependencies -for pkg in node.js go ninja readline autoconf pkg-config openssl pyenv; do - # install package, or upgrade it if it is already installed +# List of homebrew formulae to install in the order they appear" +REQUIREMENTS="node.js + go + ninja + readline + autoconf + pkg-config + openssl + pyenv" + + +# Install node, go, ninja, pyenv and dependencies +for pkg in $REQUIREMENTS; do + # Install package, or upgrade it if it is already installed brew install $pkg || brew outdated $pkg || brew upgrade $pkg done From 5f52f25f130809684bfa13ef82bad2e9d15f7a9d Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Thu, 11 Feb 2016 22:15:57 +0000 Subject: [PATCH 5/5] Factor the pyenv logic out of the platform-specific layer --- ci/travis/travis_install.linux.sh | 29 ---------------------- ci/travis/travis_install.osx.sh | 40 ++++--------------------------- ci/travis/travis_install.sh | 28 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 64 deletions(-) diff --git a/ci/travis/travis_install.linux.sh b/ci/travis/travis_install.linux.sh index 9106a7b7f7..d25efaee6f 100644 --- a/ci/travis/travis_install.linux.sh +++ b/ci/travis/travis_install.linux.sh @@ -1,34 +1,5 @@ # Linux-specific installation -############# -# pyenv setup -############# - -# DON'T exit if error -set +e -git clone https://github.com/yyuu/pyenv.git ~/.pyenv -git fetch --tags -git checkout v20160202 -# Exit if error -set -e - -export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$PATH" - -eval "$(pyenv init -)" - -if [ "${YCMD_PYTHON_VERSION}" == "2.6" ]; then - PYENV_VERSION="2.6.6" -elif [ "${YCMD_PYTHON_VERSION}" == "2.7" ]; then - PYENV_VERSION="2.7.6" -else - PYENV_VERSION="3.3.0" -fi - -pyenv install --skip-existing ${PYENV_VERSION} -pyenv rehash -pyenv global ${PYENV_VERSION} - # We can't use sudo, so we have to approximate the behaviour of the following: # $ sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-3.7 100 diff --git a/ci/travis/travis_install.osx.sh b/ci/travis/travis_install.osx.sh index d50ebb0993..cfb5b6784c 100644 --- a/ci/travis/travis_install.osx.sh +++ b/ci/travis/travis_install.osx.sh @@ -4,16 +4,16 @@ # it twice to workaround. https://github.com/Homebrew/homebrew/issues/42553 brew update || brew update -# List of homebrew formulae to install in the order they appear" +# List of homebrew formulae to install in the order they appear. +# We require node, go and ninja for our build and tests, and all the others are +# dependencies of pyenv. REQUIREMENTS="node.js go ninja readline autoconf pkg-config - openssl - pyenv" - + openssl" # Install node, go, ninja, pyenv and dependencies for pkg in $REQUIREMENTS; do @@ -21,37 +21,7 @@ for pkg in $REQUIREMENTS; do brew install $pkg || brew outdated $pkg || brew upgrade $pkg done -# We use homebrew's defaults for PYENV_ROOT etc. -eval "$(pyenv init -)" - -if [ "${YCMD_PYTHON_VERSION}" == "2.6" ]; then - PYENV_VERSION="2.6.6" -elif [ "${YCMD_PYTHON_VERSION}" == "2.7" ]; then - PYENV_VERSION="2.7.6" -else - PYENV_VERSION="3.3.0" -fi - # In order to work with ycmd, python *must* be built as a shared library. The # most compatible way to do this on OS X is with --enable-framework. This is # set via the PYTHON_CONFIGURE_OPTS option -PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing ${PYENV_VERSION} -pyenv rehash -pyenv global ${PYENV_VERSION} - -YCMD_VENV_DIR=${HOME}/venvs/ycmd_test - -# virtualenv is not installed by default on OS X under python2.6, and we don't -# have sudo, so we install it manually. There is no "latest" link, so we have -# to install a specific version. -VENV_VERSION=14.0.6 - -curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${VENV_VERSION}.tar.gz -tar xvfz virtualenv-${VENV_VERSION}.tar.gz -python virtualenv-${VENV_VERSION}/virtualenv.py -p python${YCMD_PYTHON_VERSION} ${YCMD_VENV_DIR} - -# virtualenv script is noisy, so don't print every command -set +v -source ${YCMD_VENV_DIR}/bin/activate -set -v - +export PYTHON_CONFIGURE_OPTS="--enable-framework" diff --git a/ci/travis/travis_install.sh b/ci/travis/travis_install.sh index 0c0d17b671..bed096f525 100644 --- a/ci/travis/travis_install.sh +++ b/ci/travis/travis_install.sh @@ -11,6 +11,34 @@ set -ev # - setup the correct python for $YCMD_PYTHON_VERSION source ci/travis/travis_install.${TRAVIS_OS_NAME}.sh +############# +# pyenv setup +############# + +# DON'T exit if error +set +e +git clone https://github.com/yyuu/pyenv.git ~/.pyenv +git fetch --tags +git checkout v20160202 +# Exit if error +set -e + +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" + +eval "$(pyenv init -)" + +if [ "${YCMD_PYTHON_VERSION}" == "2.6" ]; then + PYENV_VERSION="2.6.6" +elif [ "${YCMD_PYTHON_VERSION}" == "2.7" ]; then + PYENV_VERSION="2.7.6" +else + PYENV_VERSION="3.3.0" +fi + +pyenv install --skip-existing ${PYENV_VERSION} +pyenv rehash +pyenv global ${PYENV_VERSION} # It is quite easy to get the above series of steps wrong. Verify that the # version of python actually in the path and used is the version that was