Build wheels for all platforms (win, linux, mac)

Add wheels building for all platforms

.gitmodules

@@ -1,6 +1,6 @@
-[submodule "terryfy"]
-	path = terryfy
-	url = https://github.com/MacPython/terryfy.git
 [submodule "gensim"]
 	path = gensim
 	url = https://github.com/piskvorky/gensim.git
+[submodule "multibuild"]
+	path = multibuild
+	url = https://github.com/matthew-brett/multibuild.git

.travis.yml

@@ -1,46 +1,113 @@
-language:
-- objective-c
-os: osx
-osx_image: xcode8.1
 env:
-  global:
-  - REPO_DIR=gensim
-  #- BUILD_COMMIT=latest-tag  # comment out to build version in submodule
-  matrix:
-  # 2.7, 3.5 and 3.6
-  # can't download - VERSION=2.6.8
-  - VERSION=2.7.12
-  - VERSION=3.5.2
-  - VERSION=3.6.0
+    global:
+        - REPO_DIR=gensim
+        - BUILD_COMMIT=1980109764f67513ef9740d39f2763aba6519ba2
+        - UNICODE_WIDTH=32
+        - BUILD_DEPENDS="numpy==1.11.3 scipy==0.18.1 six smart_open"
+        - TEST_DEPENDS="pytest pytest-rerunfailures cython pyemd testfixtures scikit-learn Morfessor==2.0.2a4"
+
+        - WHEELHOUSE_UPLOADER_USERNAME="Lev.Konstantinovskiy"
+        # generated by 'gem install travis; travis encrypt -r MacPython/gensim-wheels WHEELHOUSE_UPLOADER_SECRET=<the api key>'
+        - secure: "L99GHXSsHwj2P+lfJPTeLCIDMpRomj8yhhH2TuFJ9wP36rkVtewpy6ExVey/jae1dwMaRyoMbdXZwbMTNs0cUuvDvniBwQM8PRNxy3VdOxLNVGFbCtj6+/iOl6rxok6TPkhwlrhzQy2hq9vyZjeNsFZCkGiTFm7Idgd1dWUPDeBowR3X6ucIFK7fR5qDgNfFVVBUPKMmGOyvLy1nUB9FV7mdAK8VQCrIKk5BMVcovAsnYBgUF2eAMemgeFUtHMF9KbpXPRgjPyMZfdKhjIvqz5qbOGDhbrTn5sD1oPWx0IkA5gVazlng8EeyasiRg87br2TxknHLHHP8TEiVnj0wTJ7yYQK6zAwz6RM1ZFOQ1JI0YY4kgw1TRZpTqelehlvQYtHiwDxJXQfb/Nt2dxC0cJPjs5NMMrQKWyP5yHRP/RX4CKSRMF/48pF2kROg1gkc+i68mKKrPpckq0QXEzxTJwjuLLFq4Nos4JAQhs+3rxm4yFExpgdkfbtutRv3/I/HNW9il/aQltGgih65KCYkkJx6uFfENQnXnvLMoR16vJ94RghCKh4cnC6w4h4HRjVZgm34FZ00fe9Z30H/5FKb1/VoGm0edz56o5zOvdKEpzxoGS3wI5IrVGCh348heUFRzjxLNmWtkpOJgm3b9FZgb6h72cflq5NwbgeQtF4ld+U="
+
+
+language: python
+# The travis Python version is unrelated to the version we build and test
+# with.  This is set with the MB_PYTHON_VERSION variable.
+python: 3.5
+sudo: required
+dist: trusty
+services: docker
+
+matrix:
+  exclude:
+    # Exclude the default Python 3.5 build
+    - python: 3.5
+  include:
+
+    # Linux 
+    - os: linux
+      env:
+        - MB_PYTHON_VERSION=2.7
+        - PLAT=x86_64
+
+    - os: linux
+      env:
+        - MB_PYTHON_VERSION=2.7
+        - PLAT=i686
+
+    - os: linux
+      env:
+        - MB_PYTHON_VERSION=2.7
+        - PLAT=x86_64
+        - UNICODE_WIDTH=16
+
+    - os: linux
+      env:
+        - MB_PYTHON_VERSION=2.7
+        - PLAT=i686
+        - UNICODE_WIDTH=16
+
+    - os: linux
+      env:
+        - MB_PYTHON_VERSION=3.5
+        - PLAT=x86_64
+
+    - os: linux
+      env: 
+        - MB_PYTHON_VERSION=3.5
+        - PLAT=i686
+
+    - os: linux
+      env: 
+        - MB_PYTHON_VERSION=3.6
+        - PLAT=x86_64
+
+    - os: linux
+      env: 
+        - MB_PYTHON_VERSION=3.6
+        - PLAT=i686
+
+    # MacOS
+    - os: osx
+      language: generic
+      env:
+        - MB_PYTHON_VERSION=2.7
+
+    - os: osx
+      language: generic
+      env:
+        - MB_PYTHON_VERSION=3.5
+
+    - os: osx
+      language: generic
+      env:
+        - MB_PYTHON_VERSION=3.6
+
+
+before_install:
+    - source multibuild/common_utils.sh
+    - source multibuild/travis_steps.sh
+    - before_install
+
 install:
-- source terryfy/travis_tools.sh
-- get_python_environment macpython $VERSION venv
-- pip install numpy scipy sklearn
-- pip install -U cython wheel six
-- if [ -n "$BUILD_COMMIT" ]; then
-      checkout_commit $REPO_DIR $BUILD_COMMIT;
-  fi
-- cd $REPO_DIR
-- python setup.py bdist_wheel
-- rename_wheels dist
-- pip install dist/*.whl
-- cd ..
+    - clean_code $REPO_DIR $BUILD_COMMIT
+    - build_wheel $REPO_DIR $PLAT
+
 script:
-# - if [ "${VERSION:0:3}" != "3.3" ]; then pip install -U numpy; fi
-- pip install nose pyemd testfixtures unittest2 morfessor==2.0.2a4 sklearn
-
-- mkdir tmp_for_test
-- cd tmp_for_test
-#  make sure we test the wheel
-- nosetests --exe -v gensim
-# setup.py doesn't test wheel, just the local files: - python ../setup.py test
-- cd ..
-before_deploy: cd $REPO_DIR/dist
-deploy:
-  provider: cloudfiles
-  username: travis-worker
-  api_key:
-    secure: "lZW7U67gVsceiI+Ofi2n+X3BRyou8D/ul5BuJK0OBtt4aiccKzrUAb8vraHy6ED0l1t/j3GWfrg8jaP9G9BRGIfoDQROJ5w+BZ/pTL0p+OnJrqZkNNiG82qvEOSmlVrE+afl2YMeLKDou+uX1wuXp6xYaN9W/lJJW0qskS6D/aiPY7KwagFWv+QsUF07SpomMBiZ/EE4FsMJBhg9q7sUg/tG4mbiTgZY2jGx+R2u5ktq3zPr6MIzIqz+7vswMR/BTg2WJTDvxQTMpWGYihi3vOxX9ty0vIUKti0zHIcj6B8RYgiDoAC6P2OUtk2J1qHiyvAooi9Vm+WkPnxQqF+8EOXkbajvsuniwPwenmy6xYh7fmNxs4huxFb3H8FYC6qI1256cMXd9CssFCroeLuQoUa7sn1X+cmb2IjrE1mOyboIF6iQ5M31xcQBUzIuauONnx4xz58ASovYDfHc1RkhatBtaW2nKhjiti6uvs8kVgok/dzoGOExRblbN1IjpjmCvhDQh6WQryMupoUmjJWjIsgs1MfaMaT7jW8AM1as5beHoEJiB445KJTBIb61BHxyfXcneuUJrRmlM7F8Tw56Gpilwl7hE/ufd3teabJDdxV0J18r4Rq/HI8N5vh4YFvV3h+p4drEnZYPHV3narMj8nGnud65qXAFfp0e25cLY3c="
-  region: ORD
-  container: wheels
-  skip_cleanup: true
+    - install_run $PLAT
+
+after_success:
+    - pip install wheelhouse-uploader
+    - echo ${TRAVIS_BUILD_DIR}/wheelhouse/
+    - ls -laht ${TRAVIS_BUILD_DIR}/wheelhouse/
+    # This uploads the wheels to a Rackspace container owned by the
+    # scikit-learn team, available at http://wheels.scipy.org.  See above
+    # for information on using this account or choosing another.
+    - python -m wheelhouse_uploader upload --local-folder ${TRAVIS_BUILD_DIR}/wheelhouse/ --no-ssl-check gensim-wheels
+
+notifications:
+  email:
+    - ivan+gensimwheels@rare-technologies.com
+  on_success: always
+  on_failure: always

appveyor.yml

@@ -0,0 +1,95 @@
+environment:
+  global:
+    BUILD_COMMIT: 1980109764f67513ef9740d39f2763aba6519ba2
+    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+    # /E:ON and /V:ON options are not enabled in the batch script intepreter
+    # See: http://stackoverflow.com/a/13751649/163740
+    CMD_IN_ENV: "cmd /E:ON /V:ON /C ..\\appveyor\\run_with_env.cmd"
+    WHEELHOUSE_UPLOADER_USERNAME: "Lev.Konstantinovskiy"
+    WHEELHOUSE_UPLOADER_SECRET:
+      secure: qXqY3dFmLOqvxa3Om2gQi/BjotTOK+EP2IPLolBNo0c61yDtNWxbmE4wH3up72Be
+
+  matrix:
+    - PYTHON: "C:\\Python27"
+      PYTHON_VERSION: "2.7.12"
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7.12"
+      PYTHON_ARCH: "64"
+
+    - PYTHON: "C:\\Python35"
+      PYTHON_VERSION: "3.5.2"
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python35-x64"
+      PYTHON_VERSION: "3.5.2"
+      PYTHON_ARCH: "64"
+
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6.0"
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6.0"
+      PYTHON_ARCH: "64"
+
+
+matrix:
+  fast_finish: true
+
+
+install:
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "python -m pip install -U pip setuptools wheel wheelhouse_uploader"
+
+  # Check that we have the expected version and architecture for Python
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+  - "git submodule update --init --recursive"
+  - "cd gensim"
+  - "git checkout %BUILD_COMMIT%"
+  - "git clean -fxd"
+  - "git reset --hard"
+
+
+  # Install the build and runtime dependencies of the project.
+  - "%CMD_IN_ENV% pip install --timeout=60 --trusted-host 8003f898b26602186ccc-7b1aad3fab3cc330e1fd9d109892382a.ssl.cf2.rackcdn.com/ --find-links https://8003f898b26602186ccc-7b1aad3fab3cc330e1fd9d109892382a.ssl.cf2.rackcdn.com/ numpy==1.11.3 scipy==0.18.1 smart_open pytest pytest-rerunfailures cython pyemd testfixtures scikit-learn Morfessor==2.0.2a4"
+  - "%CMD_IN_ENV% python setup.py bdist_wheel bdist_wininst"
+  - ps: "ls dist"
+
+  # Install the generated wheel package to test it
+  - "pip install --pre --no-index --force-reinstall --find-links dist/ gensim"
+
+
+build: false
+
+
+test_script:
+  # Change to a non-source folder to make sure we run the tests on the
+  # installed library.
+  - "cd .."
+  - "mkdir empty_folder"
+  - "cd empty_folder"
+  - "pytest -rfxEXs --durations=20 --showlocals --rerun 3 --pyargs gensim"
+  - "cd ../gensim/dist"
+  - "ls"
+
+
+artifacts:
+  # Archive the generated wheel package in the ci.appveyor.com build report.
+  - path: "gensim\\dist\\*"
+
+
+on_success:
+  # Upload the generated wheel package to Rackspace
+  # On Windows, Apache Libcloud cannot find a standard CA cert bundle so we
+  # disable the ssl checks.
+  - "python -m wheelhouse_uploader upload --no-ssl-check --local-folder=.  gensim-wheels"
+
+
+notifications:
+  - provider: Email
+    to: ivan+gensimwheels@rare-technologies.com
+    on_build_success: true
+    on_build_failure: true

appveyor/run_with_env.cmd

@@ -0,0 +1,88 @@
+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
+::
+:: To build extensions for 64 bit Python 2, we need to configure environment
+:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
+::
+:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific
+:: environment configurations.
+::
+:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
+:: cmd interpreter, at least for (SDK v7.0)
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
+:: http://stackoverflow.com/a/13751649/163740
+::
+:: Author: Olivier Grisel
+:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+::
+:: Notes about batch files for Python people:
+::
+:: Quotes in values are literally part of the values:
+::      SET FOO="bar"
+:: FOO is now five characters long: " b a r "
+:: If you don't want quotes, don't include them on the right-hand side.
+::
+:: The CALL lines at the end of this file look redundant, but if you move them
+:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y
+:: case, I don't know why.
+@ECHO OFF
+
+SET COMMAND_TO_RUN=%*
+SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
+SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf
+
+:: Extract the major and minor versions, and allow for the minor version to be
+:: more than 9.  This requires the version number to have two dots in it.
+SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1%
+IF "%PYTHON_VERSION:~3,1%" == "." (
+    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1%
+) ELSE (
+    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2%
+)
+
+:: Based on the Python version, determine what SDK version to use, and whether
+:: to set the SDK for 64-bit.
+IF %MAJOR_PYTHON_VERSION% == 2 (
+    SET WINDOWS_SDK_VERSION="v7.0"
+    SET SET_SDK_64=Y
+) ELSE (
+    IF %MAJOR_PYTHON_VERSION% == 3 (
+        SET WINDOWS_SDK_VERSION="v7.1"
+        IF %MINOR_PYTHON_VERSION% LEQ 4 (
+            SET SET_SDK_64=Y
+        ) ELSE (
+            SET SET_SDK_64=N
+            IF EXIST "%WIN_WDK%" (
+                :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/
+                REN "%WIN_WDK%" 0wdf
+            )
+        )
+    ) ELSE (
+        ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
+        EXIT 1
+    )
+)
+
+IF %PYTHON_ARCH% == 64 (
+    IF %SET_SDK_64% == Y (
+        ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
+        SET DISTUTILS_USE_SDK=1
+        SET MSSdk=1
+        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
+        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
+        ECHO Executing: %COMMAND_TO_RUN%
+        call %COMMAND_TO_RUN% || EXIT 1
+    ) ELSE (
+        ECHO Using default MSVC build environment for 64 bit architecture
+        ECHO Executing: %COMMAND_TO_RUN%
+        call %COMMAND_TO_RUN% || EXIT 1
+    )
+) ELSE (
+    ECHO Using default MSVC build environment for 32 bit architecture
+    ECHO Executing: %COMMAND_TO_RUN%
+    call %COMMAND_TO_RUN% || EXIT 1
+)

config.sh

@@ -0,0 +1,24 @@
+# Define custom utilities
+# Test for OSX with [ -n "$IS_OSX" ]
+
+function pre_build {
+    # Any stuff that you need to do before you start building the wheels
+    # Runs in the root directory of this repository.
+    :
+}
+
+function run_tests {
+    # Runs tests on installed distribution from an empty directory
+    python --version
+    if [ "$PLAT" -ne "i686" ]; then
+        pip install "tensorflow<=1.3.0" "keras>=2.0.4"  # additional deps, available only for x64
+    fi
+
+    if [[ "$IS_OSX" -eq 0 ]]; then
+        pip install annoy  # annoy works only with linux
+    fi
+
+    python -c 'import gensim; print(gensim.__file__, gensim.models.word2vec.FAST_VERSION)'
+    pip freeze
+    pytest -rfxEXs --durations=20 --showlocals --rerun 3 --pyargs gensim -vv
+}