Merge pull request #365 from micbou/windows-more-py3

[READY] Fix Python 2 support and add Python 3 support on Windows

appveyor.yml

@@ -6,18 +6,23 @@ branches:
 environment:
   USE_CLANG_COMPLETER: true
   matrix:
-  - msvc: 11
-    arch: 32
+  # We only test MSVC 11 and 12 with Python 3.5 on 64 bits.
   - msvc: 11
     arch: 64
-  - msvc: 12
-    arch: 32
+    python: 35
   - msvc: 12
     arch: 64
+    python: 35
   - msvc: 14
     arch: 32
+    python: 35
+  # We only test Python 2.7 with MSVC 14 on 64 bits.
+  - msvc: 14
+    arch: 64
+    python: 27
   - msvc: 14
     arch: 64
+    python: 35
 install:
   - ci\appveyor\appveyor_install.bat
 build_script:

build.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python
 
-from __future__ import unicode_literals
+# Passing an environment variable containing unicode literals to a subprocess
+# on Windows and Python2 raises a TypeError. Since there is no unicode
+# string in this script, we don't import unicode_literals to avoid the issue.
 from __future__ import print_function
 from __future__ import division
 from __future__ import absolute_import
+
 import os
 import subprocess
 import os.path as p
@@ -281,7 +284,10 @@ def RunYcmdTests( build_dir ):
   new_env = os.environ.copy()
 
   if OnWindows():
-    new_env[ 'PATH' ] = DIR_OF_THIS_SCRIPT
+    # We prepend the folder of the ycm_core_tests executable to the PATH
+    # instead of overwriting it so that the executable is able to find the
+    # python35.dll library.
+    new_env[ 'PATH' ] = DIR_OF_THIS_SCRIPT + ';' + new_env[ 'PATH' ]
   else:
     new_env[ 'LD_LIBRARY_PATH' ] = DIR_OF_THIS_SCRIPT
 

ci/appveyor/appveyor_install.bat

@@ -19,17 +19,26 @@ if exist racerd_target (
 ::
 
 if %arch% == 32 (
-  set python=C:\Python27
+  set python_path=C:\Python%python%
 ) else (
-  set python=C:\Python27-x64
+  set python_path=C:\Python%python%-x64
 )
 
-set PATH=%python%;%python%\Scripts;%PATH%
+set PATH=%python_path%;%python_path%\Scripts;%PATH%
 python --version
 :: Manually setting PYTHONHOME for python 2.7.11 fix the following error when
 :: running core tests: "ImportError: No module named site"
 :: TODO: check if this is still needed when python 2.7.12 is released.
-set PYTHONHOME=%python%
+if %python% == 27 (
+  set PYTHONHOME=%python_path%
+)
+
+:: When using Python 3 on AppVeyor, CMake will always pick the 64 bits
+:: libraries. We specifically tell CMake the right path to the libraries
+:: according to the architecture.
+if %python% == 35 (
+  set EXTRA_CMAKE_ARGS="-DPYTHON_LIBRARY=%python_path%\libs\python%python%.lib"
+)
 
 appveyor DownloadFile https://bootstrap.pypa.io/get-pip.py
 python get-pip.py

run_tests.py

@@ -163,9 +163,11 @@ def FixupCompleters( parsed_args ):
 
 def BuildYcmdLibs( args ):
   if not args.skip_build:
-    extra_cmake_args = [ '-DUSE_DEV_FLAGS=ON' ]
+    if 'EXTRA_CMAKE_ARGS' in os.environ:
+      os.environ[ 'EXTRA_CMAKE_ARGS' ] += ' -DUSE_DEV_FLAGS=ON'
+    else:
+      os.environ[ 'EXTRA_CMAKE_ARGS' ] = '-DUSE_DEV_FLAGS=ON'
 
-    os.environ[ 'EXTRA_CMAKE_ARGS' ] = ' '.join(extra_cmake_args)
     os.environ[ 'YCM_TESTRUN' ] = '1'
 
     build_cmd = [

ycmd/completers/rust/rust_completer.py

@@ -24,7 +24,7 @@
 from future import standard_library
 standard_library.install_aliases()
 
-from ycmd.utils import ToBytes, ProcessIsRunning
+from ycmd.utils import ToBytes, SetEnviron, ProcessIsRunning
 from ycmd.completers.completer import Completer
 from ycmd import responses, utils, hmac_utils
 
@@ -261,7 +261,7 @@ def _StartServer( self ):
 
       # Enable logging of crashes
       env = os.environ.copy()
-      env[ 'RUST_BACKTRACE' ] = '1'
+      SetEnviron( env, 'RUST_BACKTRACE', '1' )
 
       if self._rust_source_path:
         args.extend( [ '--rust-src-path', self._rust_source_path ] )

ycmd/completers/typescript/typescript_completer.py

@@ -104,7 +104,7 @@ def __init__( self, user_options ):
     # source code it seems like this is the way:
     # https://github.com/Microsoft/TypeScript/blob/8a93b489454fdcbdf544edef05f73a913449be1d/src/server/server.ts#L136
     self._environ = os.environ.copy()
-    self._environ[ 'TSS_LOG' ] = tsserver_log
+    utils.SetEnviron( self._environ, 'TSS_LOG', tsserver_log )
 
     # Each request sent to tsserver must have a sequence id.
     # Responses contain the id sent in the corresponding request.

ycmd/tests/cs/cs_handlers_test.py

@@ -22,9 +22,10 @@
 from future import standard_library
 standard_library.install_aliases()
 from builtins import *  # noqa
+from future.utils import PY2
 
 from ..handlers_test import Handlers_test
-from ycmd.utils import OnTravis
+from ycmd.utils import OnTravis, OnWindows
 import time
 from contextlib import contextmanager
 
@@ -33,7 +34,10 @@
 # Omnisharp instances between individual test cases. Non caching (false) is
 # much faster, but test cases are not totally isolated from each other.
 # For test case isolation, set to true.
-INSTANCE_PER_TEST = False
+# Reusing Omnisharp instances this way on Windows and Python 3 will randomly
+# raise the error "OSError: [WinError 6] The handle is invalid" in tests so
+# we set it to true in this case.
+INSTANCE_PER_TEST = True if OnWindows() and not PY2 else False
 
 
 class Cs_Handlers_test( Handlers_test ):

ycmd/utils.py

@@ -22,6 +22,7 @@
 from future import standard_library
 standard_library.install_aliases()
 from builtins import *  # noqa
+from future.utils import PY2, native
 
 import tempfile
 import os
@@ -31,8 +32,6 @@
 import stat
 import subprocess
 
-from future.utils import PY2, iteritems, native
-
 # Creation flag to disable creating a console window on Windows. See
 # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx
 CREATE_NO_WINDOW = 0x08000000
@@ -270,14 +269,6 @@ def ForceSemanticCompletion( request_data ):
 
 # A wrapper for subprocess.Popen that fixes quirks on Windows.
 def SafePopen( args, **kwargs ):
-  def ToNativeStringDict( dict_obj ):
-    new_dict = dict()
-    for k, v in iteritems( dict_obj ):
-      if PY2:
-        new_dict[ native( ToBytes( k ) ) ] = native( ToBytes( v ) )
-      else:
-        new_dict[ ToUnicode( k ) ] = ToUnicode( v )
-    return new_dict
 
   if OnWindows():
     # We need this to start the server otherwise bad things happen.
@@ -291,17 +282,22 @@ def ToNativeStringDict( dict_obj ):
     # http://bugs.python.org/issue1759845.
     # Since paths are likely to contains such characters, we convert them to
     # short ones to obtain paths with only ascii characters.
-    args = ConvertArgsToShortPath( args )
-
-    if 'env' in kwargs:
-      # Popen requires that on Windows, the environment has only native strings
-      # on py2 and py3.
-      kwargs[ 'env' ] = ToNativeStringDict( kwargs[ 'env' ] )
+    if PY2:
+      args = ConvertArgsToShortPath( args )
 
   kwargs.pop( 'stdin_windows', None )
   return subprocess.Popen( args, **kwargs )
 
 
+# We need to convert environment variables to native strings on Windows and
+# Python 2 to prevent a TypeError when passing them to a subprocess.
+def SetEnviron( environ, variable, value ):
+  if OnWindows() and PY2:
+    environ[ native( ToBytes( variable ) ) ] = native( ToBytes( value ) )
+  else:
+    environ[ variable ] = value
+
+
 # Convert paths in arguments command to short path ones
 def ConvertArgsToShortPath( args ):
   def ConvertIfPath( arg ):
@@ -317,7 +313,6 @@ def ConvertIfPath( arg ):
 # Get the Windows short path name.
 # Based on http://stackoverflow.com/a/23598461/200291
 def GetShortPathName( path ):
-  path = native( ToBytes( path ) )
   from ctypes import windll, wintypes, create_unicode_buffer
 
   # Set the GetShortPathNameW prototype