Merge pull request #188 from jcfr/fix-py27-vs2015-build-and-add-patch…

…-mgmt

Fix py27 vs2015 build and add patch mgmt

CMakeLists.txt

@@ -176,7 +176,7 @@ get_filename_component(SRC_DIR "${SRC_DIR}" ABSOLUTE)
 get_filename_component(_parent_dir ${CMAKE_CURRENT_BINARY_DIR} PATH)
 string(REGEX REPLACE "rc[1-9]$" "" _py_version_patch_no_rc ${PY_VERSION_PATCH})
 set(_py_version_no_rc "${PY_VERSION_MAJOR}.${PY_VERSION_MINOR}.${_py_version_patch_no_rc}")
-set(_download_link "http://www.python.org/ftp/python/${_py_version_no_rc}/Python-${PY_VERSION}.tgz")
+set(_download_link "https://www.python.org/ftp/python/${_py_version_no_rc}/Python-${PY_VERSION}.tgz")
 # Variable below represent the set of supported python version.
 set(_download_2.7.3_md5 "2cf641732ac23b18d139be077bd906cd")
 set(_download_2.7.4_md5 "592603cfaf4490a980e93ecb92bde44a")
@@ -235,6 +235,12 @@ If you already downloaded the source, you could try to re-configure this project
 endif()
 message(STATUS "SRC_DIR: ${SRC_DIR}")
 
+# Apply patches
+option(PYTHON_APPLY_PATCHES "Apply patches" ON)
+if(PYTHON_APPLY_PATCHES)
+  include(cmake/PythonApplyPatches.cmake)
+endif()
+
 # Extract version from python source (Copied from FindPythonLibs.cmake)
 file(STRINGS "${SRC_DIR}/Include/patchlevel.h" python_version_str
     REGEX "^#define[ \t]+PY_VERSION[ \t]+\"[^\"]+\"")

CTestConfig.cmake

@@ -5,3 +5,4 @@ set(CTEST_DROP_METHOD "http")
 set(CTEST_DROP_SITE "open.cdash.org")
 set(CTEST_DROP_LOCATION "/submit.php?project=CPython")
 set(CTEST_DROP_SITE_CDASH TRUE)
+set(CTEST_TEST_TIMEOUT "60")

README.rst

@@ -61,6 +61,12 @@ options on the commandline with `-DOPTION=VALUE`, or use the "ccmake" gui.
   PYTHON_VERSION=major.minor.patch (defaults to 3.5.2)
     The version of Python to build.
 
+  PYTHON_APPLY_PATCHES=ON|OFF (defaults to ON)
+    Apply patches required to build CPython based on the system and compiler
+    found when configuring the project. Note that when cross-compiling, patches
+    coresponding to the target system are applied.
+    Patches can be found in "patches" directory.
+
   CMAKE_BUILD_TYPE=Debug|Release
     Build with debugging symbols or with optimisations.
 

appveyor.yml

@@ -1,4 +1,6 @@
 
+image: Visual Studio 2015
+
 branches:
  only:
   - master
@@ -17,6 +19,10 @@ environment:
   - PY_VERSION: 3.5.1
     GENERATOR: Visual Studio 12 2013 Win64
   - PY_VERSION: 2.7.13
+  - PY_VERSION: 2.7.13
+    GENERATOR: Visual Studio 12 2013 Win64
+  - PY_VERSION: 2.7.13
+    GENERATOR: Visual Studio 14 2015 Win64
   - PY_VERSION: 2.7.12
   - PY_VERSION: 2.7.11
   - PY_VERSION: 2.7.10

cmake/PythonApplyPatches.cmake

@@ -0,0 +1,67 @@
+
+set(_x86 "(x86)")  # Indirection required to avoid error related to CMP0053
+find_program(PATCH_EXECUTABLE
+  NAME patch
+  PATHS "$ENV{ProgramFiles}/Git/usr/bin"
+        "$ENV{ProgramFiles${_x86}}/Git/usr/bin"
+        "$ENV{ProgramFiles}/GnuWin32/bin"
+        "$ENV{ProgramFiles${_x86}}/GnuWin32/bin"
+        "$ENV{ProgramFiles}/Git/bin"
+        "$ENV{ProgramFiles${_x86}}/Git/bin"
+  )
+if(NOT PATCH_EXECUTABLE)
+  message(FATAL_ERROR "Could NOT find patch (missing: PATCH_EXECUTABLE)")
+endif()
+
+set(patches_dir "${Python_SOURCE_DIR}/patches")
+
+function(_apply_patches _subdir)
+  if(NOT EXISTS ${patches_dir}/${_subdir})
+    message(STATUS "Skipping patches: Directory '${patches_dir}/${_subdir}' does not exist")
+    return()
+  endif()
+  file(GLOB _patches RELATIVE ${patches_dir} "${patches_dir}/${_subdir}/*.patch")
+  if(NOT _patches)
+    return()
+  endif()
+  message(STATUS "")
+  list(SORT _patches)
+  foreach(patch IN LISTS _patches)
+    set(msg "Applying '${patch}'")
+    message(STATUS "${msg}")
+    set(applied ${PROJECT_BINARY_DIR}/CMakeFiles/patches/${patch}.applied)
+    if(EXISTS ${applied})
+      message(STATUS "${msg} - skipping (already applied)")
+      continue()
+    endif()
+    execute_process(
+      COMMAND ${PATCH_EXECUTABLE} --quiet -p1 -i ${patches_dir}/${patch}
+      WORKING_DIRECTORY ${SRC_DIR}
+      RESULT_VARIABLE result
+      ERROR_VARIABLE error
+      ERROR_STRIP_TRAILING_WHITESPACE
+      OUTPUT_VARIABLE output
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      )
+    if(result EQUAL 0)
+      message(STATUS "${msg} - done")
+      get_filename_component(_dir ${applied} DIRECTORY)
+      execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${_dir})
+      execute_process(COMMAND ${CMAKE_COMMAND} -E touch ${applied})
+    else()
+      message(STATUS "${msg} - failed")
+      message(FATAL_ERROR "${output}\n${error}")
+    endif()
+  endforeach()
+  message(STATUS "")
+endfunction()
+
+# Apply patches
+_apply_patches("${PY_VERSION}")
+_apply_patches("${PY_VERSION}/${CMAKE_SYSTEM_NAME}")
+_apply_patches("${PY_VERSION}/${CMAKE_SYSTEM_NAME}-${CMAKE_C_COMPILER_ID}")
+set(_version ${CMAKE_C_COMPILER_VERSION})
+if(MSVC)
+  set(_version ${MSVC_VERSION})
+endif()
+_apply_patches("${PY_VERSION}/${CMAKE_SYSTEM_NAME}-${CMAKE_C_COMPILER_ID}/${_version}")

cmake/libpython/CMakeLists.txt

@@ -84,6 +84,13 @@ set(OBJECT2_SOURCES
     ${SRC_DIR}/Objects/intobject.c
     ${SRC_DIR}/Objects/stringobject.c
 )
+if(MSVC)
+    if(EXISTS ${SRC_DIR}/PC/invalid_parameter_handler.c)
+        list(APPEND OBJECT2_SOURCES
+            ${SRC_DIR}/PC/invalid_parameter_handler.c
+        )
+    endif()
+endif()
 
 set(OBJECT3_SOURCES
     ${SRC_DIR}/Objects/accu.c

patches/2.7.13/Windows-MSVC/1600/0001-VS2010-Support-Backport-Fix-13210.-Port-the-Windows-.patch

@@ -0,0 +1,227 @@
+From 4988ea4f4ce6af0b567e6a2dc2c23aaf7cc5f12d Mon Sep 17 00:00:00 2001
+From: Jean-Christophe Fillion-Robin <jchris.fillionr@kitware.com>
+Date: Mon, 31 Jul 2017 10:59:47 -0400
+Subject: [PATCH 1/3] VS2010 Support: Backport "Fix #13210. Port the Windows
+ build from VS2008 to VS2010."
+
+This commit is a partial backport of python/cpython@401f9f3. It was
+originally designed to work with python-cmake-buildsystem.
+
+The following modules have NOT been backported:
+
+* Tools/msi
+* Tools/buildbot
+* PCBuild
+---
+ Lib/distutils/command/build_ext.py |  2 +-
+ Lib/distutils/msvc9compiler.py     | 11 ++++++-----
+ PC/dl_nt.c                         | 11 ++++++++++-
+ PC/msvcrtmodule.c                  | 15 ++++++++++++++-
+ PC/pyconfig.h                      |  9 +++++++--
+ Python/dynload_win.c               |  8 ++++++++
+ 6 files changed, 46 insertions(+), 10 deletions(-)
+
+diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py
+index 2c68be3..f1d184b 100644
+--- a/Lib/distutils/command/build_ext.py
++++ b/Lib/distutils/command/build_ext.py
+@@ -193,7 +193,7 @@ class build_ext (Command):
+             # Append the source distribution include and library directories,
+             # this allows distutils on windows to work in the source tree
+             self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC'))
+-            if MSVC_VERSION == 9:
++            if MSVC_VERSION >= 9:
+                 # Use the .lib files for the correct architecture
+                 if self.plat_name == 'win32':
+                     suffix = ''
+diff --git a/Lib/distutils/msvc9compiler.py b/Lib/distutils/msvc9compiler.py
+index 33d3e51..f6de11c 100644
+--- a/Lib/distutils/msvc9compiler.py
++++ b/Lib/distutils/msvc9compiler.py
+@@ -662,11 +662,12 @@ class MSVCCompiler(CCompiler) :
+             if mfinfo is not None:
+                 mffilename, mfid = mfinfo
+                 out_arg = '-outputresource:%s;%s' % (output_filename, mfid)
+-                try:
+-                    self.spawn(['mt.exe', '-nologo', '-manifest',
+-                                mffilename, out_arg])
+-                except DistutilsExecError, msg:
+-                    raise LinkError(msg)
++                if self.__version < 10:
++                    try:
++                        self.spawn(['mt.exe', '-nologo', '-manifest',
++                                    temp_manifest, out_arg])
++                    except PackagingExecError as msg:
++                        raise LinkError(msg)
+         else:
+             log.debug("skipping %s (up-to-date)", output_filename)
+
+diff --git a/PC/dl_nt.c b/PC/dl_nt.c
+index ef1ce09..5ff07fd 100644
+--- a/PC/dl_nt.c
++++ b/PC/dl_nt.c
+@@ -18,7 +18,8 @@ char dllVersionBuffer[16] = ""; // a private buffer
+ HMODULE PyWin_DLLhModule = NULL;
+ const char *PyWin_DLLVersionString = dllVersionBuffer;
+
+-// Windows "Activation Context" work:
++#if HAVE_SXS
++// Windows "Activation Context" work.
+ // Our .pyd extension modules are generally built without a manifest (ie,
+ // those included with Python and those built with a default distutils.
+ // This requires we perform some "activation context" magic when loading our
+@@ -29,6 +30,8 @@ const char *PyWin_DLLVersionString = dllVersionBuffer;
+ // As an added complication, this magic only works on XP or later - we simply
+ // use the existence (or not) of the relevant function pointers from kernel32.
+ // See bug 4566 (http://python.org/sf/4566) for more details.
++// In Visual Studio 2010, side by side assemblies are no longer used by
++// default.
+
+ typedef BOOL (WINAPI * PFN_GETCURRENTACTCTX)(HANDLE *);
+ typedef BOOL (WINAPI * PFN_ACTIVATEACTCTX)(HANDLE, ULONG_PTR *);
+@@ -76,6 +79,8 @@ void _Py_DeactivateActCtx(ULONG_PTR cookie)
+             OutputDebugString("Python failed to de-activate the activation context\n");
+ }
+
++#endif /* HAVE_SXS */
++
+ BOOL    WINAPI  DllMain (HANDLE hInst,
+                                                 ULONG ul_reason_for_call,
+                                                 LPVOID lpReserved)
+@@ -87,17 +92,21 @@ BOOL    WINAPI  DllMain (HANDLE hInst,
+             // 1000 is a magic number I picked out of the air.  Could do with a #define, I spose...
+             LoadString(hInst, 1000, dllVersionBuffer, sizeof(dllVersionBuffer));
+
++#if HAVE_SXS
+             // and capture our activation context for use when loading extensions.
+             _LoadActCtxPointers();
+             if (pfnGetCurrentActCtx && pfnAddRefActCtx)
+                 if ((*pfnGetCurrentActCtx)(&PyWin_DLLhActivationContext))
+                     if (!(*pfnAddRefActCtx)(PyWin_DLLhActivationContext))
+                         OutputDebugString("Python failed to load the default activation context\n");
++#endif
+             break;
+
+         case DLL_PROCESS_DETACH:
++#if HAVE_SXS
+             if (pfnReleaseActCtx)
+                 (*pfnReleaseActCtx)(PyWin_DLLhActivationContext);
++#endif
+             break;
+     }
+     return TRUE;
+diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c
+index 44c82e4..68928dd 100644
+--- a/PC/msvcrtmodule.c
++++ b/PC/msvcrtmodule.c
+@@ -25,6 +25,8 @@
+ #ifdef _MSC_VER
+ #if _MSC_VER >= 1500 && _MSC_VER < 1600
+ #include <crtassem.h>
++#elif _MSC_VER >= 1600
++#include <crtversion.h>
+ #endif
+ #endif
+
+@@ -398,7 +400,7 @@ PyMODINIT_FUNC
+ initmsvcrt(void)
+ {
+     int st;
+-    PyObject *d;
++    PyObject *d, *version;
+     PyObject *m = Py_InitModule("msvcrt", msvcrt_functions);
+     if (m == NULL)
+         return;
+@@ -412,6 +414,7 @@ initmsvcrt(void)
+     insertint(d, "LK_UNLCK", _LK_UNLCK);
+
+     /* constants for the crt versions */
++    (void)st;
+ #ifdef _VC_ASSEMBLY_PUBLICKEYTOKEN
+     st = PyModule_AddStringConstant(m, "VC_ASSEMBLY_PUBLICKEYTOKEN",
+                                     _VC_ASSEMBLY_PUBLICKEYTOKEN);
+@@ -427,4 +430,14 @@ initmsvcrt(void)
+                                     __LIBRARIES_ASSEMBLY_NAME_PREFIX);
+     if (st < 0)return;
+ #endif
++
++/* constants for the 2010 crt versions */
++#if defined(_VC_CRT_MAJOR_VERSION) && defined (_VC_CRT_MINOR_VERSION) && defined(_VC_CRT_BUILD_VERSION) && defined(_VC_CRT_RBUILD_VERSION)
++    version = PyUnicode_FromFormat("%d.%d.%d.%d", _VC_CRT_MAJOR_VERSION,
++                                                  _VC_CRT_MINOR_VERSION,
++                                                  _VC_CRT_BUILD_VERSION,
++                                                  _VC_CRT_RBUILD_VERSION);
++    st = PyModule_AddObject(m, "CRT_ASSEMBLY_VERSION", version);
++    if (st < 0) return NULL;
++#endif
+ }
+diff --git a/PC/pyconfig.h b/PC/pyconfig.h
+index 5d1c90a..b60af1e 100644
+--- a/PC/pyconfig.h
++++ b/PC/pyconfig.h
+@@ -231,14 +231,19 @@ typedef int pid_t;
+ #define hypot _hypot
+ #endif
+
+-#endif /* _MSC_VER */
++/* Side by Side assemblies supported in VS 2005 and VS 2008 but not 2010*/
++#if _MSC_VER >= 1400 && _MSC_VER < 1600
++#define HAVE_SXS 1
++#endif
+
+ /* define some ANSI types that are not defined in earlier Win headers */
+-#if defined(_MSC_VER) && _MSC_VER >= 1200
++#if _MSC_VER >= 1200
+ /* This file only exists in VC 6.0 or higher */
+ #include <basetsd.h>
+ #endif
+
++#endif /* _MSC_VER */
++
+ /* ------------------------------------------------------------------------*/
+ /* The Borland compiler defines __BORLANDC__ */
+ /* XXX These defines are likely incomplete, but should be easy to fix. */
+diff --git a/Python/dynload_win.c b/Python/dynload_win.c
+index 4e5555e..8626642 100644
+--- a/Python/dynload_win.c
++++ b/Python/dynload_win.c
+@@ -12,8 +12,10 @@
+ #include <windows.h>
+
+ // "activation context" magic - see dl_nt.c...
++#if HAVE_SXS
+ extern ULONG_PTR _Py_ActivateActCtx();
+ void _Py_DeactivateActCtx(ULONG_PTR cookie);
++#endif
+
+ const struct filedescr _PyImport_DynLoadFiletab[] = {
+ #ifdef _DEBUG
+@@ -176,7 +178,9 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname,
+         char pathbuf[260];
+         LPTSTR dummy;
+         unsigned int old_mode;
++#if HAVE_SXS
+         ULONG_PTR cookie = 0;
++#endif
+         /* We use LoadLibraryEx so Windows looks for dependent DLLs
+             in directory of pathname first.  However, Windows95
+             can sometimes not work correctly unless the absolute
+@@ -190,11 +194,15 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *fqname, const char *shortname,
+                             sizeof(pathbuf),
+                             pathbuf,
+                             &dummy)) {
++#if HAVE_SXS
+             ULONG_PTR cookie = _Py_ActivateActCtx();
++#endif
+             /* XXX This call doesn't exist in Windows CE */
+             hDLL = LoadLibraryEx(pathname, NULL,
+                                  LOAD_WITH_ALTERED_SEARCH_PATH);
++#if HAVE_SXS
+             _Py_DeactivateActCtx(cookie);
++#endif
+         }
+
+         /* restore old error mode settings */
+-- 
+2.5.0
+