[python3] Build interpreter (#14891)

microsoft · Dec 18, 2020 · bdb225b · bdb225b
1 parent e141791
commit bdb225b
Show file tree

Hide file tree

Showing 16 changed files with 542 additions and 282 deletions.
diff --git a/docs/maintainers/vcpkg_build_make.md b/docs/maintainers/vcpkg_build_make.md
@@ -25,8 +25,10 @@ IF the port supports the install target use vcpkg_install_make() instead of vcpk
 Specifies the Makefile as a relative path from the root of the sources passed to `vcpkg_configure_make()`
 
 ### BUILD_TARGET
-The target passed to the make build command (`./make <target>`). If not specified, the 'all' target will
-be passed.
+The target passed to the make build command (`./make <target>`). Defaults to 'all'.
+
+### INSTALL_TARGET
+The target passed to the make build command (`./make <target>`) if `ENABLE_INSTALL` is used. Defaults to 'install'.
 
 ### DISABLE_PARALLEL
 The underlying buildsystem will be instructed to not parallelize

diff --git a/ports/python3/0001-static-library.patch b/ports/python3/0001-static-library.patch
@@ -1,16 +1,16 @@
-From dffd6c572fb60f955bf3d98a87e5739163ab8f3f Mon Sep 17 00:00:00 2001
+From 4bd06273a9ed63f17966f113dc61a6051bad8b86 Mon Sep 17 00:00:00 2001
 From: Adam Johnson <AdamJohnso@gmail.com>
 Date: Thu, 28 May 2020 17:25:21 -0400
-Subject: [PATCH 1/2] static library
+Subject: [PATCH 1/6] static library
 
 builds the pythoncore as a static library instead of a DLL
 ---
- PC/pyconfig.h              |  6 ++++++
+ PC/pyconfig.h              |  7 +++++++
  PCbuild/pythoncore.vcxproj | 16 ++++++++++++++--
- 2 files changed, 20 insertions(+), 2 deletions(-)
+ 2 files changed, 21 insertions(+), 2 deletions(-)
 
 diff --git a/PC/pyconfig.h b/PC/pyconfig.h
-index 02216b5068..d359c884e2 100644
+index 02216b5068..7e0ba881a5 100644
 --- a/PC/pyconfig.h
 +++ b/PC/pyconfig.h
 @@ -252,6 +252,7 @@ typedef int pid_t;
@@ -21,7 +21,7 @@ index 02216b5068..d359c884e2 100644
  #if !defined(MS_NO_COREDLL) && !defined(Py_NO_ENABLE_SHARED)
  #       define Py_ENABLE_SHARED 1 /* standard symbol for shared library */
  #       define MS_COREDLL       /* deprecated old symbol */
-@@ -277,6 +278,11 @@ Py_NO_ENABLE_SHARED to find out.  Also support MS_NO_COREDLL for b/w compat */
+@@ -277,6 +278,12 @@ Py_NO_ENABLE_SHARED to find out.  Also support MS_NO_COREDLL for b/w compat */
  #                       endif /* _DEBUG */
  #               endif /* _MSC_VER */
  #       endif /* Py_BUILD_CORE */
@@ -30,6 +30,7 @@ index 02216b5068..d359c884e2 100644
 +#       pragma comment(lib, "version.lib")
 +#       pragma comment(lib, "shlwapi.lib")
 +#       pragma comment(lib, "ws2_32.lib")
++#       pragma comment(lib, "pathcch.lib")
  #endif /* MS_COREDLL */
 
  #if defined(MS_WIN64)

diff --git a/ports/python3/0002-static-crt.patch b/ports/python3/0002-static-crt.patch
diff --git a/ports/python3/0002-use-vcpkg-zlib.patch b/ports/python3/0002-use-vcpkg-zlib.patch
@@ -0,0 +1,39 @@
+From 196555707236f4387875749f66620fb519166963 Mon Sep 17 00:00:00 2001
+From: Adam Johnson <AdamJohnso@gmail.com>
+Date: Wed, 9 Sep 2020 15:20:36 -0400
+Subject: [PATCH 2/6] use vcpkg zlib
+
+building without zlib is not a supported configuration, per the warning
+messages.
+---
+ PCbuild/pythoncore.vcxproj | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+
+diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
+index dbe236829a..5f30a35eb3 100644
+--- a/PCbuild/pythoncore.vcxproj
++++ b/PCbuild/pythoncore.vcxproj
+@@ -507,8 +507,10 @@
+     <ClCompile Include="..\Python\thread.c" />
+     <ClCompile Include="..\Python\traceback.c" />
+   </ItemGroup>
+-  <ItemGroup Condition="$(IncludeExternals)">
++  <ItemGroup>
+     <ClCompile Include="..\Modules\zlibmodule.c" />
++  </ItemGroup>
++  <ItemGroup Condition="false">
+     <ClCompile Include="$(zlibDir)\adler32.c" />
+     <ClCompile Include="$(zlibDir)\compress.c" />
+     <ClCompile Include="$(zlibDir)\crc32.c" />
+@@ -556,7 +558,7 @@
+   <Target Name="_WarnAboutToolset" BeforeTargets="PrepareForBuild" Condition="$(PlatformToolset) != 'v140' and $(PlatformToolset) != 'v141' and $(PlatformToolset) != 'v142'">
+     <Warning Text="Toolset $(PlatformToolset) is not used for official builds. Your build may have errors or incompatibilities." />
+   </Target>
+-  <Target Name="_WarnAboutZlib" BeforeTargets="PrepareForBuild" Condition="!$(IncludeExternals)">
++  <Target Name="_WarnAboutZlib" BeforeTargets="PrepareForBuild" Condition="false">
+     <Warning Text="Not including zlib is not a supported configuration." />
+   </Target>
+
+-- 
+2.28.0.windows.1
+
diff --git a/ports/python3/0003-devendor-external-dependencies.patch b/ports/python3/0003-devendor-external-dependencies.patch
@@ -0,0 +1,117 @@
+From 6c5c3793cbc6ba2a1d0d623a2bdaa9c2125be516 Mon Sep 17 00:00:00 2001
+From: Adam Johnson <AdamJohnso@gmail.com>
+Date: Wed, 9 Sep 2020 15:24:38 -0400
+Subject: [PATCH 3/6] devendor external dependencies
+
+externally fetched libraries may cause linker errors resulting from
+duplicate symbols in downstream projects.
+---
+ PCbuild/_bz2.vcxproj     | 4 +++-
+ PCbuild/_lzma.vcxproj    | 5 ++---
+ PCbuild/_sqlite3.vcxproj | 2 +-
+ PCbuild/_ssl.vcxproj     | 2 +-
+ PCbuild/pyexpat.vcxproj  | 6 ++++--
+ 5 files changed, 11 insertions(+), 8 deletions(-)
+
+diff --git a/PCbuild/_bz2.vcxproj b/PCbuild/_bz2.vcxproj
+index 3fe95fbf83..6b12e8818e 100644
+--- a/PCbuild/_bz2.vcxproj
++++ b/PCbuild/_bz2.vcxproj
+@@ -101,6 +101,8 @@
+   </ItemDefinitionGroup>
+   <ItemGroup>
+     <ClCompile Include="..\Modules\_bz2module.c" />
++  </ItemGroup>
++  <ItemGroup Condition="false">
+     <ClCompile Include="$(bz2Dir)\blocksort.c" />
+     <ClCompile Include="$(bz2Dir)\bzlib.c" />
+     <ClCompile Include="$(bz2Dir)\compress.c" />
+@@ -109,7 +111,7 @@
+     <ClCompile Include="$(bz2Dir)\huffman.c" />
+     <ClCompile Include="$(bz2Dir)\randtable.c" />
+   </ItemGroup>
+-  <ItemGroup>
++  <ItemGroup Condition="false">
+     <ClInclude Include="$(bz2Dir)\bzlib.h" />
+     <ClInclude Include="$(bz2Dir)\bzlib_private.h" />
+   </ItemGroup>
+diff --git a/PCbuild/_lzma.vcxproj b/PCbuild/_lzma.vcxproj
+index fe076a6fc5..70cc61dd95 100644
+--- a/PCbuild/_lzma.vcxproj
++++ b/PCbuild/_lzma.vcxproj
+@@ -94,10 +94,9 @@
+   <ItemDefinitionGroup>
+     <ClCompile>
+       <AdditionalIncludeDirectories>$(lzmaDir)src/liblzma/api;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+-      <PreprocessorDefinitions>WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;LZMA_API_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
++      <PreprocessorDefinitions>WIN32;_FILE_OFFSET_BITS=64;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+     </ClCompile>
+     <Link>
+-      <AdditionalDependencies>$(OutDir)liblzma$(PyDebugExt).lib;%(AdditionalDependencies)</AdditionalDependencies>
+     </Link>
+   </ItemDefinitionGroup>
+   <ItemGroup>
+@@ -111,7 +110,7 @@
+       <Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>
+       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+     </ProjectReference>
+-    <ProjectReference Include="liblzma.vcxproj">
++    <ProjectReference Condition="false" Include="liblzma.vcxproj">
+       <Project>{12728250-16eC-4dc6-94d7-e21dd88947f8}</Project>
+       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+     </ProjectReference>
+diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj
+index 7e0062692b..6fb3279a20 100644
+--- a/PCbuild/_sqlite3.vcxproj
++++ b/PCbuild/_sqlite3.vcxproj
+@@ -127,7 +127,7 @@
+       <Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>
+       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+     </ProjectReference>
+-    <ProjectReference Include="sqlite3.vcxproj">
++    <ProjectReference Condition="false" Include="sqlite3.vcxproj">
+       <Project>{a1a295e5-463c-437f-81ca-1f32367685da}</Project>
+       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+     </ProjectReference>
+diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj
+index 4907f49b66..4dffa202b7 100644
+--- a/PCbuild/_ssl.vcxproj
++++ b/PCbuild/_ssl.vcxproj
+@@ -99,7 +99,7 @@
+   </ItemDefinitionGroup>
+   <ItemGroup>
+     <ClCompile Include="..\Modules\_ssl.c" />
+-    <ClCompile Include="$(opensslIncludeDir)\applink.c">
++    <ClCompile Condition="false" Include="$(opensslIncludeDir)\applink.c">
+       <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;$(PreprocessorDefinitions)</PreprocessorDefinitions>
+     </ClCompile>
+   </ItemGroup>
+diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj
+index b2d9f5d57d..4efb826a05 100644
+--- a/PCbuild/pyexpat.vcxproj
++++ b/PCbuild/pyexpat.vcxproj
+@@ -89,17 +89,19 @@
+   </ImportGroup>
+   <PropertyGroup Label="UserMacros" />
+   <ItemDefinitionGroup>
+-    <ClCompile>
++    <ClCompile Condition="false">
+       <AdditionalIncludeDirectories>$(PySourcePath)Modules\expat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+       <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+     </ClCompile>
+   </ItemDefinitionGroup>
+-  <ItemGroup>
++  <ItemGroup Condition="false">
+     <ClInclude Include="..\Modules\expat\xmlrole.h" />
+     <ClInclude Include="..\Modules\expat\xmltok.h" />
+   </ItemGroup>
+   <ItemGroup>
+     <ClCompile Include="..\Modules\pyexpat.c" />
++  </ItemGroup>
++  <ItemGroup Condition="false">
+     <ClCompile Include="..\Modules\expat\xmlparse.c" />
+     <ClCompile Include="..\Modules\expat\xmlrole.c" />
+     <ClCompile Include="..\Modules\expat\xmltok.c" />
+-- 
+2.28.0.windows.1
+
diff --git a/ports/python3/0004-dont-copy-vcruntime.patch b/ports/python3/0004-dont-copy-vcruntime.patch
@@ -0,0 +1,26 @@
+From 8086c67fa7ada1888a7808cbdc6fe74b62abe5b1 Mon Sep 17 00:00:00 2001
+From: Adam Johnson <AdamJohnso@gmail.com>
+Date: Wed, 9 Sep 2020 16:12:49 -0400
+Subject: [PATCH 4/6] dont copy vcruntime
+
+VCRUNTIME140.dll should not be redistributed, ever.
+---
+ PCbuild/pythoncore.vcxproj | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
+index 5f30a35eb3..3cf21ba39c 100644
+--- a/PCbuild/pythoncore.vcxproj
++++ b/PCbuild/pythoncore.vcxproj
+@@ -565,7 +565,7 @@
+   <Target Name="_CopyVCRuntime" AfterTargets="Build" Inputs="@(VCRuntimeDLL)" Outputs="$(OutDir)%(Filename)%(Extension)" DependsOnTargets="FindVCRuntime">
+     <!-- bpo-38597: When we switch to another VCRuntime DLL, include vcruntime140.dll as well -->
+     <Warning Text="A copy of vcruntime140.dll is also required" Condition="!$(VCToolsRedistVersion.StartsWith(`14.`))" />
+-    <Copy SourceFiles="%(VCRuntimeDLL.FullPath)" DestinationFolder="$(OutDir)" />
++    <Copy Condition="false" SourceFiles="%(VCRuntimeDLL.FullPath)" DestinationFolder="$(OutDir)" />
+   </Target>
+   <Target Name="_CleanVCRuntime" AfterTargets="Clean">
+     <Delete Files="@(VCRuntimeDLL->'$(OutDir)%(Filename)%(Extension)')" />
+-- 
+2.28.0.windows.1
+
diff --git a/ports/python3/0005-only-build-required-projects.patch b/ports/python3/0005-only-build-required-projects.patch
@@ -0,0 +1,60 @@
+From 6492277e39dbe7dd77d32fdc9ae4b989213ef3c6 Mon Sep 17 00:00:00 2001
+From: Adam Johnson <AdamJohnso@gmail.com>
+Date: Wed, 9 Sep 2020 20:15:58 -0400
+Subject: [PATCH 5/6] only build required projects
+
+strips out tests and unsupported externals (eg tkinter).
+---
+ PCbuild/_ssl.vcxproj |  2 +-
+ PCbuild/pcbuild.proj | 14 +++++++-------
+ 2 files changed, 8 insertions(+), 8 deletions(-)
+
+diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj
+index 4dffa202b7..e661cb6fb6 100644
+--- a/PCbuild/_ssl.vcxproj
++++ b/PCbuild/_ssl.vcxproj
+@@ -111,7 +111,7 @@
+       <Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>
+       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+     </ProjectReference>
+-    <ProjectReference Include="_socket.vcxproj">
++    <ProjectReference Condition="false" Include="_socket.vcxproj">
+       <Project>{86937f53-c189-40ef-8ce8-8759d8e7d480}</Project>
+       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+     </ProjectReference>
+diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj
+index 4d416c589e..ede9868a8f 100644
+--- a/PCbuild/pcbuild.proj
++++ b/PCbuild/pcbuild.proj
+@@ -45,21 +45,21 @@
+       <BuildInParallel>false</BuildInParallel>
+     </Projects>
+     <!-- python3.dll -->
+-    <Projects Include="python3dll.vcxproj" />
++    <Projects Include="python3dll.vcxproj" Condition="false" />
+     <!-- py[w].exe -->
+-    <Projects Include="pylauncher.vcxproj;pywlauncher.vcxproj" />
++    <Projects Include="pylauncher.vcxproj;pywlauncher.vcxproj" Condition="false" />
+     <!-- pyshellext.dll -->
+-    <Projects Include="pyshellext.vcxproj" />
++    <Projects Include="pyshellext.vcxproj" Condition="false" />
+     <!-- Extension modules -->
+     <ExtensionModules Include="_asyncio;_zoneinfo;_decimal;_elementtree;_msi;_multiprocessing;_overlapped;pyexpat;_queue;select;unicodedata;winsound;_uuid" />
+     <ExtensionModules Include="_ctypes" Condition="$(IncludeCTypes)" />
+     <!-- Extension modules that require external sources -->
+     <ExternalModules Include="_bz2;_lzma;_sqlite3" />
+     <!-- venv launchers -->
+-    <Projects Include="venvlauncher.vcxproj;venvwlauncher.vcxproj" />
+-    <!-- _ssl will build _socket as well, which may cause conflicts in parallel builds -->
+-    <ExtensionModules Include="_socket" Condition="!$(IncludeSSL) or !$(IncludeExternals)" />
+-    <ExternalModules Include="_ssl;_hashlib" Condition="$(IncludeSSL)" />
++    <Projects Include="venvlauncher.vcxproj;venvwlauncher.vcxproj" Condition="false" />
++    <!-- _ssl will NOT build _socket as well -->
++    <ExtensionModules Include="_socket" Condition="true" />
++    <ExternalModules Include="_ssl;_hashlib" Condition="true" />
+     <ExternalModules Include="_tkinter" Condition="$(IncludeTkinter) and $(Platform) != 'ARM' and $(Platform) != 'ARM64'" />
+     <ExtensionModules Include="@(ExternalModules->'%(Identity)')" Condition="$(IncludeExternals)" />
+     <Projects Include="@(ExtensionModules->'%(Identity).vcxproj')" Condition="$(IncludeExtensions)" />
+-- 
+2.28.0.windows.1
+
diff --git a/ports/python3/0006-fix-duplicate-symbols.patch b/ports/python3/0006-fix-duplicate-symbols.patch
@@ -0,0 +1,39 @@
+From 62e082c1eba79d46ba8ab4b7a556a57a2253d555 Mon Sep 17 00:00:00 2001
+From: Adam Johnson <AdamJohnso@gmail.com>
+Date: Fri, 11 Sep 2020 12:35:36 -0400
+Subject: [PATCH 6/6] fix duplicate symbols
+
+---
+ Modules/_winapi.c    | 2 +-
+ Modules/overlapped.c | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/Modules/_winapi.c b/Modules/_winapi.c
+index e1672c4785..27148094d2 100644
+--- a/Modules/_winapi.c
++++ b/Modules/_winapi.c
+@@ -304,7 +304,7 @@ static PyMemberDef overlapped_members[] = {
+     {NULL}
+ };
+
+-PyTypeObject OverlappedType = {
++static PyTypeObject OverlappedType = {
+     PyVarObject_HEAD_INIT(NULL, 0)
+     /* tp_name           */ "_winapi.Overlapped",
+     /* tp_basicsize      */ sizeof(OverlappedObject),
+diff --git a/Modules/overlapped.c b/Modules/overlapped.c
+index cd7869fa8a..319ea0a4b7 100644
+--- a/Modules/overlapped.c
++++ b/Modules/overlapped.c
+@@ -1756,7 +1756,7 @@ static PyGetSetDef Overlapped_getsets[] = {
+     {NULL},
+ };
+
+-PyTypeObject OverlappedType = {
++static PyTypeObject OverlappedType = {
+     PyVarObject_HEAD_INIT(NULL, 0)
+     /* tp_name           */ "_overlapped.Overlapped",
+     /* tp_basicsize      */ sizeof(OverlappedObject),
+-- 
+2.28.0.windows.1
+
diff --git a/ports/python3/CONTROL b/ports/python3/CONTROL
diff --git a/ports/python3/openssl.props.in b/ports/python3/openssl.props.in
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemDefinitionGroup>
+    <Link>
+      <AdditionalDependencies>Crypt32.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies Condition="'$(Configuration)|$(IncludeExternals)'=='Release|true'">
+        ${CRYPTO_RELEASE};${SSL_RELEASE};%(AdditionalDependencies)
+      </AdditionalDependencies>
+      <AdditionalDependencies Condition="'$(Configuration)|$(IncludeExternals)'=='Debug|true'">
+        ${CRYPTO_DEBUG};${SSL_DEBUG};%(AdditionalDependencies)
+      </AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+</Project>