gh-113258: Write frozen modules to the build tree on Windows (GH-113303)

This ensures the source directory is not modified at build time, and different builds (e.g. different versions or GIL vs no-GIL) do not have conflicts.

Misc/NEWS.d/next/Build/2023-12-23-09-35-48.gh-issue-113258.GlsAyH.rst

@@ -0,0 +1,2 @@
+Changed the Windows build to write out generated frozen modules into the
+build tree instead of the source tree.

PCbuild/_freeze_module.vcxproj

@@ -266,117 +266,117 @@
     <None Include="..\Lib\importlib\_bootstrap.py">
       <ModName>importlib._bootstrap</ModName>
       <IntFile>$(IntDir)importlib._bootstrap.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\importlib._bootstrap.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap.h</OutFile>
     </None>
     <None Include="..\Lib\importlib\_bootstrap_external.py">
       <ModName>importlib._bootstrap_external</ModName>
       <IntFile>$(IntDir)importlib._bootstrap_external.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap_external.h</OutFile>
     </None>
     <None Include="..\Lib\zipimport.py">
       <ModName>zipimport</ModName>
       <IntFile>$(IntDir)zipimport.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\zipimport.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\zipimport.h</OutFile>
     </None>
     <None Include="..\Lib\abc.py">
       <ModName>abc</ModName>
       <IntFile>$(IntDir)abc.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\abc.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\abc.h</OutFile>
     </None>
     <None Include="..\Lib\codecs.py">
       <ModName>codecs</ModName>
       <IntFile>$(IntDir)codecs.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\codecs.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h</OutFile>
     </None>
     <None Include="..\Lib\io.py">
       <ModName>io</ModName>
       <IntFile>$(IntDir)io.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\io.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\io.h</OutFile>
     </None>
     <None Include="..\Lib\_collections_abc.py">
       <ModName>_collections_abc</ModName>
       <IntFile>$(IntDir)_collections_abc.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\_collections_abc.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\_collections_abc.h</OutFile>
     </None>
     <None Include="..\Lib\_sitebuiltins.py">
       <ModName>_sitebuiltins</ModName>
       <IntFile>$(IntDir)_sitebuiltins.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\_sitebuiltins.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\_sitebuiltins.h</OutFile>
     </None>
     <None Include="..\Lib\genericpath.py">
       <ModName>genericpath</ModName>
       <IntFile>$(IntDir)genericpath.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\genericpath.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\genericpath.h</OutFile>
     </None>
     <None Include="..\Lib\ntpath.py">
       <ModName>ntpath</ModName>
       <IntFile>$(IntDir)ntpath.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\ntpath.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\ntpath.h</OutFile>
     </None>
     <None Include="..\Lib\posixpath.py">
       <ModName>posixpath</ModName>
       <IntFile>$(IntDir)posixpath.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\posixpath.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\posixpath.h</OutFile>
     </None>
     <None Include="..\Lib\os.py">
       <ModName>os</ModName>
       <IntFile>$(IntDir)os.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\os.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\os.h</OutFile>
     </None>
     <None Include="..\Lib\site.py">
       <ModName>site</ModName>
       <IntFile>$(IntDir)site.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\site.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\site.h</OutFile>
     </None>
     <None Include="..\Lib\stat.py">
       <ModName>stat</ModName>
       <IntFile>$(IntDir)stat.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\stat.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\stat.h</OutFile>
     </None>
     <None Include="..\Lib\importlib\util.py">
       <ModName>importlib.util</ModName>
       <IntFile>$(IntDir)importlib.util.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\importlib.util.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.util.h</OutFile>
     </None>
     <None Include="..\Lib\importlib\machinery.py">
       <ModName>importlib.machinery</ModName>
       <IntFile>$(IntDir)importlib.machinery.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\importlib.machinery.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.machinery.h</OutFile>
     </None>
     <None Include="..\Lib\runpy.py">
       <ModName>runpy</ModName>
       <IntFile>$(IntDir)runpy.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\runpy.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\runpy.h</OutFile>
     </None>
     <None Include="..\Lib\__hello__.py">
       <ModName>__hello__</ModName>
       <IntFile>$(IntDir)__hello__.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\__hello__.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__hello__.h</OutFile>
     </None>
     <None Include="..\Lib\__phello__\__init__.py">
       <ModName>__phello__</ModName>
       <IntFile>$(IntDir)__phello__.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\__phello__.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.h</OutFile>
     </None>
     <None Include="..\Lib\__phello__\ham\__init__.py">
       <ModName>__phello__.ham</ModName>
       <IntFile>$(IntDir)__phello__.ham.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\__phello__.ham.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.h</OutFile>
     </None>
     <None Include="..\Lib\__phello__\ham\eggs.py">
       <ModName>__phello__.ham.eggs</ModName>
       <IntFile>$(IntDir)__phello__.ham.eggs.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.eggs.h</OutFile>
     </None>
     <None Include="..\Lib\__phello__\spam.py">
       <ModName>__phello__.spam</ModName>
       <IntFile>$(IntDir)__phello__.spam.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\__phello__.spam.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.spam.h</OutFile>
     </None>
     <None Include="..\Tools\freeze\flag.py">
       <ModName>frozen_only</ModName>
       <IntFile>$(IntDir)frozen_only.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\frozen_only.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\frozen_only.h</OutFile>
     </None>
     <!-- END frozen modules -->
   </ItemGroup>
@@ -385,34 +385,34 @@
     <GetPath Include="..\Modules\getpath.py">
       <ModName>getpath</ModName>
       <IntFile>$(IntDir)getpath.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_modules\getpath.h</OutFile>
+      <OutFile>$(GeneratedFrozenModulesDir)Python\frozen_modules\getpath.h</OutFile>
     </GetPath>
   </ItemGroup>
   <ItemGroup>
     <!-- BEGIN freeze mappings -->
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap.h" FrozenId="importlib._bootstrap" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap_external.h" FrozenId="importlib._bootstrap_external" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\zipimport.h" FrozenId="zipimport" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\abc.h" FrozenId="abc" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\codecs.h" FrozenId="codecs" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\io.h" FrozenId="io" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_collections_abc.h" FrozenId="_collections_abc" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_sitebuiltins.h" FrozenId="_sitebuiltins" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\genericpath.h" FrozenId="genericpath" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\ntpath.h" FrozenId="ntpath" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\posixpath.h" FrozenId="posixpath" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\os.h" FrozenId="os" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\site.h" FrozenId="site" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\stat.h" FrozenId="stat" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.util.h" FrozenId="importlib.util" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.machinery.h" FrozenId="importlib.machinery" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\runpy.h" FrozenId="runpy" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__hello__.h" FrozenId="__hello__" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.h" FrozenId="__phello__" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.h" FrozenId="__phello__.ham" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.eggs.h" FrozenId="__phello__.ham.eggs" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.spam.h" FrozenId="__phello__.spam" />
-    <FrozenModule Include="$(PySourcePath)\Python\frozen_modules\frozen_only.h" FrozenId="frozen_only" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap.h" FrozenId="importlib._bootstrap" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib._bootstrap_external.h" FrozenId="importlib._bootstrap_external" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\zipimport.h" FrozenId="zipimport" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\abc.h" FrozenId="abc" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h" FrozenId="codecs" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\io.h" FrozenId="io" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\_collections_abc.h" FrozenId="_collections_abc" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\_sitebuiltins.h" FrozenId="_sitebuiltins" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\genericpath.h" FrozenId="genericpath" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\ntpath.h" FrozenId="ntpath" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\posixpath.h" FrozenId="posixpath" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\os.h" FrozenId="os" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\site.h" FrozenId="site" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\stat.h" FrozenId="stat" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.util.h" FrozenId="importlib.util" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\importlib.machinery.h" FrozenId="importlib.machinery" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\runpy.h" FrozenId="runpy" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__hello__.h" FrozenId="__hello__" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.h" FrozenId="__phello__" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.h" FrozenId="__phello__.ham" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.ham.eggs.h" FrozenId="__phello__.ham.eggs" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\__phello__.spam.h" FrozenId="__phello__.spam" />
+    <FrozenModule Include="$(GeneratedFrozenModulesDir)Python\frozen_modules\frozen_only.h" FrozenId="frozen_only" />
     <!-- END freeze mappings -->
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
@@ -484,7 +484,7 @@ $(IntDir)\deepfreeze_mappings.txt
       Overwrite="true"
       Lines="@(FrozenModule->'%(FullPath):%(FrozenId)')" />
     <!-- BEGIN deepfreeze rule -->
-    <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" -f "$(IntDir)\deepfreeze_mappings.txt" -o "$(PySourcePath)Python\deepfreeze\deepfreeze.c"' />
+    <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" -f "$(IntDir)\deepfreeze_mappings.txt" -o "$(GeneratedFrozenModulesDir)deepfreeze.c"' />
     <!-- END deepfreeze rule -->
   </Target>
   <Target Name="_CleanFrozen" BeforeTargets="CoreClean" Condition="$(Configuration) != 'PGUpdate'">
@@ -493,7 +493,7 @@ $(IntDir)\deepfreeze_mappings.txt
       <Clean Include="%(None.OutFile)" />
       <Clean Include="%(GetPath.IntFile)" />
       <Clean Include="%(GetPath.OutFile)" />
-      <Clean Include="$(PySourcePath)Python\deepfreeze\deepfreeze.c" />
+      <Clean Include="$(GeneratedFrozenModulesDir)deepfreeze.c" />
     </ItemGroup>
   </Target>
 </Project>

PCbuild/pyproject.props

@@ -12,6 +12,7 @@
     <IntDir>$(IntDir.Replace(`\\`, `\`))</IntDir>
     <!-- pyconfig.h is updated by pythoncore.vcxproj, so it's always in pythoncore's IntDir -->
     <GeneratedPyConfigDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\</GeneratedPyConfigDir>
+    <GeneratedFrozenModulesDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen_$(Configuration)\</GeneratedFrozenModulesDir>
     <TargetName Condition="'$(TargetName)' == ''">$(ProjectName)</TargetName>
     <TargetName>$(TargetName)$(PyDebugExt)</TargetName>
     <GenerateManifest>false</GenerateManifest>

PCbuild/pythoncore.vcxproj

@@ -111,6 +111,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\getpath.c">
+      <AdditionalIncludeDirectories>$(GeneratedFrozenModulesDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>
         PREFIX=NULL;
         EXEC_PREFIX=NULL;
@@ -120,7 +121,6 @@
         PLATLIBDIR="DLLs";
         %(PreprocessorDefinitions)
       </PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(PySourcePath);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
   </ItemGroup>
   <ItemGroup>
@@ -562,7 +562,9 @@
     <ClCompile Include="..\Python\flowgraph.c" />
     <ClCompile Include="..\Python\formatter_unicode.c" />
     <ClCompile Include="..\Python\frame.c" />
-    <ClCompile Include="..\Python\frozen.c" />
+    <ClCompile Include="..\Python\frozen.c">
+      <AdditionalIncludeDirectories>$(GeneratedFrozenModulesDir)Python;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
     <ClCompile Include="..\Python\future.c" />
     <ClCompile Include="..\Python\getargs.c" />
     <ClCompile Include="..\Python\getcompiler.c" />
@@ -617,7 +619,7 @@
   </ItemGroup>
   <ItemGroup>
     <!-- BEGIN deepfreeze -->
-    <ClCompile Include="..\Python\deepfreeze\deepfreeze.c" />
+    <ClCompile Include="$(GeneratedFrozenModulesDir)deepfreeze.c" />
     <!-- END deepfreeze -->
   </ItemGroup>
   <ItemGroup Condition="$(IncludeExternals)">

Tools/build/freeze_modules.py

@@ -658,7 +658,7 @@ def regen_pcbuild(modules):
     filterlines = []
     corelines = []
     deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}'
-    deepfreezerules = [f'    <Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" -f "{deepfreezemappingsfile}" -o "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\' />']
+    deepfreezerules = [f'    <Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" -f "{deepfreezemappingsfile}" -o "$(GeneratedFrozenModulesDir)deepfreeze.c"\' />']
     deepfreezemappings = []
     for src in _iter_sources(modules):
         pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
@@ -667,15 +667,15 @@ def regen_pcbuild(modules):
         projlines.append(f'    <None Include="..\\{pyfile}">')
         projlines.append(f'      <ModName>{src.frozenid}</ModName>')
         projlines.append(f'      <IntFile>$(IntDir){intfile}</IntFile>')
-        projlines.append(f'      <OutFile>$(PySourcePath){header}</OutFile>')
+        projlines.append(f'      <OutFile>$(GeneratedFrozenModulesDir){header}</OutFile>')
         projlines.append(f'    </None>')
 
         filterlines.append(f'    <None Include="..\\{pyfile}">')
         filterlines.append('      <Filter>Python Files</Filter>')
         filterlines.append('    </None>')
-        deepfreezemappings.append(f'    <FrozenModule Include="$(PySourcePath)\\{header}" FrozenId="{src.frozenid}" />\n')
+        deepfreezemappings.append(f'    <FrozenModule Include="$(GeneratedFrozenModulesDir){header}" FrozenId="{src.frozenid}" />\n')
 
-    corelines.append(f'    <ClCompile Include="..\\Python\\deepfreeze\\deepfreeze.c" />')
+    corelines.append(f'    <ClCompile Include="$(GeneratedFrozenModulesDir)deepfreeze.c" />')
 
     print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}')
     with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):