bpo-45019: Add a tool to generate list of modules to include for froz…

…en modules (gh-27980)

Frozen modules must be added to several files in order to work properly. Before this change this had to be done manually. Here we add a tool to generate the relevant lines in those files instead. This helps us avoid mistakes and omissions.

https://bugs.python.org/issue45019

.gitattributes

@@ -46,8 +46,7 @@ Modules/clinic/*.h          linguist-generated=true
 Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true
 Python/clinic/*.h           linguist-generated=true
-Python/importlib.h          linguist-generated=true
-Python/importlib_external.h linguist-generated=true
+Python/frozen_modules/*.h   linguist-generated=true
 Include/internal/pycore_ast.h   linguist-generated=true
 Python/Python-ast.c         linguist-generated=true
 Include/opcode.h            linguist-generated=true

.github/workflows/build.yml

@@ -71,6 +71,7 @@ jobs:
           make regen-stdlib-module-names
       - name: Check for changes
         run: |
+          git add -u
           changes=$(git status --porcelain)
           # Check for changes in regenerated files
           if ! test -z "$changes"

.gitignore

@@ -68,7 +68,7 @@ Modules/Setup.config
 Modules/Setup.local
 Modules/config.c
 Modules/ld_so_aix
-Programs/_freeze_importlib
+Programs/_freeze_module
 Programs/_testembed
 PC/python_nt*.h
 PC/pythonnt_rc*.h

Doc/c-api/init.rst

@@ -110,7 +110,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
    Suppress error messages when calculating the module search path in
    :c:func:`Py_GetPath`.
 
-   Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs.
+   Private flag used by ``_freeze_module`` and ``frozenmain`` programs.
 
 .. c:var:: int Py_HashRandomizationFlag
 

Makefile.pre.in

@@ -574,8 +574,8 @@ coverage-lcov:
 	@echo "lcov report at $(COVERAGE_REPORT)/index.html"
 	@echo
 
-# Force regeneration of parser and importlib
-coverage-report: regen-token regen-importlib
+# Force regeneration of parser and frozen modules
+coverage-report: regen-token regen-frozen
 	@ # build with coverage info
 	$(MAKE) coverage
 	@ # run tests, ignore failures
@@ -734,45 +734,60 @@ Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
 	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
 ############################################################################
-# Importlib
+# frozen modules (including importlib)
 
-Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile
+Programs/_freeze_module.o: Programs/_freeze_module.c Makefile
 
-Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN)
-	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
+Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN)
+	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
+Tools/scripts/freeze_modules.py: Programs/_freeze_module
+
+.PHONY: regen-frozen
+regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES)
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py
+	@echo "The Makefile was updated, you may need to re-run make."
+
+# BEGIN: freezing modules
+
+Python/frozen_modules/importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py
+	$(srcdir)/Programs/_freeze_module importlib._bootstrap \
+		$(srcdir)/Lib/importlib/_bootstrap.py \
+		$(srcdir)/Python/frozen_modules/importlib__bootstrap.h
+
+Python/frozen_modules/importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py
+	$(srcdir)/Programs/_freeze_module importlib._bootstrap_external \
+		$(srcdir)/Lib/importlib/_bootstrap_external.py \
+		$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
+
+Python/frozen_modules/zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py
+	$(srcdir)/Programs/_freeze_module zipimport \
+		$(srcdir)/Lib/zipimport.py \
+		$(srcdir)/Python/frozen_modules/zipimport.h
+
+Python/frozen_modules/hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py
+	$(srcdir)/Programs/_freeze_module hello \
+		$(srcdir)/Tools/freeze/flag.py \
+		$(srcdir)/Python/frozen_modules/hello.h
+
+# END: freezing modules
+
+# We keep this renamed target around for folks with muscle memory.
 .PHONY: regen-importlib
-regen-importlib: Programs/_freeze_importlib
-	# Regenerate Python/importlib_external.h
-	# from Lib/importlib/_bootstrap_external.py using _freeze_importlib
-	./Programs/_freeze_importlib importlib._bootstrap_external \
-	    $(srcdir)/Lib/importlib/_bootstrap_external.py \
-	    $(srcdir)/Python/importlib_external.h.new
-	$(UPDATE_FILE) $(srcdir)/Python/importlib_external.h $(srcdir)/Python/importlib_external.h.new
-	# Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py
-	# using _freeze_importlib
-	./Programs/_freeze_importlib importlib._bootstrap \
-	    $(srcdir)/Lib/importlib/_bootstrap.py \
-	    $(srcdir)/Python/importlib.h.new
-	$(UPDATE_FILE) $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib.h.new
-	# Regenerate Python/importlib_zipimport.h from Lib/zipimport.py
-	# using _freeze_importlib
-	./Programs/_freeze_importlib zipimport \
-	    $(srcdir)/Lib/zipimport.py \
-	    $(srcdir)/Python/importlib_zipimport.h.new
-	$(UPDATE_FILE) $(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/importlib_zipimport.h.new
+regen-importlib: regen-frozen
 
+############################################################################
+# ABI
 
 regen-limited-abi: all
 	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.txt
 
-
 ############################################################################
 # Regenerate all generated files
 
 regen-all: regen-opcode regen-opcode-targets regen-typeslots \
-	regen-token regen-ast regen-keyword regen-importlib clinic \
-	regen-pegen-metaparser regen-pegen regen-frozen regen-test-frozenmain
+	regen-token regen-ast regen-keyword regen-frozen clinic \
+	regen-pegen-metaparser regen-pegen regen-test-frozenmain
 	@echo
 	@echo "Note: make regen-stdlib-module-names and autoconf should be run manually"
 
@@ -884,15 +899,6 @@ regen-opcode:
 		$(srcdir)/Include/opcode.h.new
 	$(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
 
-.PHONY: regen-frozen
-regen-frozen: Programs/_freeze_importlib
-	# Regenerate code for frozen module "__hello__".
-	./Programs/_freeze_importlib hello \
-		$(srcdir)/Tools/freeze/flag.py \
-		$(srcdir)/Python/frozen_hello.h.new
-	$(UPDATE_FILE) $(srcdir)/Python/frozen_hello.h \
-		$(srcdir)/Python/frozen_hello.h.new
-
 .PHONY: regen-token
 regen-token:
 	# Regenerate Doc/library/token-list.inc from Grammar/Tokens
@@ -995,8 +1001,15 @@ regen-opcode-targets:
 Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \
 		$(srcdir)/Python/condvar.h
 
-Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h \
-		$(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/frozen_hello.h
+# FROZEN_FILES is auto-generated by Tools/scripts/freeze_modules.py.
+FROZEN_FILES = \
+		$(srcdir)/Python/frozen_modules/importlib__bootstrap.h \
+		$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h \
+		$(srcdir)/Python/frozen_modules/zipimport.h \
+		$(srcdir)/Python/frozen_modules/hello.h
+# End FROZEN_FILES
+
+Python/frozen.o: $(FROZEN_FILES)
 
 # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to
 # follow our naming conventions. dtrace(1) uses the output filename to generate
@@ -1918,7 +1931,7 @@ clean-retain-profile: pycremoval
 	find build -name '*.py[co]' -exec rm -f {} ';' || true
 	-rm -f pybuilddir.txt
 	-rm -f Lib/lib2to3/*Grammar*.pickle
-	-rm -f Programs/_testembed Programs/_freeze_importlib
+	-rm -f Programs/_testembed Programs/_freeze_module
 	-find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
 	-rm -f Include/pydtrace_probes.h
 	-rm -f profile-gen-stamp

Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst

@@ -0,0 +1,3 @@
+Generate lines in relevant files for frozen modules.  Up until now each of
+the files had to be edited manually.  This change makes it easier to add to
+and modify the frozen modules.

PCbuild/_freeze_importlib.vcxproj → PCbuild/_freeze_module.vcxproj

@@ -69,7 +69,7 @@
   <PropertyGroup Label="Globals">
     <ProjectGuid>{19C0C13F-47CA-4432-AFF3-799A296A4DDC}</ProjectGuid>
     <Keyword>Win32Proj</Keyword>
-    <RootNamespace>_freeze_importlib</RootNamespace>
+    <RootNamespace>_freeze_module</RootNamespace>
     <SupportPGO>false</SupportPGO>
   </PropertyGroup>
   <Import Project="python.props" />
@@ -95,7 +95,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClCompile Include="..\Programs\_freeze_importlib.c" />
+    <ClCompile Include="..\Programs\_freeze_module.c" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="pythoncore.vcxproj">
@@ -108,31 +108,33 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <!-- BEGIN frozen modules -->
     <None Include="..\Lib\importlib\_bootstrap.py">
       <ModName>importlib._bootstrap</ModName>
-      <IntFile>$(IntDir)importlib.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\importlib.h</OutFile>
+      <IntFile>$(IntDir)importlib__bootstrap.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap.h</OutFile>
     </None>
     <None Include="..\Lib\importlib\_bootstrap_external.py">
       <ModName>importlib._bootstrap_external</ModName>
-      <IntFile>$(IntDir)importlib_external.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\importlib_external.h</OutFile>
+      <IntFile>$(IntDir)importlib__bootstrap_external.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap_external.h</OutFile>
     </None>
     <None Include="..\Lib\zipimport.py">
       <ModName>zipimport</ModName>
-      <IntFile>$(IntDir)importlib_zipimport.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\importlib_zipimport.h</OutFile>
+      <IntFile>$(IntDir)zipimport.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\zipimport.h</OutFile>
     </None>
     <None Include="..\Tools\freeze\flag.py">
       <ModName>hello</ModName>
-      <IntFile>$(IntDir)frozen_hello.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_hello.h</OutFile>
+      <IntFile>$(IntDir)ello.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\hello.h</OutFile>
     </None>
+    <!-- END frozen modules -->
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-  <Target Name="_RebuildImportLib">
+  <Target Name="_RebuildFrozen">
     <Exec Command='"$(TargetPath)" "%(None.ModName)" "%(None.FullPath)" "%(None.IntFile)"' />
 
     <Copy SourceFiles="%(None.IntFile)"
@@ -143,15 +145,18 @@
 
     <Message Text="Updated files: @(_Updated->'%(Filename)%(Extension)',', ')"
              Condition="'@(_Updated)' != ''" Importance="high" />
-    <Warning Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
+    <Warning Text="Frozen modules (e.g. importlib) were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
              Condition="'@(_Updated)' != '' and $(Configuration) == 'Debug'" />
-    <Error Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
+    <Error Text="Frozen (e.g. importlib) files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
            Condition="'@(_Updated)' != '' and $(Configuration) == 'Release'" />
   </Target>
+  <Target Name="RebuildFrozen" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'"
+          DependsOnTargets="_RebuildFrozen">
+  </Target>
   <Target Name="RebuildImportLib" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'"
-          DependsOnTargets="_RebuildImportLib">
+          DependsOnTargets="_RebuildFrozen">
   </Target>
-  <Target Name="_CleanImportLib" BeforeTargets="CoreClean">
+  <Target Name="_CleanFrozen" BeforeTargets="CoreClean">
     <ItemGroup>
       <Clean Include="%(None.IntFile)" />
     </ItemGroup>

PCbuild/_freeze_importlib.vcxproj.filters → PCbuild/_freeze_module.vcxproj.filters

@@ -10,19 +10,24 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="..\Programs\_freeze_importlib.c">
+    <ClCompile Include="..\Programs\_freeze_module.c">
       <Filter>Source Files</Filter>
     </ClCompile>
   </ItemGroup>
   <ItemGroup>
+    <!-- BEGIN frozen modules -->
     <None Include="..\Lib\importlib\_bootstrap.py">
-      <Filter>Source Files</Filter>
+      <Filter>Python Files</Filter>
+    </None>
+    <None Include="..\Lib\importlib\_bootstrap_external.py">
+      <Filter>Python Files</Filter>
     </None>
     <None Include="..\Lib\zipimport.py">
       <Filter>Python Files</Filter>
     </None>
-    <None Include="..\Lib\importlib\_bootstrap_external.py">
+    <None Include="..\Tools\freeze\flag.py">
       <Filter>Python Files</Filter>
     </None>
+    <!-- END frozen modules -->
   </ItemGroup>
-</Project>
+</Project>

PCbuild/pcbuild.proj

@@ -72,8 +72,8 @@
       <BuildInParallel>false</BuildInParallel>
     </Projects>
 
-    <!-- _freeze_importlib -->
-    <Projects2 Condition="$(Platform) != 'ARM' and $(Platform) != 'ARM64'" Include="_freeze_importlib.vcxproj" />
+    <!-- _freeze_module -->
+    <Projects2 Condition="$(Platform) != 'ARM' and $(Platform) != 'ARM64'" Include="_freeze_module.vcxproj" />
     <!-- python[w].exe -->
     <Projects2 Include="python.vcxproj;pythonw.vcxproj" />
     <Projects2 Include="python_uwp.vcxproj;pythonw_uwp.vcxproj" Condition="$(IncludeUwp)" />

PCbuild/pcbuild.sln

@@ -75,7 +75,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher.
 		{7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446}
 	EndProjectSection
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_importlib", "_freeze_importlib.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_module", "_freeze_module.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}"
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}"
 EndProject

PCbuild/readme.txt

@@ -115,9 +115,10 @@ _testembed
 
 These are miscellaneous sub-projects that don't really fit the other
 categories:
-_freeze_importlib
-    _freeze_importlib.exe, used to regenerate Python\importlib.h after
-    changes have been made to Lib\importlib\_bootstrap.py
+_freeze_module
+    _freeze_module.exe, used to regenerate frozen modules in Python
+    after changes have been made to the corresponding source files
+    (e.g. Lib\importlib\_bootstrap.py).
 pyshellext
     pyshellext.dll, the shell extension deployed with the launcher
 python3dll