Skip to content

Commit

Permalink
pythongh-109515: When generating deep frozen modules on Windows, use …
Browse files Browse the repository at this point in the history 
…a list file instead of arguments (pythonGH-109516)
  • Loading branch information
@rghe @FullteaR
rghe authored and FullteaR committed Nov 3, 2023
1 parent 6a480d8 commit af4ffa1
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 30 deletions.
64 changes: 39 additions & 25 deletions PCbuild/_freeze_module.vcxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,33 @@
<OutFile>$(PySourcePath)Python\frozen_modules\getpath.h</OutFile>
</GetPath>
</ItemGroup>
<ItemGroup>
<!-- BEGIN freeze mappings -->
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap.h" FrozenId="importlib._bootstrap" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap_external.h" FrozenId="importlib._bootstrap_external" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\zipimport.h" FrozenId="zipimport" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\abc.h" FrozenId="abc" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\codecs.h" FrozenId="codecs" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\io.h" FrozenId="io" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_collections_abc.h" FrozenId="_collections_abc" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_sitebuiltins.h" FrozenId="_sitebuiltins" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\genericpath.h" FrozenId="genericpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\ntpath.h" FrozenId="ntpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\posixpath.h" FrozenId="posixpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\os.h" FrozenId="os" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\site.h" FrozenId="site" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\stat.h" FrozenId="stat" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.util.h" FrozenId="importlib.util" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.machinery.h" FrozenId="importlib.machinery" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\runpy.h" FrozenId="runpy" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__hello__.h" FrozenId="__hello__" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.h" FrozenId="__phello__" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.h" FrozenId="__phello__.ham" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.eggs.h" FrozenId="__phello__.ham.eggs" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.spam.h" FrozenId="__phello__.spam" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\frozen_only.h" FrozenId="frozen_only" />
<!-- END freeze mappings -->
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
Expand Down Expand Up @@ -426,32 +453,19 @@
AfterTargets="_RebuildFrozen"
DependsOnTargets="FindPythonForBuild"
Condition="$(Configuration) != 'PGUpdate'">
<PropertyGroup>
<FreezeMappingsFile>
<!-- BEGIN freeze mapping file -->
$(IntDir)\deepfreeze_mappings.txt
<!-- END freeze mapping file -->
</FreezeMappingsFile>
</PropertyGroup>
<WriteLinesToFile
File="$(FreezeMappingsFile)"
Overwrite="true"
Lines="@(FrozenModule->'%(FullPath):%(FrozenId)')" />
<!-- BEGIN deepfreeze rule -->
<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" ^
"$(PySourcePath)Python\frozen_modules\importlib._bootstrap.h:importlib._bootstrap" ^
"$(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h:importlib._bootstrap_external" ^
"$(PySourcePath)Python\frozen_modules\zipimport.h:zipimport" ^
"$(PySourcePath)Python\frozen_modules\abc.h:abc" ^
"$(PySourcePath)Python\frozen_modules\codecs.h:codecs" ^
"$(PySourcePath)Python\frozen_modules\io.h:io" ^
"$(PySourcePath)Python\frozen_modules\_collections_abc.h:_collections_abc" ^
"$(PySourcePath)Python\frozen_modules\_sitebuiltins.h:_sitebuiltins" ^
"$(PySourcePath)Python\frozen_modules\genericpath.h:genericpath" ^
"$(PySourcePath)Python\frozen_modules\ntpath.h:ntpath" ^
"$(PySourcePath)Python\frozen_modules\posixpath.h:posixpath" ^
"$(PySourcePath)Python\frozen_modules\os.h:os" ^
"$(PySourcePath)Python\frozen_modules\site.h:site" ^
"$(PySourcePath)Python\frozen_modules\stat.h:stat" ^
"$(PySourcePath)Python\frozen_modules\importlib.util.h:importlib.util" ^
"$(PySourcePath)Python\frozen_modules\importlib.machinery.h:importlib.machinery" ^
"$(PySourcePath)Python\frozen_modules\runpy.h:runpy" ^
"$(PySourcePath)Python\frozen_modules\__hello__.h:__hello__" ^
"$(PySourcePath)Python\frozen_modules\__phello__.h:__phello__" ^
"$(PySourcePath)Python\frozen_modules\__phello__.ham.h:__phello__.ham" ^
"$(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h:__phello__.ham.eggs" ^
"$(PySourcePath)Python\frozen_modules\__phello__.spam.h:__phello__.spam" ^
"$(PySourcePath)Python\frozen_modules\frozen_only.h:frozen_only" ^
"-o" "$(PySourcePath)Python\deepfreeze\deepfreeze.c"'/>
<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" -f "$(IntDir)\deepfreeze_mappings.txt" -o "$(PySourcePath)Python\deepfreeze\deepfreeze.c"' />
<!-- END deepfreeze rule -->
</Target>
<Target Name="_CleanFrozen" BeforeTargets="CoreClean" Condition="$(Configuration) != 'PGUpdate'">
Expand Down
16 changes: 14 additions & 2 deletions Tools/build/deepfreeze.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,10 @@ def generate(args: list[str], output: TextIO) -> None:
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c", default="deepfreeze.c")
parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics")
parser.add_argument('args', nargs="+", help="Input file and module name (required) in file:modname format")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-f", "--file", help="read rule lines from a file")
group.add_argument('args', nargs="*", default=(),
help="Input file and module name (required) in file:modname format")

@contextlib.contextmanager
def report_time(label: str):
Expand All @@ -506,9 +509,18 @@ def main() -> None:
args = parser.parse_args()
verbose = args.verbose
output = args.output

if args.file:
if verbose:
print(f"Reading targets from {args.file}")
with open(args.file, "rt", encoding="utf-8-sig") as fin:
rules = [x.strip() for x in fin]
else:
rules = args.args

with open(output, "w", encoding="utf-8") as file:
with report_time("generate"):
generate(args.args, file)
generate(rules, file)
if verbose:
print(f"Wrote {os.path.getsize(output)} bytes to {output}")

Expand Down
28 changes: 25 additions & 3 deletions Tools/build/freeze_modules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
# .gitattributes and .gitignore files needs to be updated.
FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze')
DEEPFREEZE_MAPPING_FNAME = 'deepfreeze_mappings.txt'

FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
Expand Down Expand Up @@ -645,7 +646,9 @@ def regen_pcbuild(modules):
projlines = []
filterlines = []
corelines = []
deepfreezerules = ['\t<Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" ^']
deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}'
deepfreezerules = [f' <Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" -f "{deepfreezemappingsfile}" -o "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\' />']
deepfreezemappings = []
for src in _iter_sources(modules):
pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
Expand All @@ -659,8 +662,7 @@ def regen_pcbuild(modules):
filterlines.append(f' <None Include="..\\{pyfile}">')
filterlines.append(' <Filter>Python Files</Filter>')
filterlines.append(' </None>')
deepfreezerules.append(f'\t\t "$(PySourcePath){header}:{src.frozenid}" ^')
deepfreezerules.append('\t\t "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' )
deepfreezemappings.append(f' <FrozenModule Include="$(PySourcePath)\\{header}" FrozenId="{src.frozenid}" />\n')

corelines.append(f' <ClCompile Include="..\\Python\\deepfreeze\\deepfreeze.c" />')

Expand All @@ -675,6 +677,26 @@ def regen_pcbuild(modules):
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN freeze mappings -->',
'<!-- END freeze mappings -->',
deepfreezemappings,
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN freeze mapping file -->',
'<!-- END freeze mapping file -->',
[deepfreezemappingsfile, ],
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
Expand Down

0 comments on commit af4ffa1

Please sign in to comment.