Skip to content

Commit

Permalink
bpo-45950: Introduce Bootstrap Python again (#29859)
Browse files Browse the repository at this point in the history
The build system now uses a :program:`_bootstrap_python` interpreter for
freezing and deepfreezing again. To speed up build process the build tools
:program:`_bootstrap_python` and :program:`_freeze_module` are no longer
build with LTO.

Cross building depends on a build Python interpreter, which must have same
version and bytecode as target host Python.
  • Loading branch information
tiran committed Dec 3, 2021
1 parent ccb73a0 commit 84ca123
Show file tree
Hide file tree
Showing 9 changed files with 185 additions and 71 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ Lib/distutils/command/*.pdb
Lib/lib2to3/*.pickle
Lib/test/data/*
!Lib/test/data/README
/_bootstrap_python
/Makefile
/Makefile.pre
Mac/Makefile
Expand Down
63 changes: 42 additions & 21 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ CONFIGURE_CFLAGS_NODIST=@CFLAGS_NODIST@
# Use it when a linker flag should _not_ be part of the distutils LDFLAGS
# once Python is installed (bpo-35257)
CONFIGURE_LDFLAGS_NODIST=@LDFLAGS_NODIST@
# LDFLAGS_NOLTO is an extra flag to disable lto. It is used to speed up building
# of _bootstrap_python and _freeze_module tools, which don't need LTO.
CONFIGURE_LDFLAGS_NOLTO=@LDFLAGS_NOLTO@
CONFIGURE_CPPFLAGS= @CPPFLAGS@
CONFIGURE_LDFLAGS= @LDFLAGS@
# Avoid assigning CFLAGS, LDFLAGS, etc. so users can use them on the
Expand All @@ -103,6 +106,7 @@ PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) -I$(srcdir)/Include
PY_CPPFLAGS= $(BASECPPFLAGS) -I. -I$(srcdir)/Include $(CONFIGURE_CPPFLAGS) $(CPPFLAGS)
PY_LDFLAGS= $(CONFIGURE_LDFLAGS) $(LDFLAGS)
PY_LDFLAGS_NODIST=$(CONFIGURE_LDFLAGS_NODIST) $(LDFLAGS_NODIST)
PY_LDFLAGS_NOLTO=$(PY_LDFLAGS) $(CONFIGURE_LDFLAGS_NOLTO) $(LDFLAGS_NODIST)
NO_AS_NEEDED= @NO_AS_NEEDED@
CCSHARED= @CCSHARED@
# LINKFORSHARED are the flags passed to the $(CC) command that links
Expand Down Expand Up @@ -279,6 +283,9 @@ BUILDPYTHON= python$(BUILDEXE)
PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
# Standard builds use _bootstrap_python for freezing, cross compiling
# uses build Python, which must have the same version and bytecode,
PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@
_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
BUILD_GNU_TYPE= @build@
HOST_GNU_TYPE= @host@
Expand Down Expand Up @@ -938,75 +945,88 @@ regen-test-frozenmain: $(BUILDPYTHON)
Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)

############################################################################
# "Bootstrap Python" used to run deepfreeze.py

BOOTSTRAP_HEADERS = \
Python/frozen_modules/importlib._bootstrap.h \
Python/frozen_modules/importlib._bootstrap_external.h

Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS)

_bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local
$(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS)

############################################################################
# Deepfreeze targets

.PHONY: regen-deepfreeze
regen-deepfreeze: $(DEEPFREEZE_OBJS)

DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py
DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py _bootstrap_python

# BEGIN: deepfreeze modules

Python/deepfreeze/importlib._bootstrap.c: Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c

Python/deepfreeze/importlib._bootstrap_external.c: Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c

Python/deepfreeze/zipimport.c: Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c

Python/deepfreeze/abc.c: Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c

Python/deepfreeze/codecs.c: Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c

Python/deepfreeze/io.c: Python/frozen_modules/io.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c

Python/deepfreeze/_collections_abc.c: Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c

Python/deepfreeze/_sitebuiltins.c: Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c

Python/deepfreeze/genericpath.c: Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c

Python/deepfreeze/ntpath.c: Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c

Python/deepfreeze/posixpath.c: Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c

Python/deepfreeze/os.c: Python/frozen_modules/os.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c

Python/deepfreeze/site.c: Python/frozen_modules/site.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c

Python/deepfreeze/stat.c: Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c

Python/deepfreeze/__hello__.c: Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c

Python/deepfreeze/__phello__.c: Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c

Python/deepfreeze/__phello__.ham.c: Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c

Python/deepfreeze/__phello__.ham.eggs.c: Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c

Python/deepfreeze/__phello__.spam.c: Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c

Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c

# END: deepfreeze modules

Expand Down Expand Up @@ -2295,6 +2315,7 @@ clean-retain-profile: pycremoval
find build -name '*.py[co]' -exec rm -f {} ';' || true
-rm -f pybuilddir.txt
-rm -f Lib/lib2to3/*Grammar*.pickle
-rm -f _bootstrap_python
-rm -f Programs/_testembed Programs/_freeze_module
-rm -f Python/deepfreeze/*.[co]
-rm -f Python/frozen_modules/*.h
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
The build system now uses a :program:`_bootstrap_python` interpreter for
freezing and deepfreezing again. To speed up build process the build tools
:program:`_bootstrap_python` and :program:`_freeze_module` are no longer
build with LTO.
105 changes: 105 additions & 0 deletions Programs/_bootstrap_python.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@

/* Frozen modules bootstrap
*
* Limited and restricted Python interpreter to run
* "Tools/scripts/deepfreeze.py" on systems with no or older Python
* interpreter.
*/

#include "Python.h"
#include "pycore_import.h"

/* Includes for frozen modules: */
#include "Python/frozen_modules/importlib._bootstrap.h"
#include "Python/frozen_modules/importlib._bootstrap_external.h"
/* End includes */

/* Note that a negative size indicates a package. */

static const struct _frozen bootstrap_modules[] = {
{"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)},
{"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)},
{0, 0, 0} /* bootstrap sentinel */
};
static const struct _frozen stdlib_modules[] = {
{0, 0, 0} /* stdlib sentinel */
};
static const struct _frozen test_modules[] = {
{0, 0, 0} /* test sentinel */
};
const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;
const struct _frozen *_PyImport_FrozenStdlib = stdlib_modules;
const struct _frozen *_PyImport_FrozenTest = test_modules;

static const struct _module_alias aliases[] = {
{"_frozen_importlib", "importlib._bootstrap"},
{"_frozen_importlib_external", "importlib._bootstrap_external"},
{0, 0} /* aliases sentinel */
};
const struct _module_alias *_PyImport_FrozenAliases = aliases;

/* Embedding apps may change this pointer to point to their favorite
collection of frozen modules: */

const struct _frozen *PyImport_FrozenModules = NULL;

int
#ifdef MS_WINDOWS
wmain(int argc, wchar_t **argv)
#else
main(int argc, char **argv)
#endif
{
PyStatus status;

PyConfig config;
PyConfig_InitIsolatedConfig(&config);
// don't warn, pybuilddir.txt does not exist yet
config.pathconfig_warnings = 0;
// parse arguments
config.parse_argv = 1;
// add current script dir to sys.path
config.isolated = 0;

#ifdef MS_WINDOWS
status = PyConfig_SetArgv(&config, argc, argv);
#else
status = PyConfig_SetBytesArgv(&config, argc, argv);
#endif
if (PyStatus_Exception(status)) {
goto error;
}

status = PyConfig_Read(&config);
if (config.run_filename == NULL) {
status = PyStatus_Error("Run filename expected");
goto error;
}

#define CLEAR(ATTR) \
do { \
PyMem_RawFree(ATTR); \
ATTR = NULL; \
} while (0)

// isolate from system Python
CLEAR(config.base_prefix);
CLEAR(config.prefix);
CLEAR(config.base_exec_prefix);
CLEAR(config.exec_prefix);

status = Py_InitializeFromConfig(&config);
if (PyStatus_Exception(status)) {
goto error;
}
PyConfig_Clear(&config);

return Py_RunMain();

error:
PyConfig_Clear(&config);
if (PyStatus_IsExit(status)) {
return status.exitcode;
}
Py_ExitStatusException(status);
}
45 changes: 0 additions & 45 deletions Python/bootstrap_frozen.c

This file was deleted.

6 changes: 5 additions & 1 deletion Tools/scripts/deepfreeze.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
"""Deep freeze
The script is executed by _bootstrap_python interpreter. Shared library
extension modules are not available.
"""
import argparse
import ast
import builtins
Expand All @@ -8,7 +13,6 @@
import sys
import time
import types
import unicodedata
from typing import Dict, FrozenSet, Tuple, TextIO

import umarshal
Expand Down
2 changes: 1 addition & 1 deletion Tools/scripts/freeze_modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -589,7 +589,7 @@ def regen_makefile(modules):
])
deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)')
deepfreezerules.append(
f"\t$(PYTHON_FOR_REGEN) "
f"\t$(PYTHON_FOR_FREEZE) "
f"$(srcdir)/Tools/scripts/deepfreeze.py "
f"{frozen_header} -m {src.frozenid} -o {cfile}")
deepfreezerules.append('')
Expand Down
Loading

0 comments on commit 84ca123

Please sign in to comment.