From ac178289306d3bcf28ba54a39bead4622d481a13 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Sun, 4 Feb 2018 13:55:10 +0100 Subject: [PATCH 01/12] Apply the bpo-28833 patch without Misc/python-config.in --- Lib/distutils/command/build_ext.py | 5 +- Lib/distutils/command/install.py | 25 +++--- Lib/distutils/dist.py | 3 +- Lib/distutils/sysconfig.py | 32 +++++--- Lib/distutils/util.py | 6 +- Lib/sysconfig.py | 126 +++++++++++++++++++++++++---- Makefile.pre.in | 6 +- Misc/python-config.sh.in | 20 ++++- configure | 2 +- configure.ac | 2 +- setup.py | 10 +-- 11 files changed, 181 insertions(+), 56 deletions(-) diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index 38bb8fd93c2781b..bc5333df089ed77 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -10,7 +10,8 @@ import sys from distutils.core import Command from distutils.errors import * -from distutils.sysconfig import customize_compiler, get_python_version +from distutils.sysconfig import (customize_compiler, get_python_version, + cross_compiling) from distutils.sysconfig import get_config_h_filename from distutils.dep_util import newer_group from distutils.extension import Extension @@ -156,7 +157,7 @@ def finalize_options(self): # If in a virtualenv, add its include directory # Issue 16116 - if sys.exec_prefix != sys.base_exec_prefix: + if not cross_compiling and sys.exec_prefix != sys.base_exec_prefix: self.include_dirs.append(os.path.join(sys.exec_prefix, 'include')) # Put the Python "system" include dir at the end, so that diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index c625c95bf7e8cf9..a38eda86b0b5c84 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -8,7 +8,7 @@ from distutils import log from distutils.core import Command from distutils.debug import DEBUG -from distutils.sysconfig import get_config_vars +from distutils.sysconfig import (get_config_vars, cross_compiling) from distutils.errors import DistutilsPlatformError from distutils.file_util import write_file from distutils.util import convert_path, subst_vars, change_root @@ -281,12 +281,10 @@ def finalize_options(self): # about needing recursive variable expansion (shudder). py_version = sys.version.split()[0] - (prefix, exec_prefix) = get_config_vars('prefix', 'exec_prefix') - try: - abiflags = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - abiflags = '' + prefix, exec_prefix, abiflags = get_config_vars('prefix', + 'exec_prefix', 'ABIFLAGS') + # sys.abiflags may not be defined on all platforms. + abiflags = '' if abiflags is None else abiflags self.config_vars = {'dist_name': self.distribution.get_name(), 'dist_version': self.distribution.get_version(), 'dist_fullname': self.distribution.get_fullname(), @@ -418,8 +416,13 @@ def finalize_unix(self): raise DistutilsOptionError( "must not supply exec-prefix without prefix") - self.prefix = os.path.normpath(sys.prefix) - self.exec_prefix = os.path.normpath(sys.exec_prefix) + if cross_compiling: + prefix, exec_prefix = get_config_vars('prefix', + 'exec_prefix') + else: + prefix, exec_prefix = sys.prefix, sys.exec_prefix + self.prefix = os.path.normpath(prefix) + self.exec_prefix = os.path.normpath(exec_prefix) else: if self.exec_prefix is None: @@ -442,7 +445,9 @@ def finalize_other(self): self.select_scheme("unix_home") else: if self.prefix is None: - self.prefix = os.path.normpath(sys.prefix) + prefix = (get_config_vars().get('prefix') if + cross_compiling else sys.prefix) + self.prefix = os.path.normpath(prefix) self.install_base = self.install_platbase = self.prefix try: diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py index 6cf0a0d6632dc75..e458d5828b670a6 100644 --- a/Lib/distutils/dist.py +++ b/Lib/distutils/dist.py @@ -19,6 +19,7 @@ from distutils.util import check_environ, strtobool, rfc822_escape from distutils import log from distutils.debug import DEBUG +from distutils.sysconfig import cross_compiling # Regex to define acceptable Distutils command names. This is not *quite* # the same as a Python NAME -- I don't allow leading underscores. The fact @@ -382,7 +383,7 @@ def parse_config_files(self, filenames=None): from configparser import ConfigParser # Ignore install directory options if we have a venv - if sys.prefix != sys.base_prefix: + if not cross_compiling and sys.prefix != sys.base_prefix: ignore_options = [ 'install-base', 'install-platbase', 'install-lib', 'install-platlib', 'install-purelib', 'install-headers', diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index b51629eb94f825d..f294d24c2a0c52f 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -14,6 +14,7 @@ import re import sys +from sysconfig import get_cross_build_var, import_sysconfigdata from .errors import DistutilsPlatformError from .util import get_platform, get_host_platform @@ -22,12 +23,19 @@ EXEC_PREFIX = os.path.normpath(sys.exec_prefix) BASE_PREFIX = os.path.normpath(sys.base_prefix) BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) +cross_compiling = False # Path to the base directory of the project. On Windows the binary may # live in project/PCbuild/win32 or project/PCbuild/amd64. # set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"]) +_xbuild_project_base = get_cross_build_var('project_base') +if _xbuild_project_base is not None: + cross_compiling = True + project_base = _xbuild_project_base + PREFIX = get_cross_build_var('prefix') + EXEC_PREFIX = get_cross_build_var('exec-prefix') + BASE_PREFIX = PREFIX + BASE_EXEC_PREFIX = EXEC_PREFIX else: if sys.executable: project_base = os.path.dirname(os.path.abspath(sys.executable)) @@ -71,7 +79,9 @@ def _python_build(): build_flags = '' try: if not python_build: - build_flags = sys.abiflags + _xbuild_abiflags = get_cross_build_var('abiflags') + build_flags = (sys.abiflags if _xbuild_abiflags is None else + _xbuild_abiflags) except AttributeError: # It's not a configure-based build, so the sys module doesn't have # this attribute, which is fine. @@ -256,8 +266,11 @@ def get_makefile_filename(): return os.path.join(_sys_home or project_base, "Makefile") lib_dir = get_python_lib(plat_specific=0, standard_lib=1) config_file = 'config-{}{}'.format(get_python_version(), build_flags) - if hasattr(sys.implementation, '_multiarch'): - config_file += '-%s' % sys.implementation._multiarch + multiarch = get_cross_build_var('multiarch') + if multiarch is None: + multiarch = getattr(sys.implementation, '_multiarch', '') + if multiarch: + config_file += '-%s' % multiarch return os.path.join(lib_dir, config_file, 'Makefile') @@ -431,14 +444,7 @@ def expand_makefile_vars(s, vars): def _init_posix(): """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see the sysconfig module - name = os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - abi=sys.abiflags, - platform=sys.platform, - multiarch=getattr(sys.implementation, '_multiarch', ''), - )) - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) + _temp = import_sysconfigdata() build_time_vars = _temp.build_time_vars global _config_vars _config_vars = {} diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index 17a94bc42832415..a2fefb3d8aceece 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -14,6 +14,7 @@ from distutils.spawn import spawn from distutils import log from distutils.errors import DistutilsByteCompileError +from sysconfig import get_cross_build_var def get_host_platform(): """Return a string that identifies the current platform. This is used mainly to @@ -45,8 +46,9 @@ def get_host_platform(): return sys.platform # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] + xbuild_host_platform = get_cross_build_var('host_platform') + if xbuild_host_platform is not None: + return xbuild_host_platform if os.name != "posix" or not hasattr(os, 'uname'): # XXX what about the architecture? NT is Intel or Alpha, diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index b9e2fafbc084a6b..f5c9406b81df418 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -114,16 +114,80 @@ def _safe_realpath(path): _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir)) -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) - def _is_python_source_dir(d): for fn in ("Setup", "Setup.local"): if os.path.isfile(os.path.join(d, "Modules", fn)): return True return False +_cross_build_vars = None +def get_cross_build_var(name): + """Return the value of the 'name' variable.""" + global _cross_build_vars + names = ('host_platform', 'version', 'abiflags', 'machdep', 'multiarch', + 'prefix', 'exec-prefix', 'sysconfigdata_name', 'project_base') + if _cross_build_vars is None: + _cross_build_vars = dict((n, None) for n in names) + if 'XBUILD_PYTHON_DIR' in os.environ: + # XBUILD_PYTHON_DIR must not be set when building a native + # interpreter since, at the time generate-posix-vars is run, the + # _posixsubprocess module has not been built yet and the import + # will fail. This does not happen when generate-posix-vars is run + # during cross-compilation as the native interpreter being used is + # a full-fledged interpreter. + try: + import subprocess + except ImportError: + print('Error: XBUILD_PYTHON_DIR is set in the environment.', + file=sys.stderr) + raise + + pyconfig_dir = _safe_realpath(os.environ['XBUILD_PYTHON_DIR']) + if _is_python_source_dir(pyconfig_dir): + python_build = True + python_config = os.path.join(pyconfig_dir, 'python-config') + else: + python_build = False + python_config = os.path.join(pyconfig_dir, 'python3-config') + + output = subprocess.check_output(['sh', python_config, + '--host_platform', '--version', '--abiflags', '--machdep', + '--multiarch', '--prefix', '--exec-prefix'], + universal_newlines=True).split('\n') + # _PYTHON_HOST_PLATFORM is substituted by config.status and not + # empty for cross builds. + if output[0]: + d = _cross_build_vars + d['host_platform'] = output[0] + d['version'] = output[1] + d['abiflags'] = output[2] + d['machdep'] = output[3] + d['multiarch'] = output[4] + d['prefix'] = output[5] + d['exec-prefix'] = output[6] + sysconf = ('_sysconfigdata_%s_%s' % + (d['abiflags'], d['machdep'])) + if d['multiarch']: + sysconf = '%s_%s' % (sysconf, d['multiarch']) + d['sysconfigdata_name'] = sysconf + d['project_base'] = (pyconfig_dir if + python_build else d['prefix']) + + # The specification of the sysconfigdata file name may differ + # across versions, forbid Python versions mismatch. + sys_version = '%d.%d' % sys.version_info[:2] + if d['version'] != sys_version: + raise RuntimeError('the running python version (%s) does ' + 'not match the cross-compiled version (%s)' % + (sys_version, d['version'])) + + return _cross_build_vars[name] + +# set for cross builds +_xbuild_project_base = get_cross_build_var('project_base') +if _xbuild_project_base is not None: + _PROJECT_BASE = _xbuild_project_base + _sys_home = getattr(sys, '_home', None) if os.name == 'nt': @@ -344,13 +408,13 @@ def get_makefile_filename(): def _get_sysconfigdata_name(): - return os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - abi=sys.abiflags, - platform=sys.platform, - multiarch=getattr(sys.implementation, '_multiarch', ''), - )) - + xbuild_sysconfigdata_name = get_cross_build_var('sysconfigdata_name') + if xbuild_sysconfigdata_name is not None: + return xbuild_sysconfigdata_name + sysconf = '_sysconfigdata_%s_%s' % (sys.abiflags, sys.platform) + if hasattr(sys.implementation, '_multiarch'): + sysconf = '%s_%s' % (sysconf, sys.implementation._multiarch) + return sysconf def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" @@ -415,11 +479,35 @@ def _generate_posix_vars(): with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" +def import_sysconfigdata(): # _sysconfigdata is generated at build time, see _generate_posix_vars() name = _get_sysconfigdata_name() - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) + pop_first_path = False + try: + # Temporarily update sys.path to import the sysconfigdata module when + # cross compiling. + xbuild_project_base = get_cross_build_var('project_base') + if xbuild_project_base is not None: + if _is_python_source_dir(xbuild_project_base): + bdir_text = os.path.join(xbuild_project_base, 'pybuilddir.txt') + with open(bdir_text) as f: + pybuilddir = f.read().strip() + pybuilddir = os.path.join(xbuild_project_base, pybuilddir) + sys.path.insert(0, pybuilddir) + pop_first_path = True + else: + stdlib_dir = os.path.join(get_cross_build_var('prefix'), + 'lib', 'python%s' % get_cross_build_var('version')) + sys.path.insert(0, stdlib_dir) + pop_first_path = True + return __import__(name, globals(), locals(), ['build_time_vars'], 0) + finally: + if pop_first_path: + sys.path.pop(0) + +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" + _temp = import_sysconfigdata() build_time_vars = _temp.build_time_vars vars.update(build_time_vars) @@ -637,8 +725,9 @@ def get_platform(): return sys.platform # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] + xbuild_host_platform = get_cross_build_var('host_platform') + if xbuild_host_platform is not None: + return xbuild_host_platform # Try to distinguish various flavours of Unix osname, host, release, version, machine = os.uname() @@ -682,6 +771,11 @@ def get_platform(): return "%s-%s-%s" % (osname, release, machine) +def get_host_platform(): + """Return the host platform for cross builds, None otherwise.""" + return get_cross_build_var('host_platform') + + def get_python_version(): return _PY_VERSION_SHORT diff --git a/Makefile.pre.in b/Makefile.pre.in index d08c78df394b375..06d300c18e5d75c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -246,7 +246,7 @@ BUILDPYTHON= python$(BUILDEXE) PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@ UPDATE_FILE=@PYTHON_FOR_REGEN@ $(srcdir)/Tools/scripts/update_file.py PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ -_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ +PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ HOST_GNU_TYPE= @host@ @@ -448,7 +448,7 @@ DTRACE_DEPS = \ # Default target all: @DEF_MAKE_ALL_RULE@ build_all: check-clean-src $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks \ - Programs/_testembed python-config + Programs/_testembed # Check that the source is clean when building out of source. check-clean-src: @@ -575,7 +575,7 @@ platform: $(BUILDPYTHON) pybuilddir.txt # problems by creating a dummy pybuilddir.txt just to allow interpreter # initialization to succeed. It will be overwritten by generate-posix-vars # or removed in case of failure. -pybuilddir.txt: $(BUILDPYTHON) +pybuilddir.txt: python-config $(BUILDPYTHON) @echo "none" > ./pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\ if test $$? -ne 0 ; then \ diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 2602fe24c0402e1..28fffdc6ab336c3 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -4,7 +4,9 @@ exit_with_usage () { - echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|\ + --ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed|\ + --host_platform|--version|--machdep|--multiarch" exit $1 } @@ -66,7 +68,9 @@ do --embed) PY_EMBED=1 ;; - --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--abiflags|--configdir) + --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|\ + --extension-suffix|--abiflags|--configdir|\ + --host_platform|--version|--machdep|--multiarch) ;; *) exit_with_usage 1 @@ -112,5 +116,17 @@ do --configdir) echo "$LIBPL" ;; + --host_platform) + echo "@_PYTHON_HOST_PLATFORM@" + ;; + --version) + echo "$VERSION" + ;; + --machdep) + echo "@MACHDEP@" + ;; + --multiarch) + echo "@MULTIARCH@" + ;; esac done diff --git a/configure b/configure index 44f14c3c2cfe1b6..acddcbc69552933 100755 --- a/configure +++ b/configure @@ -2947,7 +2947,7 @@ $as_echo_n "checking for python interpreter for cross build... " >&6; } fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $interp" >&5 $as_echo "$interp" >&6; } - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp + PYTHON_FOR_BUILD='XBUILD_PYTHON_DIR=$(abs_builddir) '$interp fi elif test "$cross_compiling" = maybe; then as_fn_error $? "Cross compiling required --host=HOST-TUPLE and --build=ARCH" "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 0b28dda44cdb8e5..b82a8b15a179907 100644 --- a/configure.ac +++ b/configure.ac @@ -75,7 +75,7 @@ if test "$cross_compiling" = yes; then AC_MSG_ERROR([python$PACKAGE_VERSION interpreter not found]) fi AC_MSG_RESULT($interp) - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp + PYTHON_FOR_BUILD='XBUILD_PYTHON_DIR=$(abs_builddir) '$interp fi elif test "$cross_compiling" = maybe; then AC_MSG_ERROR([Cross compiling required --host=HOST-TUPLE and --build=ARCH]) diff --git a/setup.py b/setup.py index 02f523c42d355f2..8f8444069e5a52e 100644 --- a/setup.py +++ b/setup.py @@ -28,17 +28,17 @@ def get_platform(): - # Cross compiling - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - + # cross build + host_platform = sysconfig.get_host_platform() + if host_platform: + return host_platform # Get value of sys.platform if sys.platform.startswith('osf1'): return 'osf1' return sys.platform -CROSS_COMPILING = ("_PYTHON_HOST_PLATFORM" in os.environ) +CROSS_COMPILING = bool(sysconfig.get_host_platform()) HOST_PLATFORM = get_platform() MS_WINDOWS = (HOST_PLATFORM == 'win32') CYGWIN = (HOST_PLATFORM == 'cygwin') From 73134003a5417685797339e2c58676c4668a857c Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Mon, 5 Feb 2018 16:50:36 +0100 Subject: [PATCH 02/12] Substitute 'xbuild' to 'cross_build' --- Lib/distutils/sysconfig.py | 12 ++++++------ Lib/distutils/util.py | 6 +++--- Lib/sysconfig.py | 36 ++++++++++++++++++------------------ configure | 2 +- configure.ac | 2 +- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index f294d24c2a0c52f..52eae9bd814a285 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -28,10 +28,10 @@ # Path to the base directory of the project. On Windows the binary may # live in project/PCbuild/win32 or project/PCbuild/amd64. # set for cross builds -_xbuild_project_base = get_cross_build_var('project_base') -if _xbuild_project_base is not None: +_cross_build_project_base = get_cross_build_var('project_base') +if _cross_build_project_base is not None: cross_compiling = True - project_base = _xbuild_project_base + project_base = _cross_build_project_base PREFIX = get_cross_build_var('prefix') EXEC_PREFIX = get_cross_build_var('exec-prefix') BASE_PREFIX = PREFIX @@ -79,9 +79,9 @@ def _python_build(): build_flags = '' try: if not python_build: - _xbuild_abiflags = get_cross_build_var('abiflags') - build_flags = (sys.abiflags if _xbuild_abiflags is None else - _xbuild_abiflags) + _cross_build_abiflags = get_cross_build_var('abiflags') + build_flags = (sys.abiflags if _cross_build_abiflags is None else + _cross_build_abiflags) except AttributeError: # It's not a configure-based build, so the sys module doesn't have # this attribute, which is fine. diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index a2fefb3d8aceece..89af2bfe0cdb769 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -46,9 +46,9 @@ def get_host_platform(): return sys.platform # Set for cross builds explicitly - xbuild_host_platform = get_cross_build_var('host_platform') - if xbuild_host_platform is not None: - return xbuild_host_platform + cross_build_host_platform = get_cross_build_var('host_platform') + if cross_build_host_platform is not None: + return cross_build_host_platform if os.name != "posix" or not hasattr(os, 'uname'): # XXX what about the architecture? NT is Intel or Alpha, diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index f5c9406b81df418..21fcf14b5247eb2 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -128,8 +128,8 @@ def get_cross_build_var(name): 'prefix', 'exec-prefix', 'sysconfigdata_name', 'project_base') if _cross_build_vars is None: _cross_build_vars = dict((n, None) for n in names) - if 'XBUILD_PYTHON_DIR' in os.environ: - # XBUILD_PYTHON_DIR must not be set when building a native + if 'CROSS_BUILD_PYTHON_DIR' in os.environ: + # CROSS_BUILD_PYTHON_DIR must not be set when building a native # interpreter since, at the time generate-posix-vars is run, the # _posixsubprocess module has not been built yet and the import # will fail. This does not happen when generate-posix-vars is run @@ -138,11 +138,11 @@ def get_cross_build_var(name): try: import subprocess except ImportError: - print('Error: XBUILD_PYTHON_DIR is set in the environment.', + print('Error: CROSS_BUILD_PYTHON_DIR is set in the environment.', file=sys.stderr) raise - pyconfig_dir = _safe_realpath(os.environ['XBUILD_PYTHON_DIR']) + pyconfig_dir = _safe_realpath(os.environ['CROSS_BUILD_PYTHON_DIR']) if _is_python_source_dir(pyconfig_dir): python_build = True python_config = os.path.join(pyconfig_dir, 'python-config') @@ -184,9 +184,9 @@ def get_cross_build_var(name): return _cross_build_vars[name] # set for cross builds -_xbuild_project_base = get_cross_build_var('project_base') -if _xbuild_project_base is not None: - _PROJECT_BASE = _xbuild_project_base +_cross_build_project_base = get_cross_build_var('project_base') +if _cross_build_project_base is not None: + _PROJECT_BASE = _cross_build_project_base _sys_home = getattr(sys, '_home', None) @@ -408,9 +408,9 @@ def get_makefile_filename(): def _get_sysconfigdata_name(): - xbuild_sysconfigdata_name = get_cross_build_var('sysconfigdata_name') - if xbuild_sysconfigdata_name is not None: - return xbuild_sysconfigdata_name + cross_build_sysconfigdata_name = get_cross_build_var('sysconfigdata_name') + if cross_build_sysconfigdata_name is not None: + return cross_build_sysconfigdata_name sysconf = '_sysconfigdata_%s_%s' % (sys.abiflags, sys.platform) if hasattr(sys.implementation, '_multiarch'): sysconf = '%s_%s' % (sysconf, sys.implementation._multiarch) @@ -486,13 +486,13 @@ def import_sysconfigdata(): try: # Temporarily update sys.path to import the sysconfigdata module when # cross compiling. - xbuild_project_base = get_cross_build_var('project_base') - if xbuild_project_base is not None: - if _is_python_source_dir(xbuild_project_base): - bdir_text = os.path.join(xbuild_project_base, 'pybuilddir.txt') + cross_build_project_base = get_cross_build_var('project_base') + if cross_build_project_base is not None: + if _is_python_source_dir(cross_build_project_base): + bdir_text = os.path.join(cross_build_project_base, 'pybuilddir.txt') with open(bdir_text) as f: pybuilddir = f.read().strip() - pybuilddir = os.path.join(xbuild_project_base, pybuilddir) + pybuilddir = os.path.join(cross_build_project_base, pybuilddir) sys.path.insert(0, pybuilddir) pop_first_path = True else: @@ -725,9 +725,9 @@ def get_platform(): return sys.platform # Set for cross builds explicitly - xbuild_host_platform = get_cross_build_var('host_platform') - if xbuild_host_platform is not None: - return xbuild_host_platform + cross_build_host_platform = get_cross_build_var('host_platform') + if cross_build_host_platform is not None: + return cross_build_host_platform # Try to distinguish various flavours of Unix osname, host, release, version, machine = os.uname() diff --git a/configure b/configure index acddcbc69552933..2e550fa81677429 100755 --- a/configure +++ b/configure @@ -2947,7 +2947,7 @@ $as_echo_n "checking for python interpreter for cross build... " >&6; } fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $interp" >&5 $as_echo "$interp" >&6; } - PYTHON_FOR_BUILD='XBUILD_PYTHON_DIR=$(abs_builddir) '$interp + PYTHON_FOR_BUILD='CROSS_BUILD_PYTHON_DIR=$(abs_builddir) '$interp fi elif test "$cross_compiling" = maybe; then as_fn_error $? "Cross compiling required --host=HOST-TUPLE and --build=ARCH" "$LINENO" 5 diff --git a/configure.ac b/configure.ac index b82a8b15a179907..3d0d6255d17f057 100644 --- a/configure.ac +++ b/configure.ac @@ -75,7 +75,7 @@ if test "$cross_compiling" = yes; then AC_MSG_ERROR([python$PACKAGE_VERSION interpreter not found]) fi AC_MSG_RESULT($interp) - PYTHON_FOR_BUILD='XBUILD_PYTHON_DIR=$(abs_builddir) '$interp + PYTHON_FOR_BUILD='CROSS_BUILD_PYTHON_DIR=$(abs_builddir) '$interp fi elif test "$cross_compiling" = maybe; then AC_MSG_ERROR([Cross compiling required --host=HOST-TUPLE and --build=ARCH]) From 6034e1d7d7cfb6ff17b88266c10bfc71a5994132 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Wed, 7 Feb 2018 18:22:15 +0100 Subject: [PATCH 03/12] Fix the trailing underscore of the sysconfigdata module name --- Lib/sysconfig.py | 67 ++++++++++++++++++++++-------------------------- Makefile.pre.in | 14 +++++++--- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 21fcf14b5247eb2..8ada6b35fae3299 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -123,39 +123,32 @@ def _is_python_source_dir(d): _cross_build_vars = None def get_cross_build_var(name): """Return the value of the 'name' variable.""" + global _cross_build_vars - names = ('host_platform', 'version', 'abiflags', 'machdep', 'multiarch', - 'prefix', 'exec-prefix', 'sysconfigdata_name', 'project_base') + if _cross_build_vars is None: - _cross_build_vars = dict((n, None) for n in names) + _cross_build_vars = {} if 'CROSS_BUILD_PYTHON_DIR' in os.environ: - # CROSS_BUILD_PYTHON_DIR must not be set when building a native - # interpreter since, at the time generate-posix-vars is run, the - # _posixsubprocess module has not been built yet and the import - # will fail. This does not happen when generate-posix-vars is run - # during cross-compilation as the native interpreter being used is - # a full-fledged interpreter. - try: - import subprocess - except ImportError: - print('Error: CROSS_BUILD_PYTHON_DIR is set in the environment.', - file=sys.stderr) - raise - - pyconfig_dir = _safe_realpath(os.environ['CROSS_BUILD_PYTHON_DIR']) - if _is_python_source_dir(pyconfig_dir): + # The import would fail when building a native interpreter since, + # at the time generate-posix-vars is run, the _posixsubprocess + # module has not been built yet. This does not happen when + # generate-posix-vars is run during cross-compilation as the + # native interpreter being used is a full-fledged interpreter. + import subprocess + + pyconf_dir = _safe_realpath(os.environ['CROSS_BUILD_PYTHON_DIR']) + if _is_python_source_dir(pyconf_dir): python_build = True - python_config = os.path.join(pyconfig_dir, 'python-config') + python_config = os.path.join(pyconf_dir, 'python-config') else: python_build = False - python_config = os.path.join(pyconfig_dir, 'python3-config') + python_config = os.path.join(pyconf_dir, 'bin', + 'python3-config') output = subprocess.check_output(['sh', python_config, '--host_platform', '--version', '--abiflags', '--machdep', '--multiarch', '--prefix', '--exec-prefix'], universal_newlines=True).split('\n') - # _PYTHON_HOST_PLATFORM is substituted by config.status and not - # empty for cross builds. if output[0]: d = _cross_build_vars d['host_platform'] = output[0] @@ -165,12 +158,12 @@ def get_cross_build_var(name): d['multiarch'] = output[4] d['prefix'] = output[5] d['exec-prefix'] = output[6] - sysconf = ('_sysconfigdata_%s_%s' % + sysconf_name = ('_sysconfigdata_%s_%s' % (d['abiflags'], d['machdep'])) if d['multiarch']: - sysconf = '%s_%s' % (sysconf, d['multiarch']) - d['sysconfigdata_name'] = sysconf - d['project_base'] = (pyconfig_dir if + sysconf_name = '%s_%s' % (sysconf_name, d['multiarch']) + d['sysconfigdata_name'] = sysconf_name + d['project_base'] = (pyconf_dir if python_build else d['prefix']) # The specification of the sysconfigdata file name may differ @@ -181,7 +174,7 @@ def get_cross_build_var(name): 'not match the cross-compiled version (%s)' % (sys_version, d['version'])) - return _cross_build_vars[name] + return _cross_build_vars.get(name) # set for cross builds _cross_build_project_base = get_cross_build_var('project_base') @@ -411,10 +404,10 @@ def _get_sysconfigdata_name(): cross_build_sysconfigdata_name = get_cross_build_var('sysconfigdata_name') if cross_build_sysconfigdata_name is not None: return cross_build_sysconfigdata_name - sysconf = '_sysconfigdata_%s_%s' % (sys.abiflags, sys.platform) + sysconf_name = '_sysconfigdata_%s_%s' % (sys.abiflags, sys.platform) if hasattr(sys.implementation, '_multiarch'): - sysconf = '%s_%s' % (sysconf, sys.implementation._multiarch) - return sysconf + sysconf_name = '%s_%s' % (sysconf_name, sys.implementation._multiarch) + return sysconf_name def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" @@ -486,18 +479,18 @@ def import_sysconfigdata(): try: # Temporarily update sys.path to import the sysconfigdata module when # cross compiling. - cross_build_project_base = get_cross_build_var('project_base') - if cross_build_project_base is not None: - if _is_python_source_dir(cross_build_project_base): - bdir_text = os.path.join(cross_build_project_base, 'pybuilddir.txt') - with open(bdir_text) as f: + xbuild_project_base = get_cross_build_var('project_base') + if xbuild_project_base is not None: + if _is_python_source_dir(xbuild_project_base): + bdir = os.path.join(xbuild_project_base, 'pybuilddir.txt') + with open(bdir) as f: pybuilddir = f.read().strip() - pybuilddir = os.path.join(cross_build_project_base, pybuilddir) + pybuilddir = os.path.join(xbuild_project_base, pybuilddir) sys.path.insert(0, pybuilddir) pop_first_path = True else: stdlib_dir = os.path.join(get_cross_build_var('prefix'), - 'lib', 'python%s' % get_cross_build_var('version')) + 'lib', 'python%s' % get_cross_build_var('version')) sys.path.insert(0, stdlib_dir) pop_first_path = True return __import__(name, globals(), locals(), ['build_time_vars'], 0) diff --git a/Makefile.pre.in b/Makefile.pre.in index 06d300c18e5d75c..109808095935d43 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1453,8 +1453,12 @@ libinstall: build_all $(srcdir)/Modules/xxmodule.c esac; \ done; \ done - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ - $(DESTDIR)$(LIBDEST); \ + sysconf_name=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP); \ + if test -n "$(MULTIARCH)"; then \ + sysconf_name=$$sysconf_name_$(MULTIARCH); \ + fi; \ + $(INSTALL_DATA) `cat pybuilddir.txt`/$$sysconf_name.py \ + $(DESTDIR)$(LIBDEST) $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt if test -d $(DESTDIR)$(LIBDEST)/distutils/tests; then \ $(INSTALL_DATA) $(srcdir)/Modules/xxmodule.c \ @@ -1609,7 +1613,11 @@ sharedinstall: sharedmods --install-scripts=$(BINDIR) \ --install-platlib=$(DESTSHARED) \ --root=$(DESTDIR)/ - -rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py + -sysconf_name=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP); \ + if test -n "$(MULTIARCH)"; then \ + sysconf_name=$$sysconf_name_$(MULTIARCH); \ + fi; \ + rm $(DESTDIR)$(DESTSHARED)/$$sysconf_name.py -rm -r $(DESTDIR)$(DESTSHARED)/__pycache__ # Here are a couple of targets for MacOSX again, to install a full From 5089d18eb869fc1b855fb0c1f73343f2979d4937 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Thu, 8 Feb 2018 18:11:59 +0100 Subject: [PATCH 04/12] Use 'libdir' to build extension modules from an installed python build Paths output by python-config.sh.in are *installed* paths. --- Lib/distutils/sysconfig.py | 3 ++ Lib/sysconfig.py | 80 ++++++++++++++++++-------------------- Misc/python-config.sh.in | 10 ++++- 3 files changed, 50 insertions(+), 43 deletions(-) diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 52eae9bd814a285..219170e66b6e34b 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -536,6 +536,9 @@ def get_config_vars(*args): import _osx_support _osx_support.customize_config_vars(_config_vars) + if cross_compiling: + _config_vars['LIBDIR'] = get_cross_build_var('libdir') + if args: vals = [] for name in args: diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 8ada6b35fae3299..d05269b5287dd68 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -145,41 +145,33 @@ def get_cross_build_var(name): python_config = os.path.join(pyconf_dir, 'bin', 'python3-config') - output = subprocess.check_output(['sh', python_config, - '--host_platform', '--version', '--abiflags', '--machdep', - '--multiarch', '--prefix', '--exec-prefix'], - universal_newlines=True).split('\n') - if output[0]: - d = _cross_build_vars - d['host_platform'] = output[0] - d['version'] = output[1] - d['abiflags'] = output[2] - d['machdep'] = output[3] - d['multiarch'] = output[4] - d['prefix'] = output[5] - d['exec-prefix'] = output[6] - sysconf_name = ('_sysconfigdata_%s_%s' % - (d['abiflags'], d['machdep'])) - if d['multiarch']: - sysconf_name = '%s_%s' % (sysconf_name, d['multiarch']) - d['sysconfigdata_name'] = sysconf_name - d['project_base'] = (pyconf_dir if - python_build else d['prefix']) - - # The specification of the sysconfigdata file name may differ - # across versions, forbid Python versions mismatch. - sys_version = '%d.%d' % sys.version_info[:2] - if d['version'] != sys_version: - raise RuntimeError('the running python version (%s) does ' - 'not match the cross-compiled version (%s)' % - (sys_version, d['version'])) + vars_ = ['host_platform', 'version', 'abiflags', 'machdep', + 'multiarch', 'prefix', 'exec-prefix', 'libdir'] + args = ['sh', python_config] + args.extend('--' + v for v in vars_) + output = subprocess.check_output(args, universal_newlines=True) + _cross_build_vars = dict(zip(vars_, output.split('\n'))) + assert len(_cross_build_vars) == len(vars_) + d = _cross_build_vars + d['project_base'] = (pyconf_dir if + python_build else d['prefix']) + + # The specification of the sysconfigdata file name may differ + # across versions, forbid Python versions mismatch. + sys_version = '%d.%d' % sys.version_info[:2] + if d['version'] != sys_version: + raise RuntimeError('the running python version (%s) does ' + 'not match the cross-compiled version (%s)' % + (sys_version, d['version'])) return _cross_build_vars.get(name) +cross_compiling = False # set for cross builds _cross_build_project_base = get_cross_build_var('project_base') if _cross_build_project_base is not None: _PROJECT_BASE = _cross_build_project_base + cross_compiling = True _sys_home = getattr(sys, '_home', None) @@ -401,12 +393,18 @@ def get_makefile_filename(): def _get_sysconfigdata_name(): - cross_build_sysconfigdata_name = get_cross_build_var('sysconfigdata_name') - if cross_build_sysconfigdata_name is not None: - return cross_build_sysconfigdata_name - sysconf_name = '_sysconfigdata_%s_%s' % (sys.abiflags, sys.platform) - if hasattr(sys.implementation, '_multiarch'): - sysconf_name = '%s_%s' % (sysconf_name, sys.implementation._multiarch) + if cross_compiling: + abiflags = get_cross_build_var('abiflags') + platform = get_cross_build_var('machdep') + multiarch = get_cross_build_var('multiarch') + else: + abiflags = sys.abiflags + platform = sys.platform + multiarch = getattr(sys.implementation, '_multiarch', '') + + sysconf_name = '_sysconfigdata_%s_%s' % (abiflags, platform) + if multiarch: + sysconf_name = '%s_%s' % (sysconf_name, multiarch) return sysconf_name def _generate_posix_vars(): @@ -479,13 +477,12 @@ def import_sysconfigdata(): try: # Temporarily update sys.path to import the sysconfigdata module when # cross compiling. - xbuild_project_base = get_cross_build_var('project_base') - if xbuild_project_base is not None: - if _is_python_source_dir(xbuild_project_base): - bdir = os.path.join(xbuild_project_base, 'pybuilddir.txt') + if cross_compiling: + if _is_python_source_dir(_PROJECT_BASE): + bdir = os.path.join(_PROJECT_BASE, 'pybuilddir.txt') with open(bdir) as f: pybuilddir = f.read().strip() - pybuilddir = os.path.join(xbuild_project_base, pybuilddir) + pybuilddir = os.path.join(_PROJECT_BASE, pybuilddir) sys.path.insert(0, pybuilddir) pop_first_path = True else: @@ -718,9 +715,8 @@ def get_platform(): return sys.platform # Set for cross builds explicitly - cross_build_host_platform = get_cross_build_var('host_platform') - if cross_build_host_platform is not None: - return cross_build_host_platform + if cross_compiling: + return get_cross_build_var('host_platform') # Try to distinguish various flavours of Unix osname, host, release, version, machine = os.uname() diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 28fffdc6ab336c3..33bcda02c5ce0d4 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -5,8 +5,13 @@ exit_with_usage () { echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|\ +<<<<<<< HEAD --ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed|\ --host_platform|--version|--machdep|--multiarch" +======= + --ldflags|--extension-suffix|--help|--abiflags|--configdir|\ + --host_platform|--version|--machdep|--multiarch|--libdir" +>>>>>>> Use 'libdir' to build extension modules from an installed python build exit $1 } @@ -70,7 +75,7 @@ do ;; --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|\ --extension-suffix|--abiflags|--configdir|\ - --host_platform|--version|--machdep|--multiarch) + --host_platform|--version|--machdep|--multiarch|--libdir) ;; *) exit_with_usage 1 @@ -128,5 +133,8 @@ do --multiarch) echo "@MULTIARCH@" ;; + --libdir) + echo "$libdir" + ;; esac done From 436e4209cbf9c1328cb812b3242d7433955ca92c Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Sun, 11 Feb 2018 12:30:27 +0100 Subject: [PATCH 05/12] Refactor the changes --- Lib/distutils/command/build_ext.py | 5 ++- Lib/distutils/command/install.py | 2 +- Lib/distutils/dist.py | 3 +- Lib/distutils/sysconfig.py | 23 +++++-------- Lib/distutils/util.py | 7 ++-- Lib/sysconfig.py | 53 +++++++++++------------------- setup.py | 8 ++--- 7 files changed, 39 insertions(+), 62 deletions(-) diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index bc5333df089ed77..38bb8fd93c2781b 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -10,8 +10,7 @@ import sys from distutils.core import Command from distutils.errors import * -from distutils.sysconfig import (customize_compiler, get_python_version, - cross_compiling) +from distutils.sysconfig import customize_compiler, get_python_version from distutils.sysconfig import get_config_h_filename from distutils.dep_util import newer_group from distutils.extension import Extension @@ -157,7 +156,7 @@ def finalize_options(self): # If in a virtualenv, add its include directory # Issue 16116 - if not cross_compiling and sys.exec_prefix != sys.base_exec_prefix: + if sys.exec_prefix != sys.base_exec_prefix: self.include_dirs.append(os.path.join(sys.exec_prefix, 'include')) # Put the Python "system" include dir at the end, so that diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index a38eda86b0b5c84..6c30a9eadff830c 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -8,7 +8,7 @@ from distutils import log from distutils.core import Command from distutils.debug import DEBUG -from distutils.sysconfig import (get_config_vars, cross_compiling) +from distutils.sysconfig import get_config_vars, cross_compiling from distutils.errors import DistutilsPlatformError from distutils.file_util import write_file from distutils.util import convert_path, subst_vars, change_root diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py index e458d5828b670a6..6cf0a0d6632dc75 100644 --- a/Lib/distutils/dist.py +++ b/Lib/distutils/dist.py @@ -19,7 +19,6 @@ from distutils.util import check_environ, strtobool, rfc822_escape from distutils import log from distutils.debug import DEBUG -from distutils.sysconfig import cross_compiling # Regex to define acceptable Distutils command names. This is not *quite* # the same as a Python NAME -- I don't allow leading underscores. The fact @@ -383,7 +382,7 @@ def parse_config_files(self, filenames=None): from configparser import ConfigParser # Ignore install directory options if we have a venv - if not cross_compiling and sys.prefix != sys.base_prefix: + if sys.prefix != sys.base_prefix: ignore_options = [ 'install-base', 'install-platbase', 'install-lib', 'install-platlib', 'install-purelib', 'install-headers', diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 219170e66b6e34b..be3b86a9ea41d4a 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -14,7 +14,8 @@ import re import sys -from sysconfig import get_cross_build_var, import_sysconfigdata +from sysconfig import (cross_compiling, get_cross_build_var, + get_build_time_vars) from .errors import DistutilsPlatformError from .util import get_platform, get_host_platform @@ -23,15 +24,12 @@ EXEC_PREFIX = os.path.normpath(sys.exec_prefix) BASE_PREFIX = os.path.normpath(sys.base_prefix) BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) -cross_compiling = False # Path to the base directory of the project. On Windows the binary may # live in project/PCbuild/win32 or project/PCbuild/amd64. # set for cross builds -_cross_build_project_base = get_cross_build_var('project_base') -if _cross_build_project_base is not None: - cross_compiling = True - project_base = _cross_build_project_base +if cross_compiling: + project_base = get_cross_build_var('project_base') PREFIX = get_cross_build_var('prefix') EXEC_PREFIX = get_cross_build_var('exec-prefix') BASE_PREFIX = PREFIX @@ -79,9 +77,8 @@ def _python_build(): build_flags = '' try: if not python_build: - _cross_build_abiflags = get_cross_build_var('abiflags') - build_flags = (sys.abiflags if _cross_build_abiflags is None else - _cross_build_abiflags) + build_flags = (get_cross_build_var('abiflags') if cross_compiling else + sys.abiflags) except AttributeError: # It's not a configure-based build, so the sys module doesn't have # this attribute, which is fine. @@ -266,9 +263,8 @@ def get_makefile_filename(): return os.path.join(_sys_home or project_base, "Makefile") lib_dir = get_python_lib(plat_specific=0, standard_lib=1) config_file = 'config-{}{}'.format(get_python_version(), build_flags) - multiarch = get_cross_build_var('multiarch') - if multiarch is None: - multiarch = getattr(sys.implementation, '_multiarch', '') + multiarch = (get_cross_build_var('multiarch') if cross_compiling else + getattr(sys.implementation, '_multiarch', '')) if multiarch: config_file += '-%s' % multiarch return os.path.join(lib_dir, config_file, 'Makefile') @@ -444,8 +440,7 @@ def expand_makefile_vars(s, vars): def _init_posix(): """Initialize the module as appropriate for POSIX systems.""" - _temp = import_sysconfigdata() - build_time_vars = _temp.build_time_vars + build_time_vars = get_build_time_vars() global _config_vars _config_vars = {} _config_vars.update(build_time_vars) diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index 89af2bfe0cdb769..601076bb0433f13 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -14,7 +14,7 @@ from distutils.spawn import spawn from distutils import log from distutils.errors import DistutilsByteCompileError -from sysconfig import get_cross_build_var +from sysconfig import cross_compiling, get_cross_build_var def get_host_platform(): """Return a string that identifies the current platform. This is used mainly to @@ -46,9 +46,8 @@ def get_host_platform(): return sys.platform # Set for cross builds explicitly - cross_build_host_platform = get_cross_build_var('host_platform') - if cross_build_host_platform is not None: - return cross_build_host_platform + if cross_compiling: + return get_cross_build_var('host_platform') if os.name != "posix" or not hasattr(os, 'uname'): # XXX what about the architecture? NT is Intel or Alpha, diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index d05269b5287dd68..5cc73feb5e69abf 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -470,35 +470,31 @@ def _generate_posix_vars(): with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) -def import_sysconfigdata(): +def get_build_time_vars(): # _sysconfigdata is generated at build time, see _generate_posix_vars() name = _get_sysconfigdata_name() - pop_first_path = False - try: - # Temporarily update sys.path to import the sysconfigdata module when - # cross compiling. - if cross_compiling: - if _is_python_source_dir(_PROJECT_BASE): - bdir = os.path.join(_PROJECT_BASE, 'pybuilddir.txt') - with open(bdir) as f: - pybuilddir = f.read().strip() - pybuilddir = os.path.join(_PROJECT_BASE, pybuilddir) - sys.path.insert(0, pybuilddir) - pop_first_path = True - else: - stdlib_dir = os.path.join(get_cross_build_var('prefix'), - 'lib', 'python%s' % get_cross_build_var('version')) - sys.path.insert(0, stdlib_dir) - pop_first_path = True - return __import__(name, globals(), locals(), ['build_time_vars'], 0) - finally: - if pop_first_path: - sys.path.pop(0) + if cross_compiling: + if _is_python_source_dir(_PROJECT_BASE): + bdir = os.path.join(_PROJECT_BASE, 'pybuilddir.txt') + with open(bdir, encoding='ascii') as f: + libdir = f.read().strip() + libdir = os.path.join(_PROJECT_BASE, libdir) + else: + libdir = os.path.join(get_cross_build_var('prefix'), + 'lib', 'python%s' % get_cross_build_var('version')) + sysconf = os.path.join(libdir, name + '.py') + with open(sysconf) as f: + code = f.read() + locals_ = {} + exec(code, globals(), locals_) + return locals_['build_time_vars'] + else: + _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) + return _temp.build_time_vars def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" - _temp = import_sysconfigdata() - build_time_vars = _temp.build_time_vars + build_time_vars = get_build_time_vars() vars.update(build_time_vars) def _init_non_posix(vars): @@ -714,10 +710,6 @@ def get_platform(): # XXX what about the architecture? NT is Intel or Alpha return sys.platform - # Set for cross builds explicitly - if cross_compiling: - return get_cross_build_var('host_platform') - # Try to distinguish various flavours of Unix osname, host, release, version, machine = os.uname() @@ -760,11 +752,6 @@ def get_platform(): return "%s-%s-%s" % (osname, release, machine) -def get_host_platform(): - """Return the host platform for cross builds, None otherwise.""" - return get_cross_build_var('host_platform') - - def get_python_version(): return _PY_VERSION_SHORT diff --git a/setup.py b/setup.py index 8f8444069e5a52e..6e447435be8307f 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ import sysconfig from glob import glob +from sysconfig import CROSS_COMPILING from distutils import log from distutils.command.build_ext import build_ext from distutils.command.build_scripts import build_scripts @@ -28,17 +29,14 @@ def get_platform(): - # cross build - host_platform = sysconfig.get_host_platform() - if host_platform: - return host_platform + if CROSS_COMPILING: + return sysconfig.get_cross_build_var('host_platform') # Get value of sys.platform if sys.platform.startswith('osf1'): return 'osf1' return sys.platform -CROSS_COMPILING = bool(sysconfig.get_host_platform()) HOST_PLATFORM = get_platform() MS_WINDOWS = (HOST_PLATFORM == 'win32') CYGWIN = (HOST_PLATFORM == 'cygwin') From 2f2c8f6196f512ba0fb882168a16ca955b46ed33 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Thu, 9 May 2019 16:21:03 +0200 Subject: [PATCH 06/12] Use python-config only to import the sysconfigdata module --- Lib/distutils/command/build.py | 4 +- Lib/distutils/command/build_ext.py | 8 +- Lib/distutils/command/install.py | 4 +- Lib/distutils/sysconfig.py | 57 ++++++---- Lib/distutils/util.py | 4 +- Lib/sysconfig.py | 165 ++++++++++++++++------------- Lib/test/pythoninfo.py | 4 +- Makefile.pre.in | 6 +- Misc/python-config.sh.in | 15 +-- configure | 2 +- configure.ac | 2 +- setup.py | 5 +- 12 files changed, 151 insertions(+), 125 deletions(-) diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py index a86df0bc7f92188..e8cc169f65caab0 100644 --- a/Lib/distutils/command/build.py +++ b/Lib/distutils/command/build.py @@ -6,6 +6,7 @@ from distutils.core import Command from distutils.errors import DistutilsOptionError from distutils.util import get_platform +from distutils.sysconfig import cross_compiling, get_config_var def show_compilers(): @@ -86,7 +87,8 @@ def finalize_options(self): # Make it so Python 2.x and Python 2.x with --with-pydebug don't # share the same build directories. Doing so confuses the build # process for C modules - if hasattr(sys, 'gettotalrefcount'): + if (cross_compiling and 'd' in get_config_var('ABIFLAGS') or + (not cross_compiling and hasattr(sys, "gettotalrefcount"))): plat_specifier += '-pydebug' # 'build_purelib' and 'build_platlib' just default to 'lib' and diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index 38bb8fd93c2781b..a027e5149413c95 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -237,7 +237,11 @@ def finalize_options(self): self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: # building python standard extensions - self.library_dirs.append('.') + if 'PYTHON_PROJECT_BASE' in os.environ: + self.library_dirs.append(os.path.realpath( + os.environ['PYTHON_PROJECT_BASE'])) + else: + self.library_dirs.append('.') # The argument parsing will result in self.define being a string, but # it has to be a list of 2-tuples. All the preprocessor symbols @@ -732,7 +736,7 @@ def get_libraries(self, ext): link_libpython = True elif sys.platform == 'cygwin': link_libpython = True - elif '_PYTHON_HOST_PLATFORM' in os.environ: + elif 'PYTHON_PROJECT_BASE' in os.environ: # We are cross-compiling for one of the relevant platforms if get_config_var('ANDROID_API_LEVEL') != 0: link_libpython = True diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index 6c30a9eadff830c..d14e16b6a6da874 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -445,9 +445,7 @@ def finalize_other(self): self.select_scheme("unix_home") else: if self.prefix is None: - prefix = (get_config_vars().get('prefix') if - cross_compiling else sys.prefix) - self.prefix = os.path.normpath(prefix) + self.prefix = os.path.normpath(sys.prefix) self.install_base = self.install_platbase = self.prefix try: diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index be3b86a9ea41d4a..dc2216a3abba9ff 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -13,11 +13,10 @@ import os import re import sys +import pathlib -from sysconfig import (cross_compiling, get_cross_build_var, - get_build_time_vars) +from sysconfig import cross_compiling, get_project_base, get_build_time_vars from .errors import DistutilsPlatformError -from .util import get_platform, get_host_platform # These are needed in a couple of spots, so just compute them once. PREFIX = os.path.normpath(sys.prefix) @@ -25,13 +24,32 @@ BASE_PREFIX = os.path.normpath(sys.base_prefix) BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) +def fix_cross_build_paths(project_base): + build_time_vars = get_build_time_vars() + pbase = pathlib.Path(project_base) + prefix = build_time_vars['prefix'] + pfx = pathlib.Path(prefix) + try: + # A staged install that is the result of 'make install' + # with DESTDIR set. + destdir = pbase.parents[len(pfx.parts) - 2] + staged_install = (destdir.joinpath(pfx.relative_to('/')) == pbase) + except (ValueError, IndexError): + staged_install = False + + exec_prefix = build_time_vars['exec_prefix'] + if staged_install: + prefix = project_base + epfx = pathlib.Path(exec_prefix) + exec_prefix = str(destdir.joinpath(epfx.relative_to('/'))) + return prefix, exec_prefix + # Path to the base directory of the project. On Windows the binary may # live in project/PCbuild/win32 or project/PCbuild/amd64. # set for cross builds if cross_compiling: - project_base = get_cross_build_var('project_base') - PREFIX = get_cross_build_var('prefix') - EXEC_PREFIX = get_cross_build_var('exec-prefix') + project_base = get_project_base() + PREFIX, EXEC_PREFIX = fix_cross_build_paths(project_base) BASE_PREFIX = PREFIX BASE_EXEC_PREFIX = EXEC_PREFIX else: @@ -52,7 +70,7 @@ def _is_python_source_dir(d): return True return False -_sys_home = getattr(sys, '_home', None) +_sys_home = getattr(sys, '_home', None) if not cross_compiling else None if os.name == 'nt': def _fix_pcbuild(d): @@ -75,14 +93,16 @@ def _python_build(): # to the include and lib directories only makes sense for an installation, not # an in-source build. build_flags = '' -try: - if not python_build: - build_flags = (get_cross_build_var('abiflags') if cross_compiling else - sys.abiflags) -except AttributeError: - # It's not a configure-based build, so the sys module doesn't have - # this attribute, which is fine. - pass +if not python_build: + if cross_compiling: + build_flags = get_build_time_vars()['ABIFLAGS'] + else: + try: + build_flags = sys.abiflags + except AttributeError: + # It's not a configure-based build, so the sys module doesn't have + # this attribute, which is fine. + pass def get_python_version(): """Return a string containing the major and minor Python version, @@ -263,7 +283,7 @@ def get_makefile_filename(): return os.path.join(_sys_home or project_base, "Makefile") lib_dir = get_python_lib(plat_specific=0, standard_lib=1) config_file = 'config-{}{}'.format(get_python_version(), build_flags) - multiarch = (get_cross_build_var('multiarch') if cross_compiling else + multiarch = (get_config_var('MULTIARCH') if cross_compiling else getattr(sys.implementation, '_multiarch', '')) if multiarch: config_file += '-%s' % multiarch @@ -488,6 +508,8 @@ def get_config_vars(*args): # Distutils. _config_vars['prefix'] = PREFIX _config_vars['exec_prefix'] = EXEC_PREFIX + if cross_compiling: + _config_vars['LIBDIR'] = os.path.join(EXEC_PREFIX, 'lib') # For backward compatibility, see issue19555 SO = _config_vars.get('EXT_SUFFIX') @@ -531,9 +553,6 @@ def get_config_vars(*args): import _osx_support _osx_support.customize_config_vars(_config_vars) - if cross_compiling: - _config_vars['LIBDIR'] = get_cross_build_var('libdir') - if args: vals = [] for name in args: diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index 601076bb0433f13..14d52cbc86e41ac 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -14,7 +14,7 @@ from distutils.spawn import spawn from distutils import log from distutils.errors import DistutilsByteCompileError -from sysconfig import cross_compiling, get_cross_build_var +from distutils.sysconfig import cross_compiling, get_config_var def get_host_platform(): """Return a string that identifies the current platform. This is used mainly to @@ -47,7 +47,7 @@ def get_host_platform(): # Set for cross builds explicitly if cross_compiling: - return get_cross_build_var('host_platform') + return get_config_var('PYTHON_HOST_PLATFORM') if os.name != "posix" or not hasattr(os, 'uname'): # XXX what about the architecture? NT is Intel or Alpha, diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 5cc73feb5e69abf..c61a0cdd98b83b3 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -114,66 +114,64 @@ def _safe_realpath(path): _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir)) +cross_compiling = False +# set for cross builds +if "PYTHON_PROJECT_BASE" in os.environ: + cross_compiling = True + _PROJECT_BASE = _safe_realpath(os.environ["PYTHON_PROJECT_BASE"]) + + # Not used when cross compiling. + _PREFIX = None + _BASE_PREFIX = None + _EXEC_PREFIX = None + _BASE_EXEC_PREFIX = None + +def get_project_base(): + return _PROJECT_BASE + def _is_python_source_dir(d): for fn in ("Setup", "Setup.local"): if os.path.isfile(os.path.join(d, "Modules", fn)): return True return False -_cross_build_vars = None -def get_cross_build_var(name): - """Return the value of the 'name' variable.""" - - global _cross_build_vars - - if _cross_build_vars is None: - _cross_build_vars = {} - if 'CROSS_BUILD_PYTHON_DIR' in os.environ: - # The import would fail when building a native interpreter since, - # at the time generate-posix-vars is run, the _posixsubprocess - # module has not been built yet. This does not happen when - # generate-posix-vars is run during cross-compilation as the - # native interpreter being used is a full-fledged interpreter. - import subprocess - - pyconf_dir = _safe_realpath(os.environ['CROSS_BUILD_PYTHON_DIR']) - if _is_python_source_dir(pyconf_dir): - python_build = True - python_config = os.path.join(pyconf_dir, 'python-config') - else: - python_build = False - python_config = os.path.join(pyconf_dir, 'bin', - 'python3-config') - - vars_ = ['host_platform', 'version', 'abiflags', 'machdep', - 'multiarch', 'prefix', 'exec-prefix', 'libdir'] - args = ['sh', python_config] - args.extend('--' + v for v in vars_) - output = subprocess.check_output(args, universal_newlines=True) - _cross_build_vars = dict(zip(vars_, output.split('\n'))) - assert len(_cross_build_vars) == len(vars_) - d = _cross_build_vars - d['project_base'] = (pyconf_dir if - python_build else d['prefix']) - - # The specification of the sysconfigdata file name may differ - # across versions, forbid Python versions mismatch. - sys_version = '%d.%d' % sys.version_info[:2] - if d['version'] != sys_version: - raise RuntimeError('the running python version (%s) does ' - 'not match the cross-compiled version (%s)' % - (sys_version, d['version'])) - - return _cross_build_vars.get(name) +_PYTHON_CONFIG = None +def _get_python_config(name): + """Return the value of a python configuration variable.""" -cross_compiling = False -# set for cross builds -_cross_build_project_base = get_cross_build_var('project_base') -if _cross_build_project_base is not None: - _PROJECT_BASE = _cross_build_project_base - cross_compiling = True + assert cross_compiling + global _PYTHON_CONFIG + if _PYTHON_CONFIG is None: + # The import would fail when building a native interpreter since, + # at the time generate-posix-vars is run, the _posixsubprocess + # module has not been built yet. This does not happen when + # generate-posix-vars is run during cross-compilation as the + # native interpreter being used is a full-fledged interpreter. + import subprocess -_sys_home = getattr(sys, '_home', None) + if _is_python_source_dir(_PROJECT_BASE): + python_config = os.path.join(_PROJECT_BASE, 'python-config') + else: + python_config = os.path.join(_PROJECT_BASE, 'bin', + 'python3-config') + + vars_ = ['version', 'abiflags', 'machdep', 'multiarch'] + args = ['/bin/sh', python_config] + args.extend('--' + v for v in vars_) + output = subprocess.check_output(args, universal_newlines=True) + _PYTHON_CONFIG = dict(zip(vars_, output.split('\n'))) + assert len(_PYTHON_CONFIG) == len(vars_) + + # The specification of the sysconfigdata file name may differ + # across versions, so forbid Python versions mismatch. + sys_version = '%d.%d' % sys.version_info[:2] + if _PYTHON_CONFIG['version'] != sys_version: + raise RuntimeError('the running python version (%s) does ' + 'not match the cross-compiled version (%s)' % + (sys_version, _PYTHON_CONFIG['version'])) + return _PYTHON_CONFIG.get(name) + +_sys_home = getattr(sys, '_home', None) if not cross_compiling else None if os.name == 'nt': def _fix_pcbuild(d): @@ -383,6 +381,8 @@ def get_makefile_filename(): """Return the path of the Makefile.""" if _PYTHON_BUILD: return os.path.join(_sys_home or _PROJECT_BASE, "Makefile") + + assert not cross_compiling if hasattr(sys, 'abiflags'): config_dir_name = 'config-%s%s' % (_PY_VERSION_SHORT, sys.abiflags) else: @@ -394,9 +394,9 @@ def get_makefile_filename(): def _get_sysconfigdata_name(): if cross_compiling: - abiflags = get_cross_build_var('abiflags') - platform = get_cross_build_var('machdep') - multiarch = get_cross_build_var('multiarch') + abiflags = _get_python_config('abiflags') + platform = _get_python_config('machdep') + multiarch = _get_python_config('multiarch') else: abiflags = sys.abiflags platform = sys.platform @@ -447,15 +447,17 @@ def _generate_posix_vars(): # _sysconfigdata module manually and populate it with the build vars. # This is more than sufficient for ensuring the subsequent call to # get_platform() succeeds. + # The same situation exists when cross compiling. name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: + if 'darwin' in sys.platform or cross_compiling: import types module = types.ModuleType(name) module.build_time_vars = vars sys.modules[name] = module pybuilddir = 'build/lib.%s-%s' % (get_platform(), _PY_VERSION_SHORT) - if hasattr(sys, "gettotalrefcount"): + if (cross_compiling and 'd' in vars['ABIFLAGS'] or + (not cross_compiling and hasattr(sys, "gettotalrefcount"))): pybuilddir += '-pydebug' os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') @@ -470,27 +472,35 @@ def _generate_posix_vars(): with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) +def _get_module_attr(module, attribute): + _temp = __import__(module, globals(), locals(), [attribute], 0) + return getattr(_temp, attribute) + +_BUILD_TIME_VARS = None def get_build_time_vars(): - # _sysconfigdata is generated at build time, see _generate_posix_vars() - name = _get_sysconfigdata_name() - if cross_compiling: - if _is_python_source_dir(_PROJECT_BASE): - bdir = os.path.join(_PROJECT_BASE, 'pybuilddir.txt') - with open(bdir, encoding='ascii') as f: - libdir = f.read().strip() - libdir = os.path.join(_PROJECT_BASE, libdir) + global _BUILD_TIME_VARS + if _BUILD_TIME_VARS is None: + # _sysconfigdata is generated at build time, see _generate_posix_vars() + sysconfigdata = _get_sysconfigdata_name() + if cross_compiling: + if _is_python_source_dir(_PROJECT_BASE): + bdir = os.path.join(_PROJECT_BASE, 'pybuilddir.txt') + with open(bdir, encoding='ascii') as f: + libdir = f.read().strip() + libdir = os.path.join(_PROJECT_BASE, libdir) + else: + libdir = os.path.join(_PROJECT_BASE, + 'lib', 'python%s' % _get_python_config('version')) + sys.path.insert(0, libdir) + try: + _BUILD_TIME_VARS = _get_module_attr(sysconfigdata, + 'build_time_vars') + finally: + sys.path.pop(0) else: - libdir = os.path.join(get_cross_build_var('prefix'), - 'lib', 'python%s' % get_cross_build_var('version')) - sysconf = os.path.join(libdir, name + '.py') - with open(sysconf) as f: - code = f.read() - locals_ = {} - exec(code, globals(), locals_) - return locals_['build_time_vars'] - else: - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - return _temp.build_time_vars + _BUILD_TIME_VARS = _get_module_attr(sysconfigdata, + 'build_time_vars') + return _BUILD_TIME_VARS def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" @@ -697,6 +707,9 @@ def get_platform(): For other non-POSIX platforms, currently just returns 'sys.platform'. """ + if cross_compiling: + return get_config_var('PYTHON_HOST_PLATFORM') + if os.name == 'nt': if 'amd64' in sys.version.lower(): return 'win-amd64' diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index eab82c3631fd222..932d297b47de300 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -286,9 +286,7 @@ def format_groups(groups): "VIRTUAL_ENV", "WAYLAND_DISPLAY", "WINDIR", - "_PYTHON_HOST_PLATFORM", - "_PYTHON_PROJECT_BASE", - "_PYTHON_SYSCONFIGDATA_NAME", + "PYTHON_PROJECT_BASE", "__PYVENV_LAUNCHER__", )) for name, value in os.environ.items(): diff --git a/Makefile.pre.in b/Makefile.pre.in index 109808095935d43..15afaa2923b7ea5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -575,6 +575,8 @@ platform: $(BUILDPYTHON) pybuilddir.txt # problems by creating a dummy pybuilddir.txt just to allow interpreter # initialization to succeed. It will be overwritten by generate-posix-vars # or removed in case of failure. +# python-config is a prerequisite as it is used when generating the posix +# variables if cross-compiling. pybuilddir.txt: python-config $(BUILDPYTHON) @echo "none" > ./pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\ @@ -1455,7 +1457,7 @@ libinstall: build_all $(srcdir)/Modules/xxmodule.c done sysconf_name=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP); \ if test -n "$(MULTIARCH)"; then \ - sysconf_name=$$sysconf_name_$(MULTIARCH); \ + sysconf_name=$${sysconf_name}_$(MULTIARCH); \ fi; \ $(INSTALL_DATA) `cat pybuilddir.txt`/$$sysconf_name.py \ $(DESTDIR)$(LIBDEST) @@ -1615,7 +1617,7 @@ sharedinstall: sharedmods --root=$(DESTDIR)/ -sysconf_name=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP); \ if test -n "$(MULTIARCH)"; then \ - sysconf_name=$$sysconf_name_$(MULTIARCH); \ + sysconf_name=$${sysconf_name}_$(MULTIARCH); \ fi; \ rm $(DESTDIR)$(DESTSHARED)/$$sysconf_name.py -rm -r $(DESTDIR)$(DESTSHARED)/__pycache__ diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 33bcda02c5ce0d4..6493931852785ed 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -5,13 +5,8 @@ exit_with_usage () { echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|\ -<<<<<<< HEAD --ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed|\ - --host_platform|--version|--machdep|--multiarch" -======= - --ldflags|--extension-suffix|--help|--abiflags|--configdir|\ - --host_platform|--version|--machdep|--multiarch|--libdir" ->>>>>>> Use 'libdir' to build extension modules from an installed python build + --version|--machdep|--multiarch" exit $1 } @@ -75,7 +70,7 @@ do ;; --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|\ --extension-suffix|--abiflags|--configdir|\ - --host_platform|--version|--machdep|--multiarch|--libdir) + --version|--machdep|--multiarch) ;; *) exit_with_usage 1 @@ -121,9 +116,6 @@ do --configdir) echo "$LIBPL" ;; - --host_platform) - echo "@_PYTHON_HOST_PLATFORM@" - ;; --version) echo "$VERSION" ;; @@ -133,8 +125,5 @@ do --multiarch) echo "@MULTIARCH@" ;; - --libdir) - echo "$libdir" - ;; esac done diff --git a/configure b/configure index 2e550fa81677429..db0bedf5e8e54d2 100755 --- a/configure +++ b/configure @@ -2947,7 +2947,7 @@ $as_echo_n "checking for python interpreter for cross build... " >&6; } fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $interp" >&5 $as_echo "$interp" >&6; } - PYTHON_FOR_BUILD='CROSS_BUILD_PYTHON_DIR=$(abs_builddir) '$interp + PYTHON_FOR_BUILD='PYTHON_PROJECT_BASE=$(abs_builddir) '$interp fi elif test "$cross_compiling" = maybe; then as_fn_error $? "Cross compiling required --host=HOST-TUPLE and --build=ARCH" "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 3d0d6255d17f057..24d900ce64ff5e7 100644 --- a/configure.ac +++ b/configure.ac @@ -75,7 +75,7 @@ if test "$cross_compiling" = yes; then AC_MSG_ERROR([python$PACKAGE_VERSION interpreter not found]) fi AC_MSG_RESULT($interp) - PYTHON_FOR_BUILD='CROSS_BUILD_PYTHON_DIR=$(abs_builddir) '$interp + PYTHON_FOR_BUILD='PYTHON_PROJECT_BASE=$(abs_builddir) '$interp fi elif test "$cross_compiling" = maybe; then AC_MSG_ERROR([Cross compiling required --host=HOST-TUPLE and --build=ARCH]) diff --git a/setup.py b/setup.py index 6e447435be8307f..654fd1ed0819f79 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ import sysconfig from glob import glob -from sysconfig import CROSS_COMPILING +from sysconfig import cross_compiling as CROSS_COMPILING from distutils import log from distutils.command.build_ext import build_ext from distutils.command.build_scripts import build_scripts @@ -30,7 +30,8 @@ def get_platform(): if CROSS_COMPILING: - return sysconfig.get_cross_build_var('host_platform') + return sysconfig.get_config_var('PYTHON_HOST_PLATFORM') + # Get value of sys.platform if sys.platform.startswith('osf1'): return 'osf1' From b31c6390c3e79fb0bd3edba84b4ed301d21deab0 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Sat, 30 Nov 2019 14:55:03 +0100 Subject: [PATCH 07/12] Raise RuntimeError cross-building Python extensions when PYTHON_PROJECT_BASE not a build directory --- Lib/sysconfig.py | 3 ++- Makefile.pre.in | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index c61a0cdd98b83b3..2bd77d399b791d7 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -193,6 +193,8 @@ def is_python_build(check_home=False): for scheme in ('posix_prefix', 'posix_home'): _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include' _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.' +elif cross_compiling: + raise RuntimeError('PYTHON_PROJECT_BASE is not a build directory') def _subst_vars(s, local_vars): @@ -382,7 +384,6 @@ def get_makefile_filename(): if _PYTHON_BUILD: return os.path.join(_sys_home or _PROJECT_BASE, "Makefile") - assert not cross_compiling if hasattr(sys, 'abiflags'): config_dir_name = 'config-%s%s' % (_PY_VERSION_SHORT, sys.abiflags) else: diff --git a/Makefile.pre.in b/Makefile.pre.in index 15afaa2923b7ea5..928c0c8f26fe873 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -246,7 +246,9 @@ BUILDPYTHON= python$(BUILDEXE) PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@ UPDATE_FILE=@PYTHON_FOR_REGEN@ $(srcdir)/Tools/scripts/update_file.py PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ +# To enable setup.py and sysconfig to pick up this AC_SUBST() var. PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ + BUILD_GNU_TYPE= @build@ HOST_GNU_TYPE= @host@ From 9ba355903503d4ee4082743f8ee77fa1ce3a32dc Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Sat, 30 Nov 2019 16:02:41 +0100 Subject: [PATCH 08/12] Remove support for PYTHON_PROJECT_BASE as a staged install directory --- Lib/distutils/sysconfig.py | 47 +++++++++++--------------------------- Lib/sysconfig.py | 1 - 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index dc2216a3abba9ff..afc42b40c6b89d3 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -13,7 +13,6 @@ import os import re import sys -import pathlib from sysconfig import cross_compiling, get_project_base, get_build_time_vars from .errors import DistutilsPlatformError @@ -24,33 +23,15 @@ BASE_PREFIX = os.path.normpath(sys.base_prefix) BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) -def fix_cross_build_paths(project_base): - build_time_vars = get_build_time_vars() - pbase = pathlib.Path(project_base) - prefix = build_time_vars['prefix'] - pfx = pathlib.Path(prefix) - try: - # A staged install that is the result of 'make install' - # with DESTDIR set. - destdir = pbase.parents[len(pfx.parts) - 2] - staged_install = (destdir.joinpath(pfx.relative_to('/')) == pbase) - except (ValueError, IndexError): - staged_install = False - - exec_prefix = build_time_vars['exec_prefix'] - if staged_install: - prefix = project_base - epfx = pathlib.Path(exec_prefix) - exec_prefix = str(destdir.joinpath(epfx.relative_to('/'))) - return prefix, exec_prefix - # Path to the base directory of the project. On Windows the binary may # live in project/PCbuild/win32 or project/PCbuild/amd64. # set for cross builds if cross_compiling: project_base = get_project_base() - PREFIX, EXEC_PREFIX = fix_cross_build_paths(project_base) + build_time_vars = get_build_time_vars() + PREFIX = build_time_vars['prefix'] BASE_PREFIX = PREFIX + EXEC_PREFIX = build_time_vars['exec_prefix'] BASE_EXEC_PREFIX = EXEC_PREFIX else: if sys.executable: @@ -88,21 +69,21 @@ def _python_build(): python_build = _python_build() +if cross_compiling and not python_build: + raise RuntimeError('PYTHON_PROJECT_BASE is not a build directory') + # Calculate the build qualifier flags if they are defined. Adding the flags # to the include and lib directories only makes sense for an installation, not # an in-source build. build_flags = '' -if not python_build: - if cross_compiling: - build_flags = get_build_time_vars()['ABIFLAGS'] - else: - try: - build_flags = sys.abiflags - except AttributeError: - # It's not a configure-based build, so the sys module doesn't have - # this attribute, which is fine. - pass +try: + if not python_build: + build_flags = sys.abiflags +except AttributeError: + # It's not a configure-based build, so the sys module doesn't have + # this attribute, which is fine. + pass def get_python_version(): """Return a string containing the major and minor Python version, @@ -508,8 +489,6 @@ def get_config_vars(*args): # Distutils. _config_vars['prefix'] = PREFIX _config_vars['exec_prefix'] = EXEC_PREFIX - if cross_compiling: - _config_vars['LIBDIR'] = os.path.join(EXEC_PREFIX, 'lib') # For backward compatibility, see issue19555 SO = _config_vars.get('EXT_SUFFIX') diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 2bd77d399b791d7..2b96c954c400739 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -383,7 +383,6 @@ def get_makefile_filename(): """Return the path of the Makefile.""" if _PYTHON_BUILD: return os.path.join(_sys_home or _PROJECT_BASE, "Makefile") - if hasattr(sys, 'abiflags'): config_dir_name = 'config-%s%s' % (_PY_VERSION_SHORT, sys.abiflags) else: From e8a9b1eda651129d91c7408b9ba8a0143de18362 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Sat, 30 Nov 2019 21:54:02 +0100 Subject: [PATCH 09/12] Add a NEWS entry --- .../next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst diff --git a/Misc/NEWS.d/next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst b/Misc/NEWS.d/next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst new file mode 100644 index 000000000000000..6729c75fcb29e87 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst @@ -0,0 +1,9 @@ +Fix cross-compilation of third-party extension modules. The +``PYTHON_PROJECT_BASE`` environment variable is the path to the directory +where Python has been cross-compiled. It is used by the native python +interpreter to find the target ``sysconfigdata`` module. For example the +following command builds a wheel file to be transfered and installed with +pip on the target platform, provided the native python interpreter and the +cross-compiled one both have the wheel package installed:: + + $ PYTHON_PROJECT_BASE=/path/to/builddir python setup.py bdist_wheel From 2ef971aff561348ea5c6b5cba543015c7c8960db Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Sun, 1 Dec 2019 16:59:42 +0100 Subject: [PATCH 10/12] Remove dead code after support for building with staged install directory has been removed --- Lib/sysconfig.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 2b96c954c400739..a40c8c4010dcbea 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -149,12 +149,7 @@ def _get_python_config(name): # native interpreter being used is a full-fledged interpreter. import subprocess - if _is_python_source_dir(_PROJECT_BASE): - python_config = os.path.join(_PROJECT_BASE, 'python-config') - else: - python_config = os.path.join(_PROJECT_BASE, 'bin', - 'python3-config') - + python_config = os.path.join(_PROJECT_BASE, 'python-config') vars_ = ['version', 'abiflags', 'machdep', 'multiarch'] args = ['/bin/sh', python_config] args.extend('--' + v for v in vars_) From 121a998856127e9fb7b6f0b707d09ca01c25ef1d Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Sun, 1 Dec 2019 18:11:54 +0100 Subject: [PATCH 11/12] Fix sphinx syntax error in NEWS entry --- .../next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst b/Misc/NEWS.d/next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst index 6729c75fcb29e87..c9d5fd21fd31505 100644 --- a/Misc/NEWS.d/next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst +++ b/Misc/NEWS.d/next/Build/2019-11-30-21-53-38.bpo-28833.14IeXH.rst @@ -4,6 +4,8 @@ where Python has been cross-compiled. It is used by the native python interpreter to find the target ``sysconfigdata`` module. For example the following command builds a wheel file to be transfered and installed with pip on the target platform, provided the native python interpreter and the -cross-compiled one both have the wheel package installed:: +cross-compiled one both have the wheel package installed: + +.. code-block:: shell $ PYTHON_PROJECT_BASE=/path/to/builddir python setup.py bdist_wheel From bf64b6b17a52b14f3a88f61cd21a39acebfe0dd0 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Sun, 1 Dec 2019 20:33:35 +0100 Subject: [PATCH 12/12] Add an entry in What's New in Python 3.9 --- Doc/whatsnew/3.9.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 1cd96ef3b07af30..a6e53b3d199d46e 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -367,3 +367,21 @@ CPython bytecode changes :keyword:`assert` statement. Previously, the assert statement would not work correctly if the :exc:`AssertionError` exception was being shadowed. (Contributed by Zackery Spytz in :issue:`34880`.) + +Build Changes +------------- + +* Cross compilation of third-party extension modules is done with the + ``PYTHON_PROJECT_BASE`` environment variable set to the path of the + directory where Python has been cross-compiled. The private environment + variables ``_PYTHON_HOST_PLATFORM``, ``_PYTHON_SYSCONFIGDATA_NAME`` and + ``_PYTHON_PROJECT_BASE`` are not used anymore. + + For example the following command builds a wheel file to be transfered and + installed with pip on the target platform, provided the native python + interpreter and the cross-compiled one both have the wheel package + installed: + + .. code-block:: shell + + $ PYTHON_PROJECT_BASE=/path/to/builddir python setup.py bdist_wheel