python module: Use sys_root's Python sysconfigdata to get EXT_SUFFIX/SO

Until now, the build host's EXT_SUFFIX/SO was used, resulting in native
extensions being build and installed with the wrong filename.

To do this, we load the sys_root sysconfigdata from its stdlib
directory. We assume that the same Python version as the build host
exists within the sys_root with the same stdlib directory, save for the
prefix. If not, we fall back to the build host, as we did before.

If the machine file points python at a Python within sys_root then this
should still work. You should probably only run a sys_root's binary
using some kind of wrapper, but either way, sysconfig will still return
the correct values.

We only read EXT_SUFFIX/SO from the sys_root because paths are expected
to be the same, save for the prefix, and other info such as the platform
and limited API suffix cannot be determined this way.

We could potentially guess the limited API suffix from the platform, as
there are only a small number of possible values, but this can be done
separately.

Closes: mesonbuild#7049

mesonbuild/dependencies/python.py

@@ -33,6 +33,7 @@
     from .factory import DependencyGenerator
     from ..environment import Environment
     from ..mesonlib import MachineChoice
+    from ..modules import ModuleState
 
     class PythonIntrospectionDict(TypedDict):
 
@@ -108,14 +109,25 @@ def _check_version(self, version: str) -> bool:
             return mesonlib.version_compare(version, '>= 3.0')
         return True
 
-    def sanity(self) -> bool:
+    def sanity(self, state: T.Optional['ModuleState'] = None) -> bool:
         # Sanity check, we expect to have something that at least quacks in tune
 
         import importlib.resources
 
+        # Pass the sys_root with its prefix as the first arg to python_info.py
+        # if a sys_root is defined. Otherwise pass an empty string.
+        sys_root_prefix = ''
+        if state:
+            sys_root = state.environment.properties[mesonlib.MachineChoice.HOST].get_sys_root()
+            prefix = state.environment.coredata.get_option(mesonlib.OptionKey('prefix'))
+            assert isinstance(prefix, str), 'for mypy'
+            if sys_root:
+                sys_root_prefix = sys_root + prefix
+
         with importlib.resources.path('mesonbuild.scripts', 'python_info.py') as f:
             cmd = self.get_command() + [str(f)]
             env = os.environ.copy()
+            env['MESON_SYS_ROOT_PREFIX'] = sys_root_prefix
             env['SETUPTOOLS_USE_DISTUTILS'] = 'stdlib'
             p, stdout, stderr = mesonlib.Popen_safe(cmd, env=env)
 

mesonbuild/modules/python.py

@@ -80,7 +80,7 @@ class PythonExternalProgram(BasicPythonExternalProgram):
     run_bytecompile: T.ClassVar[T.Dict[str, bool]] = {}
 
     def sanity(self, state: T.Optional['ModuleState'] = None) -> bool:
-        ret = super().sanity()
+        ret = super().sanity(state)
         if ret:
             self.platlib = self._get_path(state, 'platlib')
             self.purelib = self._get_path(state, 'purelib')

mesonbuild/scripts/python_info.py

@@ -80,17 +80,54 @@ def links_against_libpython():
 
 variables = sysconfig.get_config_vars()
 variables.update({'base_prefix': getattr(sys, 'base_prefix', sys.prefix)})
+sys_root_variables = variables
 
 is_pypy = '__pypy__' in sys.builtin_module_names
 
+# Determine some information from the sys_root if it is given in the
+# environment. When not given, it is expected to be an empty string.
+sys_root_prefix = os.environ.get('MESON_SYS_ROOT_PREFIX')
+if sys_root_prefix:
+    import glob
+    try:
+        # We need to load the sys_root sysconfigdata from its stdlib directory.
+        # We assume that the same Python version as the build host exists within
+        # the sys_root with the same stdlib directory, save for the prefix. If
+        # not, we fall back to the build host.
+        stdlib = sysconfig.get_path('stdlib', vars={
+            'base': sys_root_prefix,
+            'installed_base': sys_root_prefix
+        })
+
+        # We cannot predict exactly what the sys_root sysconfigdata filename
+        # will be, but there should only be one, so glob to find it.
+        sysconfigdatas = glob.glob(os.path.join(stdlib, '_sysconfigdata*.py'))
+
+        # Only use the sys_root sysconfigdata if we find exactly one. More than
+        # one is odd, so it is safer to fall back to the build host.
+        if len(sysconfigdatas) == 1:
+            with open(sysconfigdatas[0]) as f:
+                code = compile(f.read(), sysconfigdatas[0], 'exec')
+            data = {}
+            exec(code, data)
+            if 'build_time_vars' in data:
+                sys_root_variables = data['build_time_vars']
+    except OSError:
+        pass
+
+# Only read the suffix from the sys_root. Paths should be the same, save for the
+# prefix, and other info such as the platform cannot be determined this way.
 if sys.version_info < (3, 0):
-    suffix = variables.get('SO')
-elif sys.version_info < (3, 8, 7):
-    # https://bugs.python.org/issue?@action=redirect&bpo=39825
+    suffix = sys_root_variables.get('SO')
+# sysconfig's EXT_SUFFIX was wrong on Windows before 3.8.7 so use distutils in
+# that case. Don't do this when checking the sys_root though as you cannot use
+# distutils from there. Windows doesn't have the sysconfigdata file anyway.
+# https://bugs.python.org/issue?@action=redirect&bpo=39825
+elif variables is sys_root_variables and sys.version_info < (3, 8, 7):
     from distutils.sysconfig import get_config_var
     suffix = get_config_var('EXT_SUFFIX')
 else:
-    suffix = variables.get('EXT_SUFFIX')
+    suffix = sys_root_variables.get('EXT_SUFFIX')
 
 limited_api_suffix = None
 if sys.version_info >= (3, 2):