Use sitecustomize to emulate venv during python install (#512)

During initialization, the `site` module probes for a `sitecustomize`
module which performs site-specific initialization of various paths.

Python virtual environments modify sys.prefix in order to ensure that
any attempted package installation ends up using the virtual environment
instead of the system locations.

This approach should prevent various Linux distributions' attempts to
inject `local` into the installation locations for python packages. They
seem to have special cases for detecting virtual environments and
omit the custom logic when detected.

colcon_core/task/python/build.py

@@ -21,6 +21,7 @@
 from colcon_core.task import TaskExtensionPoint
 from colcon_core.task.python import get_data_files_mapping
 from colcon_core.task.python import get_setup_data
+from colcon_core.task.python.template import expand_template
 
 logger = colcon_logger.getChild(__name__)
 
@@ -46,14 +47,23 @@ async def build(self, *, additional_hooks=None):  # noqa: D102
             return 1
         setup_py_data = get_setup_data(self.context.pkg, env)
 
+        # override installation locations
+        prefix_override = Path(args.build_base) / 'prefix_override'
+        expand_template(
+            Path(__file__).parent / 'template' / 'sitecustomize.py.em',
+            prefix_override / 'sitecustomize.py',
+            {
+                'site_prefix': args.install_base,
+            })
+
         # `setup.py develop|install` requires the python lib path to exist
         python_lib = os.path.join(
             args.install_base, self._get_python_lib(args))
         os.makedirs(python_lib, exist_ok=True)
         # and being in the PYTHONPATH
         env = dict(env)
-        env['PYTHONPATH'] = python_lib + os.pathsep + \
-            env.get('PYTHONPATH', '')
+        env['PYTHONPATH'] = str(prefix_override) + os.pathsep + \
+            python_lib + os.pathsep + env.get('PYTHONPATH', '')
 
         # determine if setuptools specific commands are available
         available_commands = await self._get_available_commands(
@@ -79,7 +89,7 @@ async def build(self, *, additional_hooks=None):  # noqa: D102
             cmd += [
                 'build', '--build-base', os.path.join(
                     args.build_base, 'build'),
-                'install', '--prefix', args.install_base,
+                'install',
                 '--record', os.path.join(args.build_base, 'install.log')]
             if 'egg_info' in available_commands:
                 # prevent installation of dependencies specified in setup.py
@@ -102,14 +112,14 @@ async def build(self, *, additional_hooks=None):  # noqa: D102
                 # easy-install.pth file
                 cmd = [
                     executable, 'setup.py',
-                    'develop', '--prefix', args.install_base,
+                    'develop',
                     '--editable',
                     '--build-directory',
                     os.path.join(args.build_base, 'build'),
                     '--no-deps',
                 ]
                 if setup_py_data.get('data_files'):
-                    cmd += ['install_data', '--install-dir', args.install_base]
+                    cmd += ['install_data']
                 completed = await run(
                     self.context, cmd, cwd=args.build_base, env=env)
             finally:
@@ -163,7 +173,7 @@ async def _undo_develop(self, pkg, args, env):
         if os.path.exists(egg_info) and os.path.islink(setup_py_build_space):
             cmd = [
                 executable, 'setup.py',
-                'develop', '--prefix', args.install_base,
+                'develop',
                 '--uninstall', '--editable',
                 '--build-directory', os.path.join(args.build_base, 'build')
             ]

colcon_core/task/python/template/__init__.py

@@ -0,0 +1,4 @@
+# Copyright 2022 Open Source Robotics Foundation, Inc.
+# Licensed under the Apache License, Version 2.0
+
+from colcon_core.shell.template import expand_template  # noqa: F401

colcon_core/task/python/template/sitecustomize.py.em

@@ -0,0 +1,3 @@
+import sys
+sys.real_prefix = sys.prefix
+sys.prefix = sys.exec_prefix = @repr(site_prefix)

setup.cfg

@@ -145,6 +145,7 @@ pytest11 =
 
 [options.package_data]
 colcon_core.shell.template = *.em
+colcon_core.task.python.template = *.em
 
 [flake8]
 import-order-style = google

test/spell_check.words

@@ -101,6 +101,7 @@ setupscript
 setuptools
 shlex
 sigint
+sitecustomize
 sloretz
 stacklevel
 staticmethod