python plugin: do the right thing with classic.

integration_tests/snaps/assemble/binary1.after

@@ -1,5 +1,4 @@
 #!/bin/sh
 export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH"
-
-LD_LIBRARY_PATH=$SNAP_LIBRARY_PATH:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$SNAP_LIBRARY_PATH:$LD_LIBRARY_PATH
 exec "$SNAP/binary1" "$@"

integration_tests/snaps/python-entry-point/snapcraft.yaml

@@ -22,3 +22,5 @@ parts:
         - -lib/python3.5/site-packages/pip*
         - -lib/python3.5/site-packages/setuptools*
         - -lib/python3.5/site-packages/wheel*
+        # This is needed as the same package is used twice.
+        - -lib/python3.5/site-packages/test-0.1.dist-info/RECORD

integration_tests/test_python_plugin.py

@@ -80,7 +80,7 @@ def test_build_rewrites_shebangs(self):
         with open(python_entry_point) as f:
             python_shebang = f.readline().strip()
 
-        self.assertEqual('#!/usr/bin/env python', python2_shebang)
+        self.assertEqual('#!/usr/bin/env python2', python2_shebang)
         self.assertEqual('#!/usr/bin/env python3', python3_shebang)
         self.assertEqual('#!/usr/bin/env python3', python_shebang)
 

snapcraft/__init__.py

@@ -304,6 +304,7 @@
 from snapcraft import plugins                       # noqa
 from snapcraft import sources                       # noqa
 from snapcraft import file_utils                    # noqa
+from snapcraft import shell_utils                   # noqa
 from snapcraft.internal import repo                 # noqa
 
 

snapcraft/_baseplugin.py

@@ -73,13 +73,21 @@ def PLUGIN_STAGE_SOURCES(self):
         """Define alternative sources.list."""
         return getattr(self, '_PLUGIN_STAGE_SOURCES', [])
 
+    @property
+    def stage_packages(self):
+        return self._stage_packages
+
+    @stage_packages.setter
+    def stage_packages(self, value):
+        self._stage_packages = value
+
     def __init__(self, name, options, project=None):
         self.name = name
         self.build_packages = []
-        self.stage_packages = []
+        self._stage_packages = []
 
         with contextlib.suppress(AttributeError):
-            self.stage_packages = options.stage_packages.copy()
+            self._stage_packages = options.stage_packages.copy()
         with contextlib.suppress(AttributeError):
             self.build_packages = options.build_packages.copy()
 

snapcraft/internal/common.py

@@ -47,18 +47,8 @@
 logger = logging.getLogger(__name__)
 
 
-def assemble_env(include_core_library_paths=False, arch_triplet=''):
-    if include_core_library_paths:
-        core_paths = get_library_paths(
-            os.path.join('/snap', 'core', 'current'), arch_triplet,
-            existing_only=False)
-        core_library_paths = 'LD_LIBRARY_PATH="{}"'.format(
-            ':'.join(core_paths))
-        parse_env = [core_library_paths] + env
-    else:
-        parse_env = env
-
-    return '\n'.join(['export ' + e for e in parse_env])
+def assemble_env():
+    return '\n'.join(['export ' + e for e in env])
 
 
 def run(cmd, **kwargs):

snapcraft/internal/meta.py

@@ -28,6 +28,7 @@
 import yaml
 
 from snapcraft import file_utils
+from snapcraft import shell_utils
 from snapcraft.internal import common, project_loader
 from snapcraft.internal.errors import MissingGadgetError
 from snapcraft.internal.deprecations import handle_deprecation_notice
@@ -253,33 +254,41 @@ def _write_wrap_exe(self, wrapexec, wrappath,
         cwd = 'cd {}'.format(cwd) if cwd else ''
 
         # If we are dealing with classic confinement it means all our
-        # binaries are linked with `nodefaultlib` so this is harmless.
-        # We do however want to be on the safe side and make sure no
-        # ABI breakage happens by accidentally loading a library from
-        # the classic system.
-        classic_library_paths = self._config_data['confinement'] == 'classic'
-        assembled_env = common.assemble_env(classic_library_paths,
-                                            self._arch_triplet)
-        assembled_env = assembled_env.replace(self._snap_dir, '$SNAP')
-        replace_path = r'{}/[a-z0-9][a-z0-9+-]*/install'.format(
-            self._parts_dir)
-        assembled_env = re.sub(replace_path, '$SNAP', assembled_env)
+        # binaries are linked with `nodefaultlib` but we still do
+        # not want to leak PATH or other environment variables
+        # that would affect the applications view of the classic
+        # environment it is dropped into.
+        replace_path = re.compile(r'{}/[a-z0-9][a-z0-9+-]*/install'.format(
+            re.escape(self._parts_dir)))
+        if self._config_data['confinement'] == 'classic':
+            assembled_env = None
+        else:
+            assembled_env = common.assemble_env()
+            assembled_env = assembled_env.replace(self._snap_dir, '$SNAP')
+            assembled_env = replace_path.sub('$SNAP', assembled_env)
+
         executable = '"{}"'.format(wrapexec)
-        if shebang is not None:
-            new_shebang = re.sub(replace_path, '$SNAP', shebang)
+
+        if shebang:
+            if shebang.startswith('/usr/bin/env '):
+                shebang = shell_utils.which(shebang.split()[1])
+            new_shebang = replace_path.sub('$SNAP', shebang)
+            new_shebang = re.sub(self._snap_dir, '$SNAP', new_shebang)
             if new_shebang != shebang:
                 # If the shebang was pointing to and executable within the
                 # local 'parts' dir, have the wrapper script execute it
                 # directly, since we can't use $SNAP in the shebang itself.
                 executable = '"{}" "{}"'.format(new_shebang, wrapexec)
-        script = ('#!/bin/sh\n' +
-                  '{}\n'.format(assembled_env) +
-                  '{}\n'.format(cwd) +
-                  'LD_LIBRARY_PATH=$SNAP_LIBRARY_PATH:$LD_LIBRARY_PATH\n'
-                  'exec {} {}\n'.format(executable, args))
 
         with open(wrappath, 'w+') as f:
-            f.write(script)
+            print('#!/bin/sh', file=f)
+            if assembled_env:
+                print('{}'.format(assembled_env), file=f)
+                print('export LD_LIBRARY_PATH=$SNAP_LIBRARY_PATH:'
+                      '$LD_LIBRARY_PATH', file=f)
+            if cwd:
+                print('{}'.format(cwd), file=f)
+            print('exec {} {}'.format(executable, args), file=f)
 
         os.chmod(wrappath, 0o755)
 

snapcraft/plugins/python.py

@@ -43,6 +43,12 @@
     - python-version:
       (string; default: python3)
       The python version to use. Valid options are: python2 and python3
+
+If the plugin finds a python interpreter with a basename that matches
+`python-version` in the <stage> directory on the following fixed path:
+`<stage-dir>/usr/bin/<python-interpreter>` then this interpreter would
+be preferred instead and no interpreter would be brought in through
+`stage-packages` mechanisms.
 """
 
 import os
@@ -61,6 +67,23 @@
 from snapcraft.common import isurl
 
 
+_SITECUSTOMIZE_TEMPLATE = dedent("""\
+    import site
+    import os
+
+    snap_dir = os.getenv("SNAP")
+    snapcraft_stage_dir = os.getenv("SNAPCRAFT_STAGE")
+    snapcraft_part_install = os.getenv("SNAPCRAFT_PART_INSTALL")
+
+    for d in (snap_dir, snapcraft_stage_dir, snapcraft_part_install):
+        if d:
+            site_dir = os.path.join(d, "{site_dir}")
+            site.addsitedir(site_dir)
+
+    if snap_dir:
+        site.ENABLE_USER_SITE = False""")
+
+
 class PythonPlugin(snapcraft.BasePlugin):
 
     @classmethod
@@ -130,24 +153,17 @@ def plugin_stage_packages(self):
             return ['python']
 
     @property
-    def system_pip_command(self):
-        if self.options.python_version == 'python3':
-            return os.path.join(os.path.sep, 'usr', 'bin', 'pip3')
+    def stage_packages(self):
+        if not os.path.exists(self._get_python_command()):
+            return super().stage_packages + self.plugin_stage_packages
         else:
-            return os.path.join(os.path.sep, 'usr', 'bin', 'pip')
+            return super().stage_packages
 
     def __init__(self, name, options, project):
         super().__init__(name, options, project)
         self.build_packages.extend(self.plugin_build_packages)
-        self.stage_packages.extend(self.plugin_stage_packages)
         self._python_package_dir = os.path.join(self.partdir, 'packages')
 
-    def env(self, root):
-        return [
-            'PYTHONUSERBASE={}'.format(root),
-            'PYTHONHOME={}'.format(os.path.join(root, 'usr'))
-        ]
-
     def pull(self):
         super().pull()
 
@@ -163,24 +179,59 @@ def clean_pull(self):
         if os.path.isdir(self._python_package_dir):
             shutil.rmtree(self._python_package_dir)
 
+    def _get_python_command(self):
+        python_command = os.path.join(
+            'usr', 'bin', self.options.python_version)
+
+        # staged as in project.stage_dir, not from a stage-packages entry
+        # in this part.
+        staged_python = os.path.join(self.project.stage_dir, python_command)
+        unstaged_python = os.path.join(self.installdir, python_command)
+
+        if os.path.exists(staged_python):
+            return staged_python
+        else:
+            return unstaged_python
+
     def _install_pip(self, download):
         env = os.environ.copy()
         env['PYTHONUSERBASE'] = self.installdir
+        # since we are using an independent env we need to export this too
+        # TODO: figure out if we can move back to common.run
+        env['SNAPCRAFT_STAGE'] = self.project.stage_dir
+        env['SNAPCRAFT_PART_INSTALL'] = self.installdir
 
         args = ['pip', 'setuptools', 'wheel']
 
+        pip_command = [self._get_python_command(), '-m', 'pip']
+
+        # If python_command it is not from stage we don't have pip, which means
+        # we are going to need to resort to the pip installed on the system
+        # that came from build-packages. This shouldn't be a problem as
+        # stage-packages and build-packages should match.
+        if not self._get_python_command().startswith(self.project.stage_dir):
+            env['PYTHONHOME'] = '/usr'
+
         pip = _Pip(exec_func=subprocess.check_call,
-                   runnable=self.system_pip_command,
+                   runnable=pip_command,
                    package_dir=self._python_package_dir, env=env,
                    extra_install_args=['--ignore-installed'])
 
         if download:
             pip.download(args)
-        pip.wheel(args)
         pip.install(args)
 
     def _get_build_env(self):
         env = os.environ.copy()
+        env['PYTHONUSERBASE'] = self.installdir
+        if self._get_python_command().startswith(self.project.stage_dir):
+            env['PYTHONHOME'] = os.path.join(self.project.stage_dir, 'usr')
+        else:
+            env['PYTHONHOME'] = os.path.join(self.installdir, 'usr')
+
+        env['PATH'] = '{}:{}'.format(
+            os.path.join(self.installdir, 'usr', 'bin'),
+            os.path.expandvars('$PATH'))
         headers = glob(os.path.join(
             os.path.sep, 'usr', 'include', '{}*'.format(
                 self.options.python_version)))
@@ -217,6 +268,8 @@ def _run_pip(self, setup, download=False):
 
         env = self._get_build_env()
 
+        pip_command = [self._get_python_command(), '-m', 'pip']
+
         constraints = []
         if self.options.constraints:
             if isurl(self.options.constraints):
@@ -225,7 +278,8 @@ def _run_pip(self, setup, download=False):
                 constraints = os.path.join(self.sourcedir,
                                            self.options.constraints)
 
-        pip = _Pip(exec_func=self.run, runnable='pip',
+        pip = _Pip(exec_func=self.run,
+                   runnable=pip_command,
                    package_dir=self._python_package_dir, env=env,
                    constraints=constraints,
                    dependency_links=self.options.process_dependency_links)
@@ -267,6 +321,44 @@ def build(self):
                                    re.compile(r'^#!.*python'),
                                    r'#!/usr/bin/env python')
 
+        self._setup_sitecustomize()
+
+    def _setup_sitecustomize(self):
+        # This avoids needing to leak PYTHONUSERBASE
+        # USER_SITE and USER_BASE default to base of SNAP for when used in
+        # runtime and to SNAPCRAFT_STAGE to support chaining dependencies
+        # when used with the `after` keyword.
+        site_dir = self._get_user_site_dir()
+        sitecustomize_path = self._get_sitecustomize_path()
+
+        # Now create our sitecustomize
+        os.makedirs(os.path.dirname(sitecustomize_path), exist_ok=True)
+        with open(sitecustomize_path, 'w') as f:
+            f.write(_SITECUSTOMIZE_TEMPLATE.format(site_dir=site_dir))
+
+    def _get_user_site_dir(self):
+        user_site_dir = glob(os.path.join(
+            self.installdir, 'lib', '{}*'.format(self.options.python_version),
+            'site-packages'))[0]
+
+        return user_site_dir[len(self.installdir)+1:]
+
+    def _get_sitecustomize_path(self):
+        if self._get_python_command().startswith(self.project.stage_dir):
+            base_dir = self.project.stage_dir
+        else:
+            base_dir = self.installdir
+
+        python_site = glob(os.path.join(
+            base_dir, 'usr', 'lib',
+            '{}*'.format(self.options.python_version),
+            'site.py'))[0]
+        python_site_dir = os.path.dirname(python_site)
+
+        return os.path.join(self.installdir,
+                            python_site_dir[len(base_dir)+1:],
+                            'sitecustomize.py')
+
     def snap_fileset(self):
         fileset = super().snap_fileset()
         fileset.append('-bin/pip*')
@@ -285,7 +377,10 @@ def __init__(self, *, exec_func, runnable, package_dir, env,
                  constraints=None, dependency_links=None,
                  extra_install_args=None):
         self._exec_func = exec_func
-        self._runnable = runnable
+        if isinstance(runnable, str):
+            self._runnable = [runnable]
+        else:
+            self._runnable = runnable
         self._package_dir = package_dir
         self._env = env
 
@@ -302,7 +397,7 @@ def list(self, exec_func=None):
         """Return a dict of installed python packages with versions."""
         if not exec_func:
             exec_func = self._exec_func
-        cmd = [self._runnable, 'list']
+        cmd = [*self._runnable, 'list']
 
         output = exec_func(cmd, env=self._env)
         package_listing = {}
@@ -316,7 +411,7 @@ def list(self, exec_func=None):
 
     def wheel(self, args, **kwargs):
         cmd = [
-            self._runnable, 'wheel',
+            *self._runnable, 'wheel',
             '--disable-pip-version-check', '--no-index',
             '--find-links', self._package_dir,
         ]
@@ -339,7 +434,7 @@ def wheel(self, args, **kwargs):
 
     def download(self, args, **kwargs):
         cmd = [
-            self._runnable, 'download',
+            *self._runnable, 'download',
             '--disable-pip-version-check',
             '--dest', self._package_dir,
         ]
@@ -351,7 +446,7 @@ def download(self, args, **kwargs):
 
     def install(self, args, **kwargs):
         cmd = [
-            self._runnable, 'install', '--user', '--no-compile',
+            *self._runnable, 'install', '--user', '--no-compile',
             '--disable-pip-version-check', '--no-index',
             '--find-links', self._package_dir,
         ]