Use a recursive iglob for filesets (canonical#765)

This makes use of `**` in filesets for parts.

LP: #1616464

Signed-off-by: Sergio Schvezov <sergio.schvezov@ubuntu.com>

integration_tests/test_python_plugin.py

@@ -83,6 +83,24 @@ def test_build_rewrites_shebangs(self):
         self.assertEqual('#!/usr/bin/env python2', python2_shebang)
         self.assertEqual('#!/usr/bin/env python3', python3_shebang)
 
+    def test_build_does_not_keep_pyc_or_pth_files_in_install(self):
+        # .pyc and .pyc files collide between parts.
+        # There is no way to tell pip or setup.py to disable generation of
+        # .pyc
+        # The .pth files are only relevant if found inside the pre compiled
+        # site-packges directory so we don't want those either.
+        project_dir = 'pip-requirements-file'
+        self.run_snapcraft('stage', project_dir)
+
+        pyc_files = []
+        pth_files = []
+        for _, _, files in os.walk(os.path.join(project_dir, 'stage')):
+            pyc_files.extend([f for f in files if f.endswith('pyc')])
+            pth_files.extend([f for f in files if f.endswith('pth')])
+
+        self.assertEqual([], pyc_files)
+        self.assertEqual([], pth_files)
+
     def test_build_doesnt_get_bad_install_directory_lp1586546(self):
         """Verify that LP: #1586546 doesn't come back."""
         project_dir = 'python-pyyaml'

snapcraft/internal/pluginhandler.py

@@ -16,13 +16,13 @@
 
 import contextlib
 import filecmp
-import glob
 import importlib
 import itertools
 import logging
 import os
 import shutil
 import sys
+from glob import iglob
 
 import jsonschema
 import magic
@@ -756,7 +756,8 @@ def _generate_include_set(directory, includes):
     include_files = set()
     for include in includes:
         if '*' in include:
-            matches = glob.glob(os.path.join(directory, include))
+            pattern = os.path.join(directory, include)
+            matches = iglob(pattern, recursive=True)
             include_files |= set(matches)
         else:
             include_files |= set([os.path.join(directory, include), ])
@@ -782,7 +783,8 @@ def _generate_exclude_set(directory, excludes):
     exclude_files = set()
 
     for exclude in excludes:
-        matches = glob.glob(os.path.join(directory, exclude))
+        pattern = os.path.join(directory, exclude)
+        matches = iglob(pattern, recursive=True)
         exclude_files |= set(matches)
 
     exclude_dirs = [os.path.relpath(x, directory)

snapcraft/plugins/autotools.py

@@ -124,11 +124,8 @@ def build(self):
         self.run(['make', '-j{}'.format(self.parallel_build_count)])
         self.run(make_install_command)
 
+    def snap_fileset(self):
+        fileset = super().snap_fileset()
         # Remove .la files which don't work when they are moved around
-        self._remove_la_files()
-
-    def _remove_la_files(self):
-        for root, _, files in os.walk(self.installdir):
-            for file_name in files:
-                if file_name.endswith('.la'):
-                    os.unlink(os.path.join(root, file_name))
+        fileset.append('-**/*.la')
+        return fileset

snapcraft/plugins/python2.py

@@ -206,18 +206,11 @@ def build(self):
     def snap_fileset(self):
         fileset = super().snap_fileset()
         fileset.append('-usr/bin/pip*')
-        fileset.append('-usr/lib/python*/dist-packages/easy-install.pth')
-        fileset.append('-usr/lib/python*/dist-packages/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*.pyc')
-        fileset.append('-usr/lib/python*/*/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*/*/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*/*/*/*.pyc')
+        # .pth files are only read from the built-in site-packages directory.
+        # We use PYTHONPATH for everything so not needed.
+        fileset.append('-**/*.pth')
+        # This is a major cause of inter part conflict.
+        fileset.append('-**/*.pyc')
         return fileset
 
 

snapcraft/plugins/python3.py

@@ -184,14 +184,10 @@ def python_version(self):
     def snap_fileset(self):
         fileset = super().snap_fileset()
         fileset.append('-usr/bin/pip*')
-        fileset.append('-usr/lib/python*/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/*/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*/*/__pycache__/*.pyc')
-        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*/*/*/__pycache__/*.pyc')
+        # .pth files are only read from the built-in site-packages directory.
+        # We use PYTHONPATH for everything so not needed.
+        fileset.append('-**/*.pth')
+        # Holds all the .pyc files. It is a major cause of inter part
+        # conflict.
+        fileset.append('-**/__pycache__')
         return fileset

snapcraft/tests/test_plugin_autotools.py

@@ -308,3 +308,12 @@ def _run(cmd):
         # An exception will be raised if build can't handle the non-executable
         # autogen.
         plugin.build()
+
+    def test_fileset_ignores(self):
+        plugin = autotools.AutotoolsPlugin('test-part', self.options,
+                                           self.project_options)
+        expected_fileset = [
+            '-**/*.la',
+        ]
+        fileset = plugin.snap_fileset()
+        self.assertListEqual(expected_fileset, fileset)

snapcraft/tests/test_plugin_python2.py

@@ -180,18 +180,8 @@ def test_fileset_ignores(self):
                                        self.project_options)
         expected_fileset = [
             '-usr/bin/pip*',
-            '-usr/lib/python*/dist-packages/easy-install.pth',
-            '-usr/lib/python*/dist-packages/__pycache__/*.pyc',
-            '-usr/lib/python*/*.pyc',
-            '-usr/lib/python*/*/*.pyc',
-            '-usr/lib/python*/*/*/*.pyc',
-            '-usr/lib/python*/*/*/*/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/*/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/*/*/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/*/*/*/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/*/*/*/*/*.pyc',
+            '-**/*.pth',
+            '-**/*.pyc',
         ]
         fileset = plugin.snap_fileset()
         self.assertListEqual(expected_fileset, fileset)

snapcraft/tests/test_plugin_python3.py

@@ -130,16 +130,8 @@ def test_fileset_ignores(self):
                                        self.project_options)
         expected_fileset = [
             '-usr/bin/pip*',
-            '-usr/lib/python*/__pycache__/*.pyc',
-            '-usr/lib/python*/*/__pycache__/*.pyc',
-            '-usr/lib/python*/*/*/__pycache__/*.pyc',
-            '-usr/lib/python*/*/*/*/__pycache__/*.pyc',
-            '-usr/lib/python*/*/*/*/*/__pycache__/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/__pycache__/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/*/__pycache__/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/*/*/__pycache__/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/*/*/*/__pycache__/*.pyc',
-            '-usr/lib/python*/*/*/*/*/*/*/*/*/*/__pycache__/*.pyc',
+            '-**/*.pth',
+            '-**/__pycache__',
         ]
         fileset = plugin.snap_fileset()
         self.assertListEqual(expected_fileset, fileset)