/snapcraft

Only discover dependencies once per file. #796

Merged
merged 2 commits into from Sep 14, 2016
Commits
Jump to file or symbol
Failed to load files and symbols.
+78 −29
Split
View
34 snapcraft/internal/pluginhandler.py
@@ -17,7 +17,6 @@
import contextlib
import filecmp
import importlib
-import itertools
import logging
import os
import shutil
@@ -375,7 +374,7 @@ def prime(self, force=False):
self.notify_part_progress('Priming')
snap_files, snap_dirs = self.migratable_fileset_for('snap')
_migrate_files(snap_files, snap_dirs, self.stagedir, self.snapdir)
- dependencies = _find_dependencies(self.snapdir)
+ dependencies = _find_dependencies(self.snapdir, snap_files)
# Split the necessary dependencies into their corresponding location.
# We'll both migrate and track the system dependencies, but we'll only
@@ -710,27 +709,30 @@ def _clean_migrated_files(snap_files, snap_dirs, directory):
os.rmdir(migrated_directory)
-def _find_dependencies(workdir):
+def _find_dependencies(root, part_files):
ms = magic.open(magic.NONE)
if ms.load() != 0:
raise RuntimeError('Cannot load magic header detection')
elf_files = set()
- fs_encoding = sys.getfilesystemencoding()
- for root, dirs, files in os.walk(workdir.encode(fs_encoding)):
+ for part_file in part_files:
# Filter out object (*.o) files-- we only care about binaries.
- entries = (entry for entry in itertools.chain(files, dirs)
- if not entry.endswith(b'.o'))
- for entry in entries:
- path = os.path.join(root, entry)
- if os.path.islink(path):
- logger.debug('Skipped link {!r} when parsing {!r}'.format(
- path, workdir))
- continue
- file_m = ms.file(path)
- if file_m.startswith('ELF') and 'dynamically linked' in file_m:
- elf_files.add(path)
+ if part_file.endswith('.o'):
+ continue
+
+ # No need to crawl links-- the original should be here, too.
+ path = os.path.join(root, part_file)
+ if os.path.islink(path):
+ logger.debug('Skipped link {!r} while finding dependencies'.format(
+ path))
+ continue
+
+ # Finally, make sure this is actually an ELF before queueing it up
+ # for an ldd call.
+ file_m = ms.file(path)
+ if file_m.startswith('ELF') and 'dynamically linked' in file_m:
+ elf_files.add(path)
dependencies = []
for elf_file in elf_files:
View
73 snapcraft/tests/test_pluginhandler.py
@@ -1176,7 +1176,8 @@ def test_prime_state(self, mock_copy, mock_find_dependencies):
self.handler.prime()
self.assertEqual('prime', self.handler.last_step())
- mock_find_dependencies.assert_called_once_with(self.handler.snapdir)
+ mock_find_dependencies.assert_called_once_with(self.handler.snapdir,
+ {'bin/1', 'bin/2'})
self.assertFalse(mock_copy.called)
state = self.handler.get_state('prime')
@@ -1197,6 +1198,49 @@ def test_prime_state(self, mock_copy, mock_find_dependencies):
self.assertTrue(type(state.project_options) is OrderedDict)
self.assertEqual(0, len(state.project_options))
+ @patch('snapcraft.internal.pluginhandler._find_dependencies')
+ @patch('shutil.copy')
+ def test_prime_state_with_stuff_already_primed(self, mock_copy,
+ mock_find_dependencies):
+ mock_find_dependencies.return_value = set()
+
+ self.assertEqual(None, self.handler.last_step())
+
+ bindir = os.path.join(self.handler.code.installdir, 'bin')
+ os.makedirs(bindir)
+ open(os.path.join(bindir, '1'), 'w').close()
+ bindir = os.path.join(self.handler.snapdir, 'bin')
+ os.makedirs(bindir)
+ open(os.path.join(bindir, '2'), 'w').close()
+
+ self.handler.mark_done('build')
+ self.handler.stage()
+ self.handler.prime()
+
+ self.assertEqual('prime', self.handler.last_step())
+ # bin/2 shouldn't be in this list as it was already primed by another
+ # part.
+ mock_find_dependencies.assert_called_once_with(self.handler.snapdir,
+ {'bin/1'})
+ self.assertFalse(mock_copy.called)
+
+ state = self.handler.get_state('prime')
+
+ self.assertTrue(type(state) is states.PrimeState)
+ self.assertTrue(type(state.files) is set)
+ self.assertTrue(type(state.directories) is set)
+ self.assertTrue(type(state.dependency_paths) is set)
+ self.assertTrue(type(state.properties) is OrderedDict)
+ self.assertEqual(1, len(state.files))
+ self.assertTrue('bin/1' in state.files)
+ self.assertEqual(1, len(state.directories))
+ self.assertTrue('bin' in state.directories)
+ self.assertEqual(0, len(state.dependency_paths))
+ self.assertTrue('snap' in state.properties)
+ self.assertEqual(state.properties['snap'], ['*'])
+ self.assertTrue(type(state.project_options) is OrderedDict)
+ self.assertEqual(0, len(state.project_options))
+
@patch('snapcraft.internal.pluginhandler._find_dependencies')
@patch('snapcraft.internal.pluginhandler._migrate_files')
def test_prime_state_with_dependencies(self, mock_migrate_files,
@@ -1219,7 +1263,8 @@ def test_prime_state_with_dependencies(self, mock_migrate_files,
self.handler.prime()
self.assertEqual('prime', self.handler.last_step())
- mock_find_dependencies.assert_called_once_with(self.handler.snapdir)
+ mock_find_dependencies.assert_called_once_with(
+ self.handler.snapdir, {'bin/1', 'bin/2'})
mock_migrate_files.assert_has_calls([
call({'bin/1', 'bin/2'}, {'bin'}, self.handler.stagedir,
self.handler.snapdir),
@@ -1267,7 +1312,8 @@ def test_prime_state_with_snap_keyword(self, mock_copy,
self.handler.prime()
self.assertEqual('prime', self.handler.last_step())
- mock_find_dependencies.assert_called_once_with(self.handler.snapdir)
+ mock_find_dependencies.assert_called_once_with(self.handler.snapdir,
+ {'bin/1'})
self.assertFalse(mock_copy.called)
state = self.handler.get_state('prime')
@@ -2139,9 +2185,9 @@ def test_find_dependencies(self, mock_dependencies, mock_magic):
mock_dependencies.return_value = ['/usr/lib/libDepends.so']
- dependencies = pluginhandler._find_dependencies(workdir)
+ dependencies = pluginhandler._find_dependencies(workdir, {'linked'})
- mock_ms.file.assert_called_once_with(bytes(linked_elf_path, 'utf-8'))
+ mock_ms.file.assert_called_once_with(linked_elf_path)
self.assertEqual(dependencies, {'/usr/lib/libDepends.so'})
@patch('magic.open')
@@ -2162,7 +2208,8 @@ def test_find_dependencies_skip_object_files(self, mock_dependencies,
mock_dependencies.return_value = ['/usr/lib/libDepends.so']
- dependencies = pluginhandler._find_dependencies(workdir)
+ dependencies = pluginhandler._find_dependencies(
+ workdir, {'object_file.o'})
self.assertFalse(mock_ms.file.called,
'Expected object file to be skipped')
@@ -2186,10 +2233,10 @@ def test_no_find_dependencies_of_non_dynamically_linked(
'statically linked, for GNU/Linux 2.6.32, '
'BuildID[sha1]=XYZ, stripped')
- dependencies = pluginhandler._find_dependencies(workdir)
+ dependencies = pluginhandler._find_dependencies(
+ workdir, {'statically-linked'})
- mock_ms.file.assert_called_once_with(
- bytes(statically_linked_elf_path, 'utf-8'))
+ mock_ms.file.assert_called_once_with(statically_linked_elf_path)
self.assertFalse(
mock_dependencies.called,
@@ -2212,9 +2259,9 @@ def test_no_find_dependencies_of_non_elf_files(
mock_ms.load.return_value = 0
mock_ms.file.return_value = 'JPEG image data, Exif standard: ...'
- dependencies = pluginhandler._find_dependencies(workdir)
+ dependencies = pluginhandler._find_dependencies(workdir, {'non-elf'})
- mock_ms.file.assert_called_once_with(bytes(non_elf_path, 'utf-8'))
+ mock_ms.file.assert_called_once_with(non_elf_path)
self.assertFalse(
mock_dependencies.called,
@@ -2238,7 +2285,7 @@ def test_no_find_dependencies_of_symlinks(
mock_magic.return_value = mock_ms
mock_ms.load.return_value = 0
- dependencies = pluginhandler._find_dependencies(workdir)
+ dependencies = pluginhandler._find_dependencies(workdir, {'symlinked'})
self.assertFalse(
mock_ms.file.called, 'magic is not needed for symlinks')
@@ -2256,7 +2303,7 @@ def test_fail_to_load_magic_raises_exception(self, mock_magic):
mock_magic.return_value.load.return_value = 1
with self.assertRaises(RuntimeError) as raised:
- pluginhandler._find_dependencies('.')
+ pluginhandler._find_dependencies('.', set())
self.assertEqual(
raised.exception.__str__(), 'Cannot load magic header detection')