From 4bcdd1eb9fa5ff2b7de8e00cd580c3330021f807 Mon Sep 17 00:00:00 2001 From: Peter Odding Date: Sun, 31 Aug 2014 02:19:36 +0200 Subject: [PATCH] Command line interface for static checks (with improved test coverage) --- deb_pkg_tools/__init__.py | 4 +- deb_pkg_tools/checks.py | 93 ++++++++++++++++++++++++++------------- deb_pkg_tools/cli.py | 30 +++++++++---- deb_pkg_tools/tests.py | 58 +++++++++++++++++------- 4 files changed, 128 insertions(+), 57 deletions(-) diff --git a/deb_pkg_tools/__init__.py b/deb_pkg_tools/__init__.py index 44e1a91..938ae81 100644 --- a/deb_pkg_tools/__init__.py +++ b/deb_pkg_tools/__init__.py @@ -1,11 +1,11 @@ # Debian packaging tools. # # Author: Peter Odding -# Last Change: August 30, 2014 +# Last Change: August 31, 2014 # URL: https://github.com/xolox/python-deb-pkg-tools # Semi-standard module versioning. -__version__ = '1.26.4' +__version__ = '1.27' debian_package_dependencies = ( 'apt', # apt-get diff --git a/deb_pkg_tools/checks.py b/deb_pkg_tools/checks.py index 36614b8..bfd0e4b 100644 --- a/deb_pkg_tools/checks.py +++ b/deb_pkg_tools/checks.py @@ -1,7 +1,7 @@ # Debian packaging tools: Static analysis of package archives. # # Author: Peter Odding -# Last Change: August 30, 2014 +# Last Change: August 31, 2014 # URL: https://github.com/xolox/python-deb-pkg-tools """ @@ -15,7 +15,7 @@ import logging # External dependencies. -from humanfriendly import format_path, pluralize, Spinner +from humanfriendly import format_path, pluralize, Spinner, Timer # Modules included in our package. from deb_pkg_tools.package import collect_related_packages, inspect_package, parse_filename @@ -24,7 +24,39 @@ # Initialize a logger. logger = logging.getLogger(__name__) -def check_duplicate_files(package_archives, cache=None): +def check_package(archive, cache=None): + """ + Perform static checks on a package's dependency set. + + :param archive: The pathname of an existing ``*.deb`` archive (a string). + :param cache: The :py:class:`.PackageCache` to use (defaults to ``None``). + :raises: :py:class:`BrokenPackage` when one or more checks failed. + """ + timer = Timer() + logger.info("Checking %s ..", format_path(archive)) + dependency_set = collect_related_packages(archive, cache=cache) + failed_checks = [] + # Check for duplicate files in the dependency set. + try: + check_duplicate_files(dependency_set, cache=cache) + except BrokenPackage as e: + failed_checks.append(e) + except ValueError: + # Silenced. + pass + # Check for version conflicts in the dependency set. + try: + check_version_conflicts(dependency_set, cache=cache) + except BrokenPackage as e: + failed_checks.append(e) + if len(failed_checks) == 1: + raise failed_checks[0] + elif failed_checks: + raise BrokenPackage('\n\n'.join(map(str, failed_checks))) + else: + logger.info("Finished checking in %s, no problems found.", timer) + +def check_duplicate_files(dependency_set, cache=None): """ Check a collection of Debian package archives for conflicts. @@ -36,7 +68,7 @@ def check_duplicate_files(package_archives, cache=None): pathnames of files installed by packages it can be slow. To make it faster you can use the :py:class:`.PackageCache`. - :param package_archives: A list of filenames (strings) of ``*.deb`` files. + :param dependency_set: A list of filenames (strings) of ``*.deb`` files. :param cache: The :py:class:`.PackageCache` to use (defaults to ``None``). :raises: :py:class:`exceptions.ValueError` when less than two package archives are given (the duplicate check obviously only works if @@ -44,9 +76,10 @@ def check_duplicate_files(package_archives, cache=None): :raises: :py:class:`DuplicateFilesFound` when duplicate files are found within a group of package archives. """ - package_archives = list(map(parse_filename, package_archives)) + timer = Timer() + dependency_set = list(map(parse_filename, dependency_set)) # Make sure we have something useful to work with. - num_archives = len(package_archives) + num_archives = len(dependency_set) if num_archives < 2: msg = "To check for duplicate files you need to provide two or more packages archives! (%i given)" raise ValueError(msg % num_archives) @@ -54,8 +87,8 @@ def check_duplicate_files(package_archives, cache=None): global_contents = collections.defaultdict(set) global_fields = {} spinner = Spinner(total=num_archives) - logger.info("Scanning %i package archives for duplicate files (this can take a while) ..", num_archives) - for i, archive in enumerate(optimize_order(package_archives), start=1): + logger.info("Checking for duplicate files in %i package archives ..", num_archives) + for i, archive in enumerate(optimize_order(dependency_set), start=1): spinner.step(label="Scanning %i package archives" % num_archives, progress=i) fields, contents = inspect_package(archive.filename, cache=cache) global_fields[archive.filename] = fields @@ -122,43 +155,41 @@ def find_virtual_name(field_name): delimiter = '%s\n' % ('-' * 79) raise DuplicateFilesFound(delimiter.join(summary)) else: - logger.info("No conflicting files found in %i package(s).", len(package_archives)) + logger.info("No conflicting files found (took %s).", timer) -def check_version_conflicts(package_archives, cache=None): +def check_version_conflicts(dependency_set, cache=None): """ - Check for version conflicts in one or more dependency sets. + Check for version conflicts in a dependency set. - For each Debian binary package archive given, find the dependencies of the - package and check ik newer versions of dependencies exist in the same - repository (directory). - - This analysis can be very slow. To make it faster you can use the + For each Debian binary package archive given, check if a newer version of + the same package exists in the same repository (directory). This analysis + can be very slow. To make it faster you can use the :py:class:`.PackageCache`. - :param package_archives: A list of filenames (strings) of ``*.deb`` files. + :param dependency_set: A list of filenames (strings) of ``*.deb`` files. :param cache: The :py:class:`.PackageCache` to use (defaults to ``None``). :raises: :py:class:`VersionConflictFound` when one or more version conflicts are found. """ + timer = Timer() summary = [] - num_package_archives = 0 - for archive in sorted(map(parse_filename, package_archives)): - logger.info("Checking dependencies of %s for version conflicts ..", format_path(archive.filename)) - for related_archive in collect_related_packages(archive.filename, cache=cache): - if related_archive.newer_versions: - summary.append(compact(""" - The dependency set of {package} includes {dependency} - but newer version(s) of that package also exist and - will take precedence: - """, package=format_path(archive.filename), - dependency=format_path(related_archive.filename))) - summary.append("\n".join(" - %s" % format_path(a.filename) for a in sorted(related_archive.newer_versions))) - num_package_archives += 1 + dependency_set = list(map(parse_filename, dependency_set)) + spinner = Spinner(total=len(dependency_set)) + logger.info("Checking for version conflicts in %i package(s) ..", len(dependency_set)) + for i, archive in enumerate(dependency_set, start=1): + if archive.newer_versions: + summary.append(compact(""" + Dependency set includes {dependency} but newer version(s) + of that package also exist and will take precedence: + """, dependency=format_path(archive.filename))) + summary.append("\n".join(" - %s" % format_path(a.filename) for a in sorted(archive.newer_versions))) + spinner.step(label="Checking for version conflicts", progress=i) + spinner.clear() if summary: summary.insert(0, "One or more version conflicts found:") raise VersionConflictFound('\n\n'.join(summary)) else: - logger.info("No version conflicts found in %i package(s).", num_package_archives) + logger.info("No version conflicts found (took %s).", timer) class BrokenPackage(Exception): """ diff --git a/deb_pkg_tools/cli.py b/deb_pkg_tools/cli.py index 06a646f..500a0af 100644 --- a/deb_pkg_tools/cli.py +++ b/deb_pkg_tools/cli.py @@ -1,7 +1,7 @@ # Debian packaging tools: Command line interface # # Author: Peter Odding -# Last Change: June 25, 2014 +# Last Change: August 31, 2014 # URL: https://github.com/xolox/python-deb-pkg-tools """ @@ -13,6 +13,9 @@ -c, --collect=DIR copy the package archive(s) given as positional arguments and all packages archives required by the given package archives into a directory + -C, --check=FILE perform static analysis on a package archive and + its dependencies in order to recognize common + errors as soon as possible -p, --patch=FILE patch fields into an existing control file -s, --set=LINE a line to patch into the control file (syntax: "Name: Value") @@ -26,6 +29,7 @@ activate the repository, run the positional arguments as an external command (usually `apt-get install') and finally deactivate the repository + -y, --yes assume the answer to interactive questions is yes -v, --verbose make more noise -h, --help show this message and exit """ @@ -45,6 +49,7 @@ # Modules included in our package. from deb_pkg_tools.cache import get_default_cache +from deb_pkg_tools.checks import check_package from deb_pkg_tools.control import patch_control_file from deb_pkg_tools.package import (build_package, collect_related_packages, inspect_package, parse_filename) @@ -72,6 +77,7 @@ def main(): sys.stdout = codecs.getwriter(OUTPUT_ENCODING)(sys.stdout) sys.stderr = codecs.getwriter(OUTPUT_ENCODING)(sys.stderr) # Command line option defaults. + prompt = True actions = [] control_file = None control_fields = {} @@ -79,10 +85,10 @@ def main(): cache = get_default_cache() # Parse the command line options. try: - options, arguments = getopt.getopt(sys.argv[1:], 'i:c:p:s:b:u:a:d:w:vh', [ - 'inspect=', 'collect=', 'patch=', 'set=', 'build=', 'update-repo=', - 'activate-repo=', 'deactivate-repo=', 'with-repo=', 'verbose', - 'help' + options, arguments = getopt.getopt(sys.argv[1:], 'i:c:C:p:s:b:u:a:d:w:yvh', [ + 'inspect=', 'collect=', 'check=', 'patch=', 'set=', 'build=', + 'update-repo=', 'activate-repo=', 'deactivate-repo=', 'with-repo=', + 'yes', 'verbose', 'help' ]) for option, value in options: if option in ('-i', '--inspect'): @@ -91,8 +97,11 @@ def main(): actions.append(functools.partial(collect_packages, archives=arguments, directory=check_directory(value), + prompt=prompt, cache=cache)) arguments = [] + elif option in ('-C', '--check'): + actions.append(functools.partial(check_package, archive=value, cache=cache)) elif option in ('-p', '--patch'): control_file = os.path.abspath(value) assert os.path.isfile(control_file), "Control file does not exist!" @@ -114,6 +123,8 @@ def main(): directory=check_directory(value), command=arguments, cache=cache)) + elif option in ('-y', '--yes'): + prompt = False elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-h', '--help'): @@ -162,7 +173,7 @@ def show_package_metadata(archive): group=entry.group, size=size, modified=entry.modified, pathname=pathname)) -def collect_packages(archives, directory, cache): +def collect_packages(archives, directory, prompt=True, cache=None): # Find all related packages. related_archives = set() for filename in archives: @@ -182,9 +193,10 @@ def collect_packages(archives, directory, cache): for file_to_collect in relevant_archives: print(" - %s" % format_path(file_to_collect.filename)) try: - # Ask permission to copy the file(s). - prompt = "Copy %s to %s? [Y/n] " % (pluralized, format_path(directory)) - assert raw_input(prompt).lower() in ('', 'y', 'yes') + if prompt: + # Ask permission to copy the file(s). + prompt = "Copy %s to %s? [Y/n] " % (pluralized, format_path(directory)) + assert raw_input(prompt).lower() in ('', 'y', 'yes') # Copy the file(s). for file_to_collect in relevant_archives: copy_from = file_to_collect.filename diff --git a/deb_pkg_tools/tests.py b/deb_pkg_tools/tests.py index a829cca..0f6336d 100644 --- a/deb_pkg_tools/tests.py +++ b/deb_pkg_tools/tests.py @@ -1,7 +1,7 @@ # Debian packaging tools: Automated tests. # # Author: Peter Odding -# Last Change: August 30, 2014 +# Last Change: August 31, 2014 # URL: https://github.com/xolox/python-deb-pkg-tools # Standard library modules. @@ -366,13 +366,42 @@ def test_command_line_interface(self): directory = finalizers.mkdtemp() # Test `deb-pkg-tools --inspect PKG'. package_file = self.test_package_building(directory) - lines = call('--inspect', package_file).splitlines() + lines = call('--verbose', '--inspect', package_file).splitlines() for field, value in TEST_PACKAGE_FIELDS.items(): self.assertEqual(match('^ - %s: (.+)$' % field, lines), value) # Test `deb-pkg-tools --with-repo=DIR CMD' (we simply check whether # apt-cache sees the package). if os.getuid() == 0: call('--with-repo=%s' % directory, 'apt-cache show %s' % TEST_PACKAGE_NAME) + # Test `deb-pkg-tools --update=DIR' with a non-existing directory. + self.assertRaises(SystemExit, call, '--update', '/a/directory/that/will/never/exist') + + def test_check_package(self): + with Context() as finalizers: + directory = finalizers.mkdtemp() + root_package, conflicting_package = self.create_version_conflict(directory) + self.assertRaises(SystemExit, call, '--check', root_package) + # Test for lack of duplicate files. + os.unlink(conflicting_package) + call('--check', root_package) + + def test_version_conflicts_check(self): + with Context() as finalizers: + # Check that version conflicts raise an exception. + directory = finalizers.mkdtemp() + root_package, conflicting_package = self.create_version_conflict(directory) + packages_to_scan = collect_related_packages(root_package) + # Test the duplicate files check. + self.assertRaises(VersionConflictFound, check_version_conflicts, packages_to_scan, self.package_cache) + # Test for lack of duplicate files. + os.unlink(conflicting_package) + self.assertEqual(check_version_conflicts(packages_to_scan, cache=self.package_cache), None) + + def create_version_conflict(self, directory): + root_package = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-1', Depends='deb-pkg-tools-package-2 (=1)')) + self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-2', Version='1')) + conflicting_package = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-2', Version='2')) + return root_package, conflicting_package def test_duplicates_check(self): with Context() as finalizers: @@ -409,20 +438,17 @@ def test_duplicates_check(self): # Verify that invalid arguments are checked. self.assertRaises(ValueError, check_duplicate_files, []) - def test_version_conflicts_check(self): + def test_collect_packages(self): with Context() as finalizers: - # Check that version conflicts raise an exception. - directory = finalizers.mkdtemp() - package1 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-1', Depends='deb-pkg-tools-package-2 (=1)')) - package2 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-2', Version='1')) - package3 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-2', Version='2')) - # Test the duplicate files check. - self.assertRaises(VersionConflictFound, check_version_conflicts, [package1], self.package_cache) - # Test for lack of duplicate files. - os.unlink(package3) - self.assertEqual(check_version_conflicts([package1]), None) + source_directory = finalizers.mkdtemp() + target_directory = finalizers.mkdtemp() + package1 = self.test_package_building(source_directory, overrides=dict(Package='deb-pkg-tools-package-1', Depends='deb-pkg-tools-package-2')) + package2 = self.test_package_building(source_directory, overrides=dict(Package='deb-pkg-tools-package-2', Depends='deb-pkg-tools-package-3')) + package3 = self.test_package_building(source_directory, overrides=dict(Package='deb-pkg-tools-package-3')) + call('--yes', '--collect=%s' % target_directory, package1) + self.assertEqual(sorted(os.listdir(target_directory)), sorted(map(os.path.basename, [package1, package2, package3]))) - def test_collect_packages(self): + def test_collect_packages_interactive(self): with Context() as finalizers: directory = finalizers.mkdtemp() package1 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-1', Depends='deb-pkg-tools-package-2')) @@ -431,6 +457,7 @@ def test_collect_packages(self): package4 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-3', Version='0.2')) self.assertEqual(sorted(p.filename for p in collect_related_packages(package1, cache=self.package_cache)), [package2, package4]) + def test_repository_creation(self, preserve=False): if not SKIP_SLOW_TESTS: with Context() as finalizers: @@ -497,7 +524,8 @@ def test_gpg_key_generation(self): self.assertRaises(Exception, GPGKey, public_key_file=public_key_file) missing_secret_key_file = '/tmp/deb-pkg-tools-%i.sec' % random.randint(1, 1000) missing_public_key_file = '/tmp/deb-pkg-tools-%i.pub' % random.randint(1, 1000) - self.assertRaises(Exception, GPGKey, key_id='12345', secret_key_file=missing_secret_key_file, public_key_file=missing_public_key_file) + self.assertRaises(Exception, GPGKey, key_id='12345', secret_key_file=secret_key_file, public_key_file=missing_public_key_file) + self.assertRaises(Exception, GPGKey, key_id='12345', secret_key_file=missing_secret_key_file, public_key_file=public_key_file) os.unlink(secret_key_file) self.assertRaises(Exception, GPGKey, name="test-key", description="Whatever", secret_key_file=secret_key_file, public_key_file=public_key_file) touch(secret_key_file)