Command line interface for static checks (with improved test coverage)

deb_pkg_tools/__init__.py

@@ -1,11 +1,11 @@
 # Debian packaging tools.
 #
 # Author: Peter Odding <peter@peterodding.com>
-# Last Change: August 30, 2014
+# Last Change: August 31, 2014
 # URL: https://github.com/xolox/python-deb-pkg-tools
 
 # Semi-standard module versioning.
-__version__ = '1.26.4'
+__version__ = '1.27'
 
 debian_package_dependencies = (
     'apt',        # apt-get

deb_pkg_tools/checks.py

@@ -1,7 +1,7 @@
 # Debian packaging tools: Static analysis of package archives.
 #
 # Author: Peter Odding <peter@peterodding.com>
-# Last Change: August 30, 2014
+# Last Change: August 31, 2014
 # URL: https://github.com/xolox/python-deb-pkg-tools
 
 """
@@ -15,7 +15,7 @@
 import logging
 
 # External dependencies.
-from humanfriendly import format_path, pluralize, Spinner
+from humanfriendly import format_path, pluralize, Spinner, Timer
 
 # Modules included in our package.
 from deb_pkg_tools.package import collect_related_packages, inspect_package, parse_filename
@@ -24,7 +24,39 @@
 # Initialize a logger.
 logger = logging.getLogger(__name__)
 
-def check_duplicate_files(package_archives, cache=None):
+def check_package(archive, cache=None):
+    """
+    Perform static checks on a package's dependency set.
+
+    :param archive: The pathname of an existing ``*.deb`` archive (a string).
+    :param cache: The :py:class:`.PackageCache` to use (defaults to ``None``).
+    :raises: :py:class:`BrokenPackage` when one or more checks failed.
+    """
+    timer = Timer()
+    logger.info("Checking %s ..", format_path(archive))
+    dependency_set = collect_related_packages(archive, cache=cache)
+    failed_checks = []
+    # Check for duplicate files in the dependency set.
+    try:
+        check_duplicate_files(dependency_set, cache=cache)
+    except BrokenPackage as e:
+        failed_checks.append(e)
+    except ValueError:
+        # Silenced.
+        pass
+    # Check for version conflicts in the dependency set.
+    try:
+        check_version_conflicts(dependency_set, cache=cache)
+    except BrokenPackage as e:
+        failed_checks.append(e)
+    if len(failed_checks) == 1:
+        raise failed_checks[0]
+    elif failed_checks:
+        raise BrokenPackage('\n\n'.join(map(str, failed_checks)))
+    else:
+        logger.info("Finished checking in %s, no problems found.", timer)
+
+def check_duplicate_files(dependency_set, cache=None):
     """
     Check a collection of Debian package archives for conflicts.
 
@@ -36,26 +68,27 @@ def check_duplicate_files(package_archives, cache=None):
     pathnames of files installed by packages it can be slow. To make it faster
     you can use the :py:class:`.PackageCache`.
 
-    :param package_archives: A list of filenames (strings) of ``*.deb`` files.
+    :param dependency_set: A list of filenames (strings) of ``*.deb`` files.
     :param cache: The :py:class:`.PackageCache` to use (defaults to ``None``).
     :raises: :py:class:`exceptions.ValueError` when less than two package
              archives are given (the duplicate check obviously only works if
              there are packages to compare :-).
     :raises: :py:class:`DuplicateFilesFound` when duplicate files are found
              within a group of package archives.
     """
-    package_archives = list(map(parse_filename, package_archives))
+    timer = Timer()
+    dependency_set = list(map(parse_filename, dependency_set))
     # Make sure we have something useful to work with.
-    num_archives = len(package_archives)
+    num_archives = len(dependency_set)
     if num_archives < 2:
         msg = "To check for duplicate files you need to provide two or more packages archives! (%i given)"
         raise ValueError(msg % num_archives)
     # Build up a global map of all files contained in the given package archives.
     global_contents = collections.defaultdict(set)
     global_fields = {}
     spinner = Spinner(total=num_archives)
-    logger.info("Scanning %i package archives for duplicate files (this can take a while) ..", num_archives)
-    for i, archive in enumerate(optimize_order(package_archives), start=1):
+    logger.info("Checking for duplicate files in %i package archives ..", num_archives)
+    for i, archive in enumerate(optimize_order(dependency_set), start=1):
         spinner.step(label="Scanning %i package archives" % num_archives, progress=i)
         fields, contents = inspect_package(archive.filename, cache=cache)
         global_fields[archive.filename] = fields
@@ -122,43 +155,41 @@ def find_virtual_name(field_name):
         delimiter = '%s\n' % ('-' * 79)
         raise DuplicateFilesFound(delimiter.join(summary))
     else:
-        logger.info("No conflicting files found in %i package(s).", len(package_archives))
+        logger.info("No conflicting files found (took %s).", timer)
 
-def check_version_conflicts(package_archives, cache=None):
+def check_version_conflicts(dependency_set, cache=None):
     """
-    Check for version conflicts in one or more dependency sets.
+    Check for version conflicts in a dependency set.
 
-    For each Debian binary package archive given, find the dependencies of the
-    package and check ik newer versions of dependencies exist in the same
-    repository (directory).
-
-    This analysis can be very slow. To make it faster you can use the
+    For each Debian binary package archive given, check if a newer version of
+    the same package exists in the same repository (directory). This analysis
+    can be very slow. To make it faster you can use the
     :py:class:`.PackageCache`.
 
-    :param package_archives: A list of filenames (strings) of ``*.deb`` files.
+    :param dependency_set: A list of filenames (strings) of ``*.deb`` files.
     :param cache: The :py:class:`.PackageCache` to use (defaults to ``None``).
     :raises: :py:class:`VersionConflictFound` when one or more version
              conflicts are found.
     """
+    timer = Timer()
     summary = []
-    num_package_archives = 0
-    for archive in sorted(map(parse_filename, package_archives)):
-        logger.info("Checking dependencies of %s for version conflicts ..", format_path(archive.filename))
-        for related_archive in collect_related_packages(archive.filename, cache=cache):
-            if related_archive.newer_versions:
-                summary.append(compact("""
-                        The dependency set of {package} includes {dependency}
-                        but newer version(s) of that package also exist and
-                        will take precedence:
-                """, package=format_path(archive.filename),
-                     dependency=format_path(related_archive.filename)))
-                summary.append("\n".join(" - %s" % format_path(a.filename) for a in sorted(related_archive.newer_versions)))
-            num_package_archives += 1
+    dependency_set = list(map(parse_filename, dependency_set))
+    spinner = Spinner(total=len(dependency_set))
+    logger.info("Checking for version conflicts in %i package(s) ..", len(dependency_set))
+    for i, archive in enumerate(dependency_set, start=1):
+        if archive.newer_versions:
+            summary.append(compact("""
+                    Dependency set includes {dependency} but newer version(s)
+                    of that package also exist and will take precedence:
+            """, dependency=format_path(archive.filename)))
+            summary.append("\n".join(" - %s" % format_path(a.filename) for a in sorted(archive.newer_versions)))
+        spinner.step(label="Checking for version conflicts", progress=i)
+    spinner.clear()
     if summary:
         summary.insert(0, "One or more version conflicts found:")
         raise VersionConflictFound('\n\n'.join(summary))
     else:
-        logger.info("No version conflicts found in %i package(s).", num_package_archives)
+        logger.info("No version conflicts found (took %s).", timer)
 
 class BrokenPackage(Exception):
     """

deb_pkg_tools/cli.py

@@ -1,7 +1,7 @@
 # Debian packaging tools: Command line interface
 #
 # Author: Peter Odding <peter@peterodding.com>
-# Last Change: June 25, 2014
+# Last Change: August 31, 2014
 # URL: https://github.com/xolox/python-deb-pkg-tools
 
 """
@@ -13,6 +13,9 @@
   -c, --collect=DIR           copy the package archive(s) given as positional
                               arguments and all packages archives required by
                               the given package archives into a directory
+  -C, --check=FILE            perform static analysis on a package archive and
+                              its dependencies in order to recognize common
+                              errors as soon as possible
   -p, --patch=FILE            patch fields into an existing control file
   -s, --set=LINE              a line to patch into the control file
                               (syntax: "Name: Value")
@@ -26,6 +29,7 @@
                               activate the repository, run the positional
                               arguments as an external command (usually `apt-get
                               install') and finally deactivate the repository
+  -y, --yes                   assume the answer to interactive questions is yes
   -v, --verbose               make more noise
   -h, --help                  show this message and exit
 """
@@ -45,6 +49,7 @@
 
 # Modules included in our package.
 from deb_pkg_tools.cache import get_default_cache
+from deb_pkg_tools.checks import check_package
 from deb_pkg_tools.control import patch_control_file
 from deb_pkg_tools.package import (build_package, collect_related_packages,
                                    inspect_package, parse_filename)
@@ -72,17 +77,18 @@ def main():
         sys.stdout = codecs.getwriter(OUTPUT_ENCODING)(sys.stdout)
         sys.stderr = codecs.getwriter(OUTPUT_ENCODING)(sys.stderr)
     # Command line option defaults.
+    prompt = True
     actions = []
     control_file = None
     control_fields = {}
     # Initialize the package cache.
     cache = get_default_cache()
     # Parse the command line options.
     try:
-        options, arguments = getopt.getopt(sys.argv[1:], 'i:c:p:s:b:u:a:d:w:vh', [
-            'inspect=', 'collect=', 'patch=', 'set=', 'build=', 'update-repo=',
-            'activate-repo=', 'deactivate-repo=', 'with-repo=', 'verbose',
-            'help'
+        options, arguments = getopt.getopt(sys.argv[1:], 'i:c:C:p:s:b:u:a:d:w:yvh', [
+            'inspect=', 'collect=', 'check=', 'patch=', 'set=', 'build=',
+            'update-repo=', 'activate-repo=', 'deactivate-repo=', 'with-repo=',
+            'yes', 'verbose', 'help'
         ])
         for option, value in options:
             if option in ('-i', '--inspect'):
@@ -91,8 +97,11 @@ def main():
                 actions.append(functools.partial(collect_packages,
                                                  archives=arguments,
                                                  directory=check_directory(value),
+                                                 prompt=prompt,
                                                  cache=cache))
                 arguments = []
+            elif option in ('-C', '--check'):
+                actions.append(functools.partial(check_package, archive=value, cache=cache))
             elif option in ('-p', '--patch'):
                 control_file = os.path.abspath(value)
                 assert os.path.isfile(control_file), "Control file does not exist!"
@@ -114,6 +123,8 @@ def main():
                                                  directory=check_directory(value),
                                                  command=arguments,
                                                  cache=cache))
+            elif option in ('-y', '--yes'):
+                prompt = False
             elif option in ('-v', '--verbose'):
                 coloredlogs.increase_verbosity()
             elif option in ('-h', '--help'):
@@ -162,7 +173,7 @@ def show_package_metadata(archive):
             group=entry.group, size=size, modified=entry.modified,
             pathname=pathname))
 
-def collect_packages(archives, directory, cache):
+def collect_packages(archives, directory, prompt=True, cache=None):
     # Find all related packages.
     related_archives = set()
     for filename in archives:
@@ -182,9 +193,10 @@ def collect_packages(archives, directory, cache):
         for file_to_collect in relevant_archives:
             print(" - %s" % format_path(file_to_collect.filename))
         try:
-            # Ask permission to copy the file(s).
-            prompt = "Copy %s to %s? [Y/n] " % (pluralized, format_path(directory))
-            assert raw_input(prompt).lower() in ('', 'y', 'yes')
+            if prompt:
+                # Ask permission to copy the file(s).
+                prompt = "Copy %s to %s? [Y/n] " % (pluralized, format_path(directory))
+                assert raw_input(prompt).lower() in ('', 'y', 'yes')
             # Copy the file(s).
             for file_to_collect in relevant_archives:
                 copy_from = file_to_collect.filename

deb_pkg_tools/tests.py

@@ -1,7 +1,7 @@
 # Debian packaging tools: Automated tests.
 #
 # Author: Peter Odding <peter@peterodding.com>
-# Last Change: August 30, 2014
+# Last Change: August 31, 2014
 # URL: https://github.com/xolox/python-deb-pkg-tools
 
 # Standard library modules.
@@ -366,13 +366,42 @@ def test_command_line_interface(self):
                 directory = finalizers.mkdtemp()
                 # Test `deb-pkg-tools --inspect PKG'.
                 package_file = self.test_package_building(directory)
-                lines = call('--inspect', package_file).splitlines()
+                lines = call('--verbose', '--inspect', package_file).splitlines()
                 for field, value in TEST_PACKAGE_FIELDS.items():
                     self.assertEqual(match('^ - %s: (.+)$' % field, lines), value)
                 # Test `deb-pkg-tools --with-repo=DIR CMD' (we simply check whether
                 # apt-cache sees the package).
                 if os.getuid() == 0:
                     call('--with-repo=%s' % directory, 'apt-cache show %s' % TEST_PACKAGE_NAME)
+                # Test `deb-pkg-tools --update=DIR' with a non-existing directory.
+                self.assertRaises(SystemExit, call, '--update', '/a/directory/that/will/never/exist')
+
+    def test_check_package(self):
+        with Context() as finalizers:
+            directory = finalizers.mkdtemp()
+            root_package, conflicting_package = self.create_version_conflict(directory)
+            self.assertRaises(SystemExit, call, '--check', root_package)
+            # Test for lack of duplicate files.
+            os.unlink(conflicting_package)
+            call('--check', root_package)
+
+    def test_version_conflicts_check(self):
+        with Context() as finalizers:
+            # Check that version conflicts raise an exception.
+            directory = finalizers.mkdtemp()
+            root_package, conflicting_package = self.create_version_conflict(directory)
+            packages_to_scan = collect_related_packages(root_package)
+            # Test the duplicate files check.
+            self.assertRaises(VersionConflictFound, check_version_conflicts, packages_to_scan, self.package_cache)
+            # Test for lack of duplicate files.
+            os.unlink(conflicting_package)
+            self.assertEqual(check_version_conflicts(packages_to_scan, cache=self.package_cache), None)
+
+    def create_version_conflict(self, directory):
+        root_package = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-1', Depends='deb-pkg-tools-package-2 (=1)'))
+        self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-2', Version='1'))
+        conflicting_package = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-2', Version='2'))
+        return root_package, conflicting_package
 
     def test_duplicates_check(self):
         with Context() as finalizers:
@@ -409,20 +438,17 @@ def test_duplicates_check(self):
             # Verify that invalid arguments are checked.
             self.assertRaises(ValueError, check_duplicate_files, [])
 
-    def test_version_conflicts_check(self):
+    def test_collect_packages(self):
         with Context() as finalizers:
-            # Check that version conflicts raise an exception.
-            directory = finalizers.mkdtemp()
-            package1 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-1', Depends='deb-pkg-tools-package-2 (=1)'))
-            package2 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-2', Version='1'))
-            package3 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-2', Version='2'))
-            # Test the duplicate files check.
-            self.assertRaises(VersionConflictFound, check_version_conflicts, [package1], self.package_cache)
-            # Test for lack of duplicate files.
-            os.unlink(package3)
-            self.assertEqual(check_version_conflicts([package1]), None)
+            source_directory = finalizers.mkdtemp()
+            target_directory = finalizers.mkdtemp()
+            package1 = self.test_package_building(source_directory, overrides=dict(Package='deb-pkg-tools-package-1', Depends='deb-pkg-tools-package-2'))
+            package2 = self.test_package_building(source_directory, overrides=dict(Package='deb-pkg-tools-package-2', Depends='deb-pkg-tools-package-3'))
+            package3 = self.test_package_building(source_directory, overrides=dict(Package='deb-pkg-tools-package-3'))
+            call('--yes', '--collect=%s' % target_directory, package1)
+            self.assertEqual(sorted(os.listdir(target_directory)), sorted(map(os.path.basename, [package1, package2, package3])))
 
-    def test_collect_packages(self):
+    def test_collect_packages_interactive(self):
         with Context() as finalizers:
             directory = finalizers.mkdtemp()
             package1 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-1', Depends='deb-pkg-tools-package-2'))
@@ -431,6 +457,7 @@ def test_collect_packages(self):
             package4 = self.test_package_building(directory, overrides=dict(Package='deb-pkg-tools-package-3', Version='0.2'))
             self.assertEqual(sorted(p.filename for p in collect_related_packages(package1, cache=self.package_cache)), [package2, package4])
 
+
     def test_repository_creation(self, preserve=False):
         if not SKIP_SLOW_TESTS:
             with Context() as finalizers:
@@ -497,7 +524,8 @@ def test_gpg_key_generation(self):
                 self.assertRaises(Exception, GPGKey, public_key_file=public_key_file)
                 missing_secret_key_file = '/tmp/deb-pkg-tools-%i.sec' % random.randint(1, 1000)
                 missing_public_key_file = '/tmp/deb-pkg-tools-%i.pub' % random.randint(1, 1000)
-                self.assertRaises(Exception, GPGKey, key_id='12345', secret_key_file=missing_secret_key_file, public_key_file=missing_public_key_file)
+                self.assertRaises(Exception, GPGKey, key_id='12345', secret_key_file=secret_key_file, public_key_file=missing_public_key_file)
+                self.assertRaises(Exception, GPGKey, key_id='12345', secret_key_file=missing_secret_key_file, public_key_file=public_key_file)
                 os.unlink(secret_key_file)
                 self.assertRaises(Exception, GPGKey, name="test-key", description="Whatever", secret_key_file=secret_key_file, public_key_file=public_key_file)
                 touch(secret_key_file)