pkglistgen: integrate drop list creation into update_and_solve.

In order to generate a complete drop list the nonfree repository must also
be cached for both current and published builds. The solv files are then
merged and used to generate drop list.

pkglistgen.py

@@ -40,6 +40,8 @@
 from osc import conf
 from osclib.conf import Config
 from osclib.stagingapi import StagingAPI
+from osclib.util import project_list_family
+from osclib.util import project_list_family_prior
 from xdg.BaseDirectory import save_cache_path
 import glob
 import solv
@@ -871,7 +873,7 @@ def do_dump_solv(self, subcmd, opts, baseurl):
             name = '{}/{}.solv'.format(self.options.output_dir, build)
             if not opts.overwrite and os.path.exists(name):
                 logger.info("%s exists", name)
-                return
+                return name
             ofh = open(name + '.new', 'w')
 
         pool = solv.Pool()
@@ -901,6 +903,7 @@ def do_dump_solv(self, subcmd, opts, baseurl):
 
         if name is not None:
             os.rename(name + '.new', name)
+            return name
 
     def dump_solv_build(self, baseurl):
         """Determine repo format and build string from remote repository."""
@@ -1029,14 +1032,14 @@ def do_update_and_solve(self, subcmd, opts):
 
         if opts.scope == 'target':
             self.options.repos = ['/'.join([target_project, main_repo])]
-            self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts)
+            self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts, drop_list=True)
             return
         elif opts.scope == 'ports':
             # TODO Continue supporting #1297, but should be abstracted.
             main_repo = 'ports'
             opts.project += ':Ports'
             self.options.repos = ['/'.join([opts.project, main_repo])]
-            self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts)
+            self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts, drop_list=True)
             return
         elif opts.scope == 'rings':
             opts.project = api.rings[1]
@@ -1071,7 +1074,7 @@ def do_update_and_solve(self, subcmd, opts):
             return
 
     def update_and_solve_target(self, apiurl, target_project, target_config, main_repo, opts,
-                                skip_release=False):
+                                skip_release=False, drop_list=False):
         print('[{}] {}/{}: update and solve'.format(opts.scope, opts.project, main_repo))
 
         group = target_config.get('pkglistgen-group', '000package-groups')
@@ -1132,6 +1135,26 @@ def update_and_solve_target(self, apiurl, target_project, target_config, main_re
         print('-> do_update')
         self.do_update('update', opts)
 
+        nonfree = target_config.get('nonfree')
+        if nonfree and drop_list:
+            print('-> do_update nonfree')
+
+            # Switch to nonfree repo (ugly, but that's how the code was setup).
+            self.options.repos_ = self.options.repos
+            self.options.repos = ['/'.join([nonfree, main_repo])]
+            self.postoptparse()
+
+            opts_nonfree = copy.deepcopy(opts)
+            opts_nonfree.project = nonfree
+            self.do_update('update', opts_nonfree)
+
+            # Switch repo back to main target project.
+            self.options.repos = self.options.repos_
+            self.postoptparse()
+
+        print('-> do_update_merge')
+        self.do_update_merge('update_merge', opts)
+
         print('-> do_solve')
         opts.ignore_unresolvable = bool(target_config.get('pkglistgen-ignore-unresolvable'))
         opts.ignore_recommended = bool(target_config.get('pkglistgen-ignore-recommended'))
@@ -1140,6 +1163,26 @@ def update_and_solve_target(self, apiurl, target_project, target_config, main_re
         opts.locales_from = target_config.get('pkglistgen-locales-from')
         self.do_solve('solve', opts)
 
+        if drop_list:
+            # Ensure solv files from all releases in product family are updated.
+            print('-> solv_cache_update')
+            cache_dir_solv = save_cache_path('opensuse-packagelists', 'solv')
+            family_include = target_config.get('pkglistgen-product-family-include')
+            solv_prior = self.solv_cache_update(
+                apiurl, cache_dir_solv, target_project, family_include, opts)
+
+            # Include pre-final release solv files for target project. These
+            # files will only exist from previous runs.
+            cache_dir_solv_current = os.path.join(cache_dir_solv, target_project)
+            solv_prior.update(glob.glob(os.path.join(cache_dir_solv_current, '*.merged.solv')))
+            for solv in solv_prior:
+                logger.debug(solv.replace(cache_dir_solv, ''))
+
+            print('-> do_create_droplist')
+            # Reset to product after solv_cache_update().
+            self.options.output_dir = product_dir
+            self.do_create_droplist('create_droplist', opts, *solv_prior)
+
         delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
         self.unlink_list(product_dir, delete_products)
 
@@ -1166,6 +1209,55 @@ def update_and_solve_target(self, apiurl, target_project, target_config, main_re
             self.build_stub(release_dir, 'spec')
             self.commit_package(release_dir)
 
+    def solv_cache_update(self, apiurl, cache_dir_solv, target_project, family_include, opts):
+        """Dump solv files (do_dump_solv) for all products in family."""
+        prior = set()
+
+        project_family = project_list_family_prior(apiurl, target_project, include_self=True)
+        if family_include:
+            # Include projects from a different family if desired.
+            project_family.extend(project_list_family(apiurl, family_include))
+
+        for project in project_family:
+            config = Config(project)
+            project_config = conf.config[project]
+
+            baseurl = project_config.get('download-baseurl')
+            if not baseurl:
+                logger.warning('no baseurl configured for {}'.format(project))
+                continue
+
+            urls = [urlparse.urljoin(baseurl, 'repo/oss/')]
+            if project_config.get('nonfree'):
+                urls.append(urlparse.urljoin(baseurl, 'repo/non-oss/'))
+
+            names = []
+            for url in urls:
+                print('-> do_dump_solv for {}/{}'.format(
+                    project, os.path.basename(os.path.normpath(url))))
+                logger.debug(url)
+
+                self.options.output_dir = os.path.join(cache_dir_solv, project)
+                if not os.path.exists(self.options.output_dir):
+                    os.makedirs(self.options.output_dir)
+
+                opts.overwrite = False
+                names.append(self.do_dump_solv('dump_solv', opts, url))
+
+            if not len(names):
+                logger.warning('no solv files were dumped for {}'.format(project))
+                continue
+
+            # Merge nonfree solv with free solv or copy free solv as merged.
+            merged = names[0].replace('.solv', '.merged.solv')
+            if len(names) == 2:
+                self.solv_merge(names[0], names[1], merged)
+            else:
+                shutil.copyfile(names[0], merged)
+            prior.add(merged)
+
+        return prior
+
     def move_list(self, file_list, destination):
         for name in file_list:
             os.rename(name, os.path.join(destination, os.path.basename(name)))