repoquery: fix rich deps matching by using provide expansion from libdnf

dnf.spec

@@ -1,5 +1,5 @@
 # default dependencies
-%global hawkey_version 0.41.0
+%global hawkey_version 0.44.0
 %global libcomps_version 0.1.8
 %global libmodulemd_version 1.4.0
 %global rpm_version 4.14.0

dnf/cli/commands/repoquery.py

@@ -303,6 +303,12 @@ def configure(self):
                       "(optionally with '--alldeps', but not with '--exactdeps'), or with "
                       "'--requires <REQ> --resolve'"))
 
+        if self.opts.alldeps or self.opts.exactdeps:
+            if not (self.opts.whatrequires or self.opts.whatdepends):
+                raise dnf.cli.CliError(
+                    _("argument {} requires --whatrequires or --whatdepends option".format(
+                        '--alldeps' if self.opts.alldeps else '--exactdeps')))
+
         if self.opts.srpm:
             self.base.repos.enable_source_repos()
 
@@ -343,55 +349,62 @@ def build_format_fn(self, opts, pkg):
             # there don't exist on the dnf Package object.
             raise dnf.exceptions.Error(str(e))
 
-    def _get_recursive_deps_query(self, query_in, query_select, done=None, recursive=False,
-                                  all_deps=False):
-        done = done if done else self.base.sack.query().filterm(empty=True)
-        t = self.base.sack.query().filterm(empty=True)
-        set_requires = set()
-        set_all_deps = set()
-
-        for pkg in query_select.run():
-            pkg_provides = pkg.provides
-            set_requires.update(pkg_provides)
-            set_requires.update(pkg.files)
-            if all_deps:
-                set_all_deps.update(pkg_provides)
-
-        t = t.union(query_in.filter(requires=set_requires))
-        if set_all_deps:
-            t = t.union(query_in.filter(recommends=set_all_deps))
-            t = t.union(query_in.filter(enhances=set_all_deps))
-            t = t.union(query_in.filter(supplements=set_all_deps))
-            t = t.union(query_in.filter(suggests=set_all_deps))
-        if recursive:
-            query_select = t.difference(done)
-            if query_select:
-                done = self._get_recursive_deps_query(query_in, query_select, done=t.union(done),
-                                                      recursive=recursive, all_deps=all_deps)
-        return t.union(done)
+    def _resolve_nevras(self, nevras, base_query):
+        resolved_nevras_query = self.base.sack.query().filterm(empty=True)
+        for nevra in nevras:
+            resolved_nevras_query = resolved_nevras_query.union(base_query.intersection(
+                dnf.subject.Subject(nevra).get_best_query(
+                    self.base.sack,
+                    with_provides=False,
+                    with_filenames=False
+                )
+            ))
+
+        return resolved_nevras_query
+
+    def _do_recursive_deps(self, query_in, query_select, done=None):
+        done = done if done else query_select
+
+        query_required = query_in.filter(requires=query_select)
+
+        query_select = query_required.difference(done)
+        done = query_required.union(done)
+
+        if query_select:
+            done = self._do_recursive_deps(query_in, query_select, done=done)
 
-    def by_all_deps(self, requires_name, depends_name, query):
-        names = requires_name or depends_name
-        defaultquery = self.base.sack.query().filterm(empty=True)
-        for name in names:
-            defaultquery = defaultquery.union(query.intersection(
-                dnf.subject.Subject(name).get_best_query(self.base.sack, with_provides=False,
-                                                         with_filenames=False)))
-        requiresquery = query.filter(requires__glob=names)
-        if depends_name:
-            requiresquery = requiresquery.union(query.filter(recommends__glob=depends_name))
-            requiresquery = requiresquery.union(query.filter(enhances__glob=depends_name))
-            requiresquery = requiresquery.union(query.filter(supplements__glob=depends_name))
-            requiresquery = requiresquery.union(query.filter(suggests__glob=depends_name))
-
-        done = requiresquery.union(self._get_recursive_deps_query(query, defaultquery,
-                                                                  all_deps=depends_name))
-        if self.opts.recursive:
-            done = done.union(self._get_recursive_deps_query(query, done,
-                                                             recursive=self.opts.recursive,
-                                                             all_deps=depends_name))
         return done
 
+    def by_all_deps(self, names, query, all_dep_types=False):
+        # in case of arguments being NEVRAs, resolve them to packages
+        resolved_nevras_query = self._resolve_nevras(names, query)
+
+        # filter the arguments directly as reldeps
+        depquery = query.filter(requires__glob=names)
+
+        # filter the resolved NEVRAs as packages
+        depquery = depquery.union(query.filter(requires=resolved_nevras_query))
+
+        if all_dep_types:
+            # TODO this is very inefficient, as it resolves the `names` glob to
+            # reldeps four more times, which in a reasonably wide glob like
+            # `dnf repoquery --whatdepends "libdnf*"` can take roughly 50% of
+            # the total execution time.
+            depquery = depquery.union(query.filter(recommends__glob=names))
+            depquery = depquery.union(query.filter(enhances__glob=names))
+            depquery = depquery.union(query.filter(supplements__glob=names))
+            depquery = depquery.union(query.filter(suggests__glob=names))
+
+            depquery = depquery.union(query.filter(recommends=resolved_nevras_query))
+            depquery = depquery.union(query.filter(enhances=resolved_nevras_query))
+            depquery = depquery.union(query.filter(supplements=resolved_nevras_query))
+            depquery = depquery.union(query.filter(suggests=resolved_nevras_query))
+
+        if self.opts.recursive:
+            depquery = self._do_recursive_deps(query, depquery)
+
+        return depquery
+
     def _get_recursive_providers_query(self, query_in, providers, done=None):
         done = done if done else self.base.sack.query().filterm(empty=True)
         t = self.base.sack.query().filterm(empty=True)
@@ -486,7 +499,8 @@ def run(self):
         if self.opts.file:
             q.filterm(file__glob=self.opts.file)
         if self.opts.whatconflicts:
-            q.filterm(conflicts=self.opts.whatconflicts)
+            rels = q.filter(conflicts__glob=self.opts.whatconflicts)
+            q = rels.union(q.filter(conflicts=self._resolve_nevras(self.opts.whatconflicts, q)))
         if self.opts.whatobsoletes:
             q.filterm(obsoletes=self.opts.whatobsoletes)
         if self.opts.whatprovides:
@@ -495,36 +509,36 @@ def run(self):
                 q = query_for_provide
             else:
                 q.filterm(file__glob=self.opts.whatprovides)
-        if self.opts.alldeps or self.opts.exactdeps:
-            if not (self.opts.whatrequires or self.opts.whatdepends):
-                raise dnf.exceptions.Error(
-                    _("argument {} requires --whatrequires or --whatdepends option".format(
-                        '--alldeps' if self.opts.alldeps else '--exactdeps')))
-            if self.opts.alldeps:
-                q = self.by_all_deps(self.opts.whatrequires, self.opts.whatdepends, q)
+
+        if self.opts.whatrequires:
+            if (self.opts.exactdeps):
+                q.filterm(requires__glob=self.opts.whatrequires)
             else:
-                if self.opts.whatrequires:
-                    q.filterm(requires__glob=self.opts.whatrequires)
-                else:
-                    dependsquery = q.filter(requires__glob=self.opts.whatdepends)
-                    dependsquery = dependsquery.union(
-                        q.filter(recommends__glob=self.opts.whatdepends))
-                    dependsquery = dependsquery.union(
-                        q.filter(enhances__glob=self.opts.whatdepends))
-                    dependsquery = dependsquery.union(
-                        q.filter(supplements__glob=self.opts.whatdepends))
-                    q = dependsquery.union(q.filter(suggests__glob=self.opts.whatdepends))
-
-        elif self.opts.whatrequires or self.opts.whatdepends:
-            q = self.by_all_deps(self.opts.whatrequires, self.opts.whatdepends, q)
+                q = self.by_all_deps(self.opts.whatrequires, q)
+
+        if self.opts.whatdepends:
+            if (self.opts.exactdeps):
+                dependsquery = q.filter(requires__glob=self.opts.whatdepends)
+                dependsquery = dependsquery.union(q.filter(recommends__glob=self.opts.whatdepends))
+                dependsquery = dependsquery.union(q.filter(enhances__glob=self.opts.whatdepends))
+                dependsquery = dependsquery.union(q.filter(supplements__glob=self.opts.whatdepends))
+                q = dependsquery.union(q.filter(suggests__glob=self.opts.whatdepends))
+            else:
+                q = self.by_all_deps(self.opts.whatdepends, q, True)
+
         if self.opts.whatrecommends:
-            q.filterm(recommends__glob=self.opts.whatrecommends)
+            rels = q.filter(recommends__glob=self.opts.whatrecommends)
+            q = rels.union(q.filter(recommends=self._resolve_nevras(self.opts.whatrecommends, q)))
         if self.opts.whatenhances:
-            q.filterm(enhances__glob=self.opts.whatenhances)
+            rels = q.filter(enhances__glob=self.opts.whatenhances)
+            q = rels.union(q.filter(enhances=self._resolve_nevras(self.opts.whatenhances, q)))
         if self.opts.whatsupplements:
-            q.filterm(supplements__glob=self.opts.whatsupplements)
+            rels = q.filter(supplements__glob=self.opts.whatsupplements)
+            q = rels.union(q.filter(supplements=self._resolve_nevras(self.opts.whatsupplements, q)))
         if self.opts.whatsuggests:
-            q.filterm(suggests__glob=self.opts.whatsuggests)
+            rels = q.filter(suggests__glob=self.opts.whatsuggests)
+            q = rels.union(q.filter(suggests=self._resolve_nevras(self.opts.whatsuggests, q)))
+
         if self.opts.latest_limit:
             q = q.latest(self.opts.latest_limit)
         # reduce a query to security upgrades if they are specified
@@ -672,7 +686,7 @@ def tree_seed(self, query, aquery, opts, level=-1, usedpkgs=None):
                             ar[querypkg.name + "." + querypkg.arch] = querypkg
                     pkgquery = self.base.sack.query().filterm(pkg=list(ar.values()))
                 else:
-                    pkgquery = self.by_all_deps(pkg.name, None, aquery) if opts.alldeps \
+                    pkgquery = self.by_all_deps((pkg.name, ), aquery) if opts.alldeps \
                         else aquery.filter(requires__glob=pkg.name)
                 self.tree_seed(pkgquery, aquery, opts, level + 1, usedpkgs)
 

doc/api_queries.rst

@@ -101,17 +101,20 @@
     release           string         match against packages' releases
     reponame          string         match against packages repositories' names
     version           string         match against packages' versions
-    obsoletes         Query          match packages that obsolete any package from query
     pkg               Query          match against packages in query
     pkg*              list           match against hawkey.Packages in list
     provides          string         match against packages' provides
     provides*         Hawkey.Reldep  match against packages' provides
-    requires          string         match against packages' requirements
-    requires*         Hawkey.Reldep  match against packages' requirements
+    <DEP>             string         match against packages' <DEP>
+    <DEP>*            Hawkey.Reldep  match a reldep against packages' <DEP>
+    <DEP>*            Query          match the result of a query against packages' <DEP>
+    <DEP>*            list(Package)  match the list of hawkey.Packages against packages' <DEP>
     sourcerpm         string         match against packages' source rpm
     upgrades          boolean        see :meth:`upgrades`. Defaults to ``False``.
     ===============   ============== ======================================================
 
+    ``<DEP>`` can be any of: requires, conflicts, obsoletes, enhances, recomments, suggests, supplements
+
     \* The key can also accept a list of values with specified type.
 
     The key name can be supplemented with a relation-specifying suffix, separated by ``__``:
@@ -133,6 +136,8 @@
 
       q = base.sack.query().filter(name__substr="club")
 
+    Note that using packages or queries for dependency filtering performs a more advanced resolution than using a string or a reldep. When a package list or a query is used, rich dependencies are resolved in a more precise way than what is possible when a string or a reldep is used.
+
   .. method:: filterm(\*\*kwargs)
 
     Similar to :meth:`dnf.query.Query.filter` but it modifies the query in place.