ProfilesCheck: make.defaults checks

src/pkgcheck/addons/__init__.py

@@ -89,7 +89,7 @@ class KeywordsAddon(base.Addon):
     def __init__(self, *args):
         super().__init__(*args)
         special = {"-*"}
-        self.arches = self.options.target_repo.known_arches
+        self.arches: frozenset[str] = self.options.target_repo.known_arches
         unstable = {"~" + x for x in self.arches}
         disabled = {"-" + x for x in chain(self.arches, unstable)}
         self.valid = special | self.arches | unstable | disabled

src/pkgcheck/checks/profiles.py

@@ -2,6 +2,7 @@
 
 import os
 from collections import defaultdict
+from typing import Iterable
 
 from pkgcore.ebuild import misc
 from pkgcore.ebuild import profiles as profiles_mod
@@ -96,16 +97,61 @@ def desc(self):
 class UnknownProfileUseExpand(results.ProfilesResult, results.Warning):
     """Profile includes nonexistent USE_EXPAND group(s)."""
 
-    def __init__(self, path, groups):
+    def __init__(self, path: str, var: str, groups: Iterable[str]):
         super().__init__()
         self.path = path
+        self.var = var
         self.groups = tuple(groups)
 
     @property
     def desc(self):
         s = pluralism(self.groups)
         groups = ", ".join(self.groups)
-        return f"{self.path!r}: unknown USE_EXPAND group{s}: {groups}"
+        return f"{self.path!r}: unknown USE_EXPAND group{s} in {self.var!r}: {groups}"
+
+
+class UnknownProfileUseExpandValue(results.ProfilesResult, results.Warning):
+    """Profile defines unknown default values for USE_EXPAND group."""
+
+    def __init__(self, path: str, group: str, values: Iterable[str]):
+        super().__init__()
+        self.path = path
+        self.group = group
+        self.values = tuple(values)
+
+    @property
+    def desc(self):
+        s = pluralism(self.values)
+        values = ", ".join(self.values)
+        return f"{self.path!r}: unknown value{s} for {self.group!r}: {values}"
+
+
+class ProfileMissingImplicitExpandValues(results.ProfilesResult, results.Warning):
+    """Profile is missing USE_EXPAND_VALUES for implicit USE_EXPAND group."""
+
+    def __init__(self, path: str, groups: Iterable[str]):
+        super().__init__()
+        self.path = path
+        self.groups = tuple(groups)
+
+    @property
+    def desc(self):
+        s = pluralism(self.groups)
+        groups = ", ".join(self.groups)
+        return f"{self.path!r}: missing USE_EXPAND_VALUES for USE_EXPAND group{s}: {groups}"
+
+
+class UnknownProfileArch(results.ProfilesResult, results.Warning):
+    """Profile includes unknown ARCH."""
+
+    def __init__(self, path: str, arch: str):
+        super().__init__()
+        self.path = path
+        self.arch = arch
+
+    @property
+    def desc(self):
+        return f"{self.path!r}: unknown ARCH {self.arch!r}"
 
 
 class ProfileWarning(results.ProfilesResult, results.LogWarning):
@@ -146,28 +192,34 @@ class ProfilesCheck(Check):
     _source = sources.ProfilesRepoSource
     required_addons = (addons.UseAddon, addons.KeywordsAddon)
     known_results = frozenset(
-        [
+        {
             UnknownProfilePackage,
             UnmatchedProfilePackageUnmask,
             UnknownProfilePackageUse,
             UnknownProfileUse,
             UnknownProfilePackageKeywords,
             UnknownProfileUseExpand,
+            UnknownProfileUseExpandValue,
+            ProfileMissingImplicitExpandValues,
+            UnknownProfileArch,
             ProfileWarning,
             ProfileError,
-        ]
+        }
     )
 
     # mapping between known files and verification methods
     known_files = {}
 
-    def __init__(self, *args, use_addon, keywords_addon):
+    def __init__(self, *args, use_addon: addons.UseAddon, keywords_addon: addons.KeywordsAddon):
         super().__init__(*args)
         repo = self.options.target_repo
         self.keywords = keywords_addon
         self.search_repo = self.options.search_repo
         self.profiles_dir = repo.config.profiles_base
-        self.use_expand_groups = frozenset(x.upper() for x in repo.config.use_expand_desc)
+        self.use_expand_groups = {
+            use.upper(): frozenset({val.removeprefix(f"{use}_") for val, _desc in vals})
+            for use, vals in repo.config.use_expand_desc.items()
+        }
 
         local_iuse = {use for _pkg, (use, _desc) in repo.config.use_local_desc}
         self.available_iuse = frozenset(
@@ -275,12 +327,53 @@ def _pkg_use(self, filename, node, vals):
                     yield UnknownProfilePackage(pjoin(node.name, filename), a)
 
     @verify_files(("make.defaults", "make_defaults"))
-    def _make_defaults(self, filename, node, vals):
-        if defined := set(vals.get("USE_EXPAND", "").split()):
-            if unknown := defined - self.use_expand_groups:
-                yield UnknownProfileUseExpand(pjoin(node.name, filename), sorted(unknown))
+    def _make_defaults(self, filename: str, node: sources.ProfileNode, vals: dict[str, str]):
+        if use_flags := {
+            use.removeprefix("-")
+            for use_group in ("USE", "IUSE_IMPLICIT")
+            for use in vals.get(use_group, "").split()
+        }:
+            if unknown := use_flags - self.available_iuse:
+                yield UnknownProfileUse(pjoin(node.name, filename), sorted(unknown))
+        implicit_use_expands = set(vals.get("USE_EXPAND_IMPLICIT", "").split())
+        for use_group in (
+            "USE_EXPAND",
+            "USE_EXPAND_HIDDEN",
+            "USE_EXPAND_UNPREFIXED",
+        ):
+            values = {use.removeprefix("-") for use in vals.get(use_group, "").split()}
+            if unknown := values - self.use_expand_groups.keys() - implicit_use_expands:
+                yield UnknownProfileUseExpand(
+                    pjoin(node.name, filename), use_group, sorted(unknown)
+                )
+        for key, val in vals.items():
+            if key.startswith("USE_EXPAND_VALUES_"):
+                use_group = key[18:]
+                if use_group in implicit_use_expands:
+                    continue
+                elif allowed_values := self.use_expand_groups.get(use_group, None):
+                    if unknown := set(val.split()) - allowed_values:
+                        yield UnknownProfileUseExpandValue(
+                            pjoin(node.name, filename), key, sorted(unknown)
+                        )
+                else:
+                    yield UnknownProfileUseExpand(pjoin(node.name, filename), key, [use_group])
+        for key in vals.keys() & self.use_expand_groups.keys():
+            if unknown := set(vals.get(key, "").split()) - self.use_expand_groups[key]:
+                yield UnknownProfileUseExpandValue(pjoin(node.name, filename), key, sorted(unknown))
+        if missing_values := {
+            use_group
+            for use_group in implicit_use_expands
+            if f"USE_EXPAND_VALUES_{use_group}" not in vals
+        }:
+            yield ProfileMissingImplicitExpandValues(
+                pjoin(node.name, filename), sorted(missing_values)
+            )
+        if arch := vals.get("ARCH", None):
+            if arch not in self.keywords.arches:
+                yield UnknownProfileArch(pjoin(node.name, filename), arch)
 
-    def feed(self, profile):
+    def feed(self, profile: sources.Profile):
         for f in profile.files.intersection(self.known_files):
             attr, func = self.known_files[f]
             with base.LogReports(*_logmap) as log_reports:

testdata/data/repos/profiledir/ProfilesCheck/ProfileMissingImplicitExpandValues/expected.json

@@ -0,0 +1 @@
+{"__class__": "ProfileMissingImplicitExpandValues", "path": "unknown_use/make.defaults", "groups": ["ELIBC"]}

testdata/data/repos/profiledir/ProfilesCheck/ProfileMissingImplicitExpandValues/fix.patch

@@ -0,0 +1,12 @@
+diff -Naur profiledir/profiles/unknown_use/make.defaults fixed/profiles/unknown_use/make.defaults
+diff --git a/testdata/repos/profiledir/profiles/unknown_use/make.defaults b/testdata/repos/profiledir/profiles/unknown_use/make.defaults
+--- profiledir/profiles/unknown_use/make.defaults
++++ fixed/profiles/unknown_use/make.defaults
+@@ -4,6 +4,7 @@
+ USE_EXPAND="PYTHON_TARGETS PYTHON_SINGLE_TARGET"
+ USE_EXPAND_UNPREFIXED="ARCH LUA_TARGETS PYTHON_TARGETS"
+ USE_EXPAND_VALUES_ARCH="amd64 x86"
++USE_EXPAND_VALUES_ELIBC="GNU"
+ USE_EXPAND_VALUES_LUA_TARGETS="lua5_1"
+ USE_EXPAND_VALUES_PYTHON_TARGETS="python3_9 python3_10"
+ PYTHON_TARGETS="python3_9 python3_10"

testdata/data/repos/profiledir/ProfilesCheck/UnknownProfileArch/expected.json

@@ -0,0 +1 @@
+{"__class__": "UnknownProfileArch", "path": "unknown_arch/make.defaults", "arch": "unknown"}

testdata/data/repos/profiledir/ProfilesCheck/UnknownProfileArch/fix.patch

@@ -0,0 +1,6 @@
+diff -Naur profiledir/profiles/unknown_arch/make.defaults fixed/profiles/unknown_arch/make.defaults
+--- profiledir/profiles/unknown_arch/make.defaults	2019-11-12 20:35:43.195830784 -0700
++++ fixed/profiles/unknown_arch/make.defaults	2019-11-12 20:45:16.582848028 -0700
+@@ -1 +1 @@
+-ARCH="unknown"
++ARCH="amd64"

testdata/data/repos/profiledir/ProfilesCheck/UnknownProfileUse/expected.json

@@ -1,3 +1,4 @@
+{"__class__": "UnknownProfileUse", "path": "unknown_use/make.defaults", "flags": ["defaults_iuse_implicit", "defaults_use"]}
 {"__class__": "UnknownProfileUse", "path": "unknown_use/unknown_stable_use/use.force", "flags": ["-use_force"]}
 {"__class__": "UnknownProfileUse", "path": "unknown_use/unknown_stable_use/use.stable.force", "flags": ["use_stable_force"]}
 {"__class__": "UnknownProfileUse", "path": "unknown_use/unknown_stable_use_mask/use.mask", "flags": ["-use_mask"]}

testdata/data/repos/profiledir/ProfilesCheck/UnknownProfileUse/fix.patch

@@ -1,9 +1,11 @@
 diff -Naur profiledir/profiles/use.desc fixed/profiles/use.desc
 --- profiledir/profiles/use.desc	2020-11-23 10:54:01.018477444 -0700
 +++ fixed/profiles/use.desc	2020-11-23 11:54:08.071178614 -0700
-@@ -1 +1,5 @@
+@@ -1 +1,7 @@
  used - used global flag
 +use_force - use.force
 +use_mask - use.mask
 +use_stable_force - use.stable.force
 +use_stable_mask - use.stable.mask
++defaults_use - make.defaults USE
++defaults_iuse_implicit - make.defaults IUSE_IMPLICIT

testdata/data/repos/profiledir/ProfilesCheck/UnknownProfileUseExpand/expected.json

@@ -0,0 +1,3 @@
+{"__class__": "UnknownProfileUseExpand", "path": "unknown_use/make.defaults", "var": "USE_EXPAND", "groups": ["PYTHON_SINGLE_TARGET"]}
+{"__class__": "UnknownProfileUseExpand", "path": "unknown_use/make.defaults", "var": "USE_EXPAND_UNPREFIXED", "groups": ["LUA_TARGETS"]}
+{"__class__": "UnknownProfileUseExpand", "path": "unknown_use/make.defaults", "var": "USE_EXPAND_VALUES_LUA_TARGETS", "groups": ["LUA_TARGETS"]}

testdata/data/repos/profiledir/ProfilesCheck/UnknownProfileUseExpand/fix.patch

@@ -0,0 +1,26 @@
+diff -Naur profiledir/profiles/desc/python_single_target.desc fixed/profiles/desc/python_single_target.desc
+new file mode 100644
+index 00000000..dcf7e163
+--- /dev/null
++++ fixed/profiles/desc/python_single_target.desc
+@@ -0,0 +1,3 @@
++python3_10 - Build for Python 3.10 only
++python3_11 - Build for Python 3.11 only
++
+diff -Naur profiledir/profiles/unknown_use/make.defaults fixed/profiles/unknown_use/make.defaults
+diff --git a/testdata/repos/profiledir/profiles/unknown_use/make.defaults b/testdata/repos/profiledir/profiles/unknown_use/make.defaults
+index 6d789215..2c98efd3 100644
+--- profiledir/profiles/unknown_use/make.defaults
++++ fixed/profiles/unknown_use/make.defaults
+@@ -1,9 +1,8 @@
+ USE="defaults_use used"
+ IUSE_IMPLICIT="defaults_iuse_implicit used"
+ USE_EXPAND_IMPLICIT="ARCH ELIBC"
+-USE_EXPAND="PYTHON_TARGETS PYTHON_SINGLE_TARGET"
+-USE_EXPAND_UNPREFIXED="ARCH LUA_TARGETS PYTHON_TARGETS"
++USE_EXPAND="PYTHON_TARGETS"
++USE_EXPAND_UNPREFIXED="ARCH PYTHON_TARGETS"
+ USE_EXPAND_VALUES_ARCH="amd64 x86"
+-USE_EXPAND_VALUES_LUA_TARGETS="lua5_1"
+ USE_EXPAND_VALUES_PYTHON_TARGETS="python3_9 python3_10"
+ PYTHON_TARGETS="python3_9 python3_10"

testdata/data/repos/profiledir/ProfilesCheck/UnknownProfileUseExpandValue/expected.json

@@ -0,0 +1,2 @@
+{"__class__": "UnknownProfileUseExpandValue", "path": "unknown_use/make.defaults", "group": "PYTHON_TARGETS", "values": ["python3_9"]}
+{"__class__": "UnknownProfileUseExpandValue", "path": "unknown_use/make.defaults", "group": "USE_EXPAND_VALUES_PYTHON_TARGETS", "values": ["python3_9"]}

testdata/data/repos/profiledir/ProfilesCheck/UnknownProfileUseExpandValue/fix.patch

@@ -0,0 +1,13 @@
+diff -Naur profiledir/profiles/unknown_use/make.defaults fixed/profiles/unknown_use/make.defaults
+diff --git a/testdata/repos/profiledir/profiles/unknown_use/make.defaults b/testdata/repos/profiledir/profiles/unknown_use/make.defaults
+index 6d789215..2c98efd3 100644
+--- profiledir/profiles/unknown_use/make.defaults
++++ fixed/profiles/unknown_use/make.defaults
+@@ -5,5 +5,5 @@
+ USE_EXPAND_UNPREFIXED="ARCH LUA_TARGETS PYTHON_TARGETS"
+ USE_EXPAND_VALUES_ARCH="amd64 x86"
+ USE_EXPAND_VALUES_LUA_TARGETS="lua5_1"
+-USE_EXPAND_VALUES_PYTHON_TARGETS="python3_9 python3_10"
+-PYTHON_TARGETS="python3_9 python3_10"
++USE_EXPAND_VALUES_PYTHON_TARGETS="python3_10"
++PYTHON_TARGETS="python3_10"

testdata/repos/profiledir/profiles/desc/python_targets.desc

@@ -0,0 +1,2 @@
+python3_10 - Build with Python 3.10
+python3_11 - Build with Python 3.11

testdata/repos/profiledir/profiles/profiles.desc

@@ -6,6 +6,7 @@ amd64 nonexistent exp
 amd64 unknown_pkgs exp
 amd64 unknown_kwds exp
 amd64 unmatched_unmasks exp
+amd64 unknown_arch exp
 
 amd64 unknown_use exp
 amd64 unknown_use/unknown_stable_use exp

testdata/repos/profiledir/profiles/unknown_arch/eapi

@@ -0,0 +1 @@
+4

testdata/repos/profiledir/profiles/unknown_arch/make.defaults

@@ -0,0 +1 @@
+ARCH="unknown"

testdata/repos/profiledir/profiles/unknown_use/make.defaults

@@ -0,0 +1,9 @@
+USE="defaults_use used"
+IUSE_IMPLICIT="defaults_iuse_implicit used"
+USE_EXPAND_IMPLICIT="ARCH ELIBC"
+USE_EXPAND="PYTHON_TARGETS PYTHON_SINGLE_TARGET"
+USE_EXPAND_UNPREFIXED="ARCH LUA_TARGETS PYTHON_TARGETS"
+USE_EXPAND_VALUES_ARCH="amd64 x86"
+USE_EXPAND_VALUES_LUA_TARGETS="lua5_1"
+USE_EXPAND_VALUES_PYTHON_TARGETS="python3_9 python3_10"
+PYTHON_TARGETS="python3_9 python3_10"

tests/scripts/test_pkgcheck_scan.py

@@ -611,7 +611,7 @@ def test_scan_verify(self, repo, tmp_path):
         if repo not in self._results:
             self.test_scan_repo(repo, tmp_path, verbosity=0)
         if repo not in self._verbose_results:
-            self.test_scan_repo(repo, tmp_path, verbosity=0)
+            self.test_scan_repo(repo, tmp_path, verbosity=1)
         for check, keywords in self._checks[repo].items():
             for keyword in keywords:
                 # verify the expected results were seen during the repo scans