Refactored the removal of releases for release_plugins to happen insi…

…de of Package

Plugins apart of the FilterReleasePlugin group no longer remove the releases that are passed into the filter.  Release removal is done now in a new function of the Package class ._filter_all_releases().  Also changed the plugin group of the exclude_platform filter to release_file_plugin.

Tests updated to account for new API.  Corrected a linux platform specifier in TestExcludePlatformFilter based on pypi/warehouse#2010

setup.cfg

@@ -58,12 +58,12 @@ bandersnatch_filter_plugins.v2.release =
     prerelease_release = bandersnatch_filter_plugins.prerelease_name:PreReleaseFilter
     regex_release = bandersnatch_filter_plugins.regex_name:RegexReleaseFilter
     latest_release = bandersnatch_filter_plugins.latest_name:LatestReleaseFilter
-    exclude_platform = bandersnatch_filter_plugins.filename_name:ExcludePlatformFilter
 
 # This entrypoint group must match the value of bandersnatch.filter.RELEASE_FILE_PLUGIN_RESOURCE
 bandersnatch_filter_plugins.v2.release_file =
     regex_release_file_metadata = bandersnatch_filter_plugins.metadata_filter:RegexReleaseFileMetadataFilter
     version_range_release_file_metadata = bandersnatch_filter_plugins.metadata_filter:VersionRangeReleaseFileMetadataFilter
+    exclude_platform = bandersnatch_filter_plugins.filename_name:ExcludePlatformFilter
 
 console_scripts =
     bandersnatch = bandersnatch.main:main

src/bandersnatch/package.py

@@ -156,7 +156,7 @@ async def sync(self, filters: "LoadedFilters", attempts: int = 3) -> None:
                 )
 
             self._filter_all_releases_files(filters.filter_release_file_plugins())
-            self._filter_releases(filters.filter_release_plugins())
+            self._filter_all_releases(filters.filter_release_plugins())
 
             await self.sync_release_files()
             self.sync_simple_page()
@@ -177,35 +177,47 @@ def _filter_metadata(
         Run the metadata filtering plugins
         """
         global display_filter_log
-        filter_plugins = metadata_filters
-        if not filter_plugins:
+        if not metadata_filters:
             if display_filter_log:
                 logger.info(
                     "No metadata filters are enabled. Skipping metadata filtering"
                 )
                 display_filter_log = False
             return True
 
-        return all(plugin.filter(metadata) for plugin in filter_plugins)
+        return all(plugin.filter(metadata) for plugin in metadata_filters)
 
-    def _filter_releases(self, release_filters: List["Filter"]) -> bool:
+    def _filter_release(
+        self, release_data: Dict, release_filters: List["Filter"]
+    ) -> bool:
         """
         Run the release filtering plugins
         """
         global display_filter_log
-        filter_plugins = release_filters
-        if not filter_plugins:
+        if not release_filters:
             if display_filter_log:
                 logger.info(
                     "No release filters are enabled. Skipping release filtering"
                 )
                 display_filter_log = False
             return True
 
-        return all(
-            plugin.filter({"info": self.info, "releases": self.releases})
-            for plugin in filter_plugins
-        )
+        return all(plugin.filter(release_data) for plugin in release_filters)
+
+    def _filter_all_releases(self, release_filters: List["Filter"]) -> bool:
+        """
+        Filter releases and removes releases that fail the filters
+        """
+        releases = list(self.releases.keys())
+        for version in releases:
+            if not self._filter_release(
+                {"version": version, "releases": self.releases, "info": self.info},
+                release_filters,
+            ):
+                del self.releases[version]
+        if releases:
+            return True
+        return False
 
     def _filter_release_file(
         self, metadata: Dict, release_file_filters: List["Filter"]
@@ -214,36 +226,35 @@ def _filter_release_file(
         Run the release file filtering plugins
         """
         global display_filter_log
-        filter_plugins = release_file_filters
-        if not filter_plugins:
+        if not release_file_filters:
             if display_filter_log:
                 logger.info(
                     "No release file filters are enabled. Skipping release file filtering"  # noqa: E501
                 )
                 display_filter_log = False
             return True
 
-        return all(plugin.filter(metadata) for plugin in filter_plugins)
+        return all(plugin.filter(metadata) for plugin in release_file_filters)
 
     def _filter_all_releases_files(self, release_file_filters: List["Filter"]) -> bool:
         """
         Filter release files and remove empty releases after doing so.
         """
         releases = list(self.releases.keys())
-        for release in releases:
-            release_files = list(self.releases[release])
+        for version in releases:
+            release_files = list(self.releases[version])
             for rfindex in reversed(range(len(release_files))):
                 if not self._filter_release_file(
                     {
                         "info": self.info,
-                        "release": release,
-                        "release_file": self.releases[release][rfindex],
+                        "release": version,
+                        "release_file": self.releases[version][rfindex],
                     },
                     release_file_filters,
                 ):
-                    del self.releases[release][rfindex]
-            if not self.releases[release]:
-                del self.releases[release]
+                    del self.releases[version][rfindex]
+            if not self.releases[version]:
+                del self.releases[version]
 
         if releases:
             return True

src/bandersnatch/tests/plugins/test_blacklist_name.py

@@ -171,6 +171,6 @@ def test__filter__matches__release(self) -> None:
             "releases": {"1.2.0": {}, "1.2.1": {}},
         }
 
-        pkg._filter_releases(mirror.filters.filter_release_plugins())
+        pkg._filter_all_releases(mirror.filters.filter_release_plugins())
 
         self.assertEqual(pkg.releases, {"1.2.1": {}})

src/bandersnatch/tests/plugins/test_filename.py

@@ -42,13 +42,13 @@ class TestExcludePlatformFilter(BasePluginTestCase):
     windows
     freebsd
     macos
-    linux-armv7l
+    linux_armv7l
 """
 
     def test_plugin_compiles_patterns(self) -> None:
         mock_config(self.config_contents)
 
-        plugins = bandersnatch.filter.LoadedFilters().filter_release_plugins()
+        plugins = bandersnatch.filter.LoadedFilters().filter_release_file_plugins()
 
         assert any(
             type(plugin) == filename_name.ExcludePlatformFilter for plugin in plugins
@@ -151,12 +151,12 @@ def test_exclude_platform(self) -> None:
         rv = pkg.releases.values()
         keep_count = sum(f["flag"] == "KEEP" for r in rv for f in r)
 
-        pkg._filter_releases(mirror.filters.filter_release_plugins())
+        pkg._filter_all_releases_files(mirror.filters.filter_release_file_plugins())
 
         # we should have the same keep count and no drop
         rv = pkg.releases.values()
         assert sum(f["flag"] == "KEEP" for r in rv for f in r) == keep_count
-        assert all(f["flag"] == "DROP" for r in rv for f in r) is False
+        assert sum(f["flag"] == "DROP" for r in rv for f in r) == 0
 
         # the release "0.2" should have been deleted since there is no more file in it
         assert len(pkg.releases.keys()) == 2

src/bandersnatch/tests/plugins/test_latest_release.py

@@ -73,7 +73,7 @@ def test_latest_releases_keep_latest(self) -> None:
             },
         }
 
-        pkg._filter_releases(mirror.filters.filter_release_plugins())
+        pkg._filter_all_releases(mirror.filters.filter_release_plugins())
 
         assert pkg.releases == {"1.1.3": {}, "2.0.0": {}}
 
@@ -96,7 +96,7 @@ def test_latest_releases_keep_stable(self) -> None:
             },
         }
 
-        pkg._filter_releases(mirror.filters.filter_release_plugins())
+        pkg._filter_all_releases(mirror.filters.filter_release_plugins())
 
         assert pkg.releases == {"2.0.1b2": {}, "2.0.0": {}}
 
@@ -141,7 +141,7 @@ def test_latest_releases_uninitialized(self) -> None:
             },
         }
 
-        pkg._filter_releases(mirror.filters.filter_release_plugins())
+        pkg._filter_all_releases(mirror.filters.filter_release_plugins())
 
         assert pkg.releases == {
             "1.0.0": {},

src/bandersnatch/tests/plugins/test_prerelease_name.py

@@ -74,6 +74,6 @@ def test_plugin_check_match(self) -> None:
             },
         }
 
-        pkg._filter_releases(mirror.filters.filter_release_plugins())
+        pkg._filter_all_releases(mirror.filters.filter_release_plugins())
 
         assert pkg.releases == {"1.2.0": {}}

src/bandersnatch/tests/plugins/test_regex_name.py

@@ -67,7 +67,7 @@ def test_plugin_check_match(self) -> None:
             "releases": {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}},
         }
 
-        pkg._filter_releases(mirror.filters.filter_release_plugins())
+        pkg._filter_all_releases(mirror.filters.filter_release_plugins())
 
         assert pkg.releases == {"foo-1.2.0": {}}
 

src/bandersnatch/tests/test_filter.py

@@ -62,7 +62,6 @@ def test__filter_release_plugins__loads(self) -> None:
             "blacklist_release",
             "prerelease_release",
             "regex_release",
-            "exclude_platform",
             "latest_release",
         ]
 

src/bandersnatch_filter_plugins/blacklist_name.py

@@ -134,15 +134,13 @@ def _determine_filtered_package_requirements(self) -> List[Requirement]:
         return list(filtered_requirements)
 
     def filter(self, metadata: Dict) -> bool:
+        """
+        Returns False if version fails the filter,
+        i.e. matches a blocklist version specifier
+        """
         name = metadata["info"]["name"]
-        releases = metadata["releases"]
-        for version in list(releases.keys()):
-            if self._check_match(name, version):
-                del releases[version]
-        if not releases:
-            return False
-        else:
-            return True
+        version = metadata["version"]
+        return not self._check_match(name, version)
 
     def _check_match(self, name: str, version_string: str) -> bool:
         """

src/bandersnatch_filter_plugins/filename_name.py

@@ -1,12 +1,12 @@
 import logging
 from typing import Dict, List
 
-from bandersnatch.filter import FilterReleasePlugin
+from bandersnatch.filter import FilterReleaseFilePlugin
 
 logger = logging.getLogger("bandersnatch")
 
 
-class ExcludePlatformFilter(FilterReleasePlugin):
+class ExcludePlatformFilter(FilterReleaseFilePlugin):
     """
     Filters releases based on regex patters defined by the user.
     """
@@ -78,31 +78,11 @@ def initialize_plugin(self) -> None:
         logger.info(f"Initialized {self.name} plugin with {self._patterns!r}")
 
     def filter(self, metadata: Dict) -> bool:
-        releases = metadata["releases"]
         """
-        Remove files from `releases` that match any pattern.
+        Returns False if file matches any of the filename patterns
         """
-
-        # Make a copy of releases keys
-        # as we may delete packages during iteration
-        removed = 0
-        versions = list(releases.keys())
-        for version in versions:
-            new_files = []
-            for file_desc in releases[version]:
-                if self._check_match(file_desc):
-                    removed += 1
-                else:
-                    new_files.append(file_desc)
-            if len(new_files) == 0:
-                del releases[version]
-            else:
-                releases[version] = new_files
-        logger.debug(f"{self.name}: filenames removed: {removed}")
-        if not releases:
-            return False
-        else:
-            return True
+        file = metadata["release_file"]
+        return not self._check_match(file)
 
     def _check_match(self, file_desc: Dict) -> bool:
         """

src/bandersnatch_filter_plugins/latest_name.py

@@ -1,6 +1,6 @@
 import logging
 from operator import itemgetter
-from typing import Dict, List, Optional, Set, Union
+from typing import Dict, Sequence
 
 from packaging.version import parse
 
@@ -16,6 +16,7 @@ class LatestReleaseFilter(FilterReleasePlugin):
 
     name = "latest_release"
     keep = 0  # by default, keep 'em all
+    latest: Sequence = []
 
     def initialize_plugin(self) -> None:
         """
@@ -33,44 +34,31 @@ def initialize_plugin(self) -> None:
         if self.keep > 0:
             logger.info(f"Initialized latest releases plugin with keep={self.keep}")
 
-    def filter(self, metadata: Dict) -> Optional[bool]:  # type: ignore[override]
+    def filter(self, metadata: Dict) -> bool:
         """
-        Keep the latest releases
+        Returns False if version fails the filter, i.e. is not a latest/current release
         """
-        latest: Union[List, Set]
-        info = metadata["info"]
-        releases = metadata["releases"]
-
         if self.keep == 0:
-            return None
-
-        versions = list(releases.keys())
-        before = len(versions)
-
-        if before <= self.keep:
-            # not enough releases: do nothing
-            return None
+            return True
 
-        versions_pair = map(lambda v: (parse(v), v), versions)
-        latest = sorted(versions_pair)[-self.keep :]  # noqa: E203
-        latest = list(map(itemgetter(1), latest))
+        if not self.latest:
+            info = metadata["info"]
+            releases = metadata["releases"]
+            versions = list(releases.keys())
+            before = len(versions)
 
-        current_version = info.get("version")
-        if current_version and (current_version not in latest):
-            # never remove the stable/official version
-            latest[0] = current_version
+            if before <= self.keep:
+                # not enough releases: do nothing
+                return True
 
-        logger.debug(f"old {versions}")
-        logger.debug(f"new {latest}")
+            versions_pair = map(lambda v: (parse(v), v), versions)
+            self.latest = sorted(versions_pair)[-self.keep :]  # noqa: E203
+            self.latest = list(map(itemgetter(1), self.latest))
 
-        after = len(latest)
-        latest = set(latest)
-        for version in list(releases.keys()):
-            if version not in latest:
-                del releases[version]
+            current_version = info.get("version")
+            if current_version and (current_version not in self.latest):
+                # never remove the stable/official version
+                self.latest[0] = current_version
 
-        logger.debug(f"{self.name}: releases removed: {before - after}")
-        if not releases:
-            return False
-        else:
-            return True
+        version = metadata["version"]
+        return version in self.latest