diff --git a/packagedb/api.py b/packagedb/api.py index 3dd17d7e..a55a2c2a 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -109,27 +109,30 @@ def filter_by_checksums(self, request, *args, **kwargs): against the packagedb. Supported checksum fields are: + - md5 - sha1 Example: - { - "sha1": [ - "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", - "27afff2610b5a94274a2311f8b15e514446b0e76 - ] - } + + { + "sha1": [ + "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", + "27afff2610b5a94274a2311f8b15e514446b0e76 + ] + } Multiple checksums algorithms can be passed together: - { - "sha1": [ - "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", - "27afff2610b5a94274a2311f8b15e514446b0e76 - ], - "md5": [ - "e927df60b093456d4e611ae235c1aa5b" - ] - } + + { + "sha1": [ + "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", + "27afff2610b5a94274a2311f8b15e514446b0e76 + ], + "md5": [ + "e927df60b093456d4e611ae235c1aa5b" + ] + } This will return Resources whose sha1 or md5 matches those values. """ @@ -376,20 +379,30 @@ def get_enhanced_package_data(self, request, *args, **kwargs): @action(detail=False, methods=['post']) def index_packages(self, request, *args, **kwargs): """ - Take a list of dictionary where each dictionary has either resolved PURL i.e. PURL with - version or version-less PURL along with vers range. Then return a mapping containing + Take a list of `packages` where each item is a dictionary containing either PURL + or versionless PURL along with vers range. + **Note:** When a versionless PURL is supplied without a vers range, then all the versions + of that package will be indexed. + + **Input example:** - Input example: - [ - { - "purl": "pkg:npm/foobar@12.3.1", - }, { - "purl": "pkg:npm/foobar", - "vers": "vers:npm/>=1.0.0|<=4.1.0" + "packages": [ + { + "purl": "pkg:npm/foobar@12.3.1", + }, + { + "purl": "pkg:npm/foobar", + "vers": "vers:npm/>=1.0.0|<=4.1.0" + }, + { + "purl": "pkg:npm/foobar2", + } + ... + ] } - ... - ] + + Then return a mapping containing: - queued_packages_count - The number of package urls placed on the queue. @@ -413,7 +426,6 @@ def index_packages(self, request, *args, **kwargs): - unsupported_vers - A list of vers range that are not supported by the univers or package_manager. """ - packages = request.data.get('packages') or [] queued_packages = [] unqueued_packages = [] @@ -465,6 +477,7 @@ def reindex_packages(self, request, *args, **kwargs): package set as the packages from `package_urls` will be reindexed. Then return a mapping containing: + - requeued_packages_count - The number of package urls placed on the queue. - requeued_packages @@ -517,29 +530,32 @@ def filter_by_checksums(self, request, *args, **kwargs): against the packagedb. Supported checksum fields are: + - md5 - sha1 - sha256 - sha512 Example: - { - "sha1": [ - "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", - "27afff2610b5a94274a2311f8b15e514446b0e76 - ] - } + + { + "sha1": [ + "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", + "27afff2610b5a94274a2311f8b15e514446b0e76 + ] + } Multiple checksums algorithms can be passed together: - { - "sha1": [ - "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", - "27afff2610b5a94274a2311f8b15e514446b0e76 - ], - "md5": [ - "e927df60b093456d4e611ae235c1aa5b" - ] - } + + { + "sha1": [ + "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911", + "27afff2610b5a94274a2311f8b15e514446b0e76 + ], + "md5": [ + "e927df60b093456d4e611ae235c1aa5b" + ] + } This will return Packages whose sha1 or md5 matches those values. """ @@ -735,12 +751,18 @@ def get_resolved_purls(packages, supported_ecosystems): unsupported_purls.add(purl) continue + if parsed_purl.type not in supported_ecosystems: + unsupported_purls.add(purl) + continue + if parsed_purl.version: unique_resolved_purls.add(purl) continue - if not vers or parsed_purl.type not in supported_ecosystems: - unsupported_purls.add(purl) + # Versionless PURL without any vers-range should give all versions. + if not vers and not parsed_purl.version: + if resolved:= resolve_all_versions(parsed_purl): + unique_resolved_purls.update(resolved) continue if resolved:= resolve_versions(parsed_purl, vers): @@ -750,7 +772,23 @@ def get_resolved_purls(packages, supported_ecosystems): return list(unique_resolved_purls), list(unsupported_purls), list(unsupported_vers) +def resolve_all_versions(parsed_purl): + """ + Take versionless and return a list of PURLs for all the released versions. + """ + all_versions = get_all_versions(parsed_purl) or [] + return [ + str( + PackageURL( + type=parsed_purl.type, + namespace=parsed_purl.namespace, + name=parsed_purl.name, + version=version.string, + ) + ) + for version in all_versions + ] def resolve_versions(parsed_purl, vers): """ diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 5010d5d2..357e2ab4 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -576,6 +576,62 @@ def test_package_api_index_packages_endpoint_with_vers(self, mock_get_all_versio priority_resource_uris_count = PriorityResourceURI.objects.all().count() self.assertEqual(9, priority_resource_uris_count) + @mock.patch("packagedb.api.get_all_versions") + def test_package_api_index_packages_endpoint_all_version_index(self, mock_get_all_versions): + priority_resource_uris_count = PriorityResourceURI.objects.all().count() + self.assertEqual(0, priority_resource_uris_count) + packages = [ + { + "purl": "pkg:maven/ch.qos.reload4j/reload4j", + }, + ] + data = {"packages": packages} + + mock_get_all_versions.return_value = [ + MavenVersion("1.2.18.0"), + MavenVersion("1.2.18.1"), + MavenVersion("1.2.18.2"), + MavenVersion("1.2.18.3"), + MavenVersion("1.2.18.4"), + MavenVersion("1.2.18.5"), + MavenVersion("1.2.19"), + MavenVersion("1.2.20"), + MavenVersion("1.2.21"), + MavenVersion("1.2.22"), + MavenVersion("1.2.23"), + MavenVersion("1.2.24"), + MavenVersion("1.2.25"), + ] + + response = self.client.post( + "/api/packages/index_packages/", data=data, format="json" + ) + self.assertEqual(13, response.data["queued_packages_count"]) + + expected_queued_packages = [ + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.0", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.1", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.2", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.3", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.4", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.18.5", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.19", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.20", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.21", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.22", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.23", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.24", + "pkg:maven/ch.qos.reload4j/reload4j@1.2.25", + ] + self.assertEqual( + sorted(expected_queued_packages), sorted(response.data["queued_packages"]) + ) + self.assertEqual(0, response.data["unqueued_packages_count"]) + self.assertEqual([], response.data["unqueued_packages"]) + self.assertEqual(0, response.data["unsupported_packages_count"]) + priority_resource_uris_count = PriorityResourceURI.objects.all().count() + self.assertEqual(13, priority_resource_uris_count) + def test_package_api_filter_by_checksums(self): sha1s = [ 'testsha1', diff --git a/setup.cfg b/setup.cfg index bf2f2267..b28f9eb7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,6 +46,7 @@ install_requires = gunicorn == 20.1.0 ftputil == 5.0.4 jawa == 2.2.0 + markdown == 3.5.1 natsort == 8.2.0 packageurl-python == 0.11.2 psycopg2-binary == 2.9.3