Skip to content

Support indexing of all the versions of a package in index_packages endpoint #214

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 8, 2023
Merged

Support indexing of all the versions of a package in index_packages endpoint #214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d93a25f
Support all versions indexing in index_packages endpoint
keshav-space Nov 7, 2023
71c0aba
Update `index_packages` docstring
keshav-space Nov 7, 2023
4a5c54e
Use Markdown for properly rendering API description
keshav-space Nov 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 82 additions & 44 deletions packagedb/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,27 +109,30 @@ def filter_by_checksums(self, request, *args, **kwargs):
against the packagedb.

Supported checksum fields are:

- md5
- sha1

Example:
{
"sha1": [
"b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
"27afff2610b5a94274a2311f8b15e514446b0e76
]
}

{
"sha1": [
"b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
"27afff2610b5a94274a2311f8b15e514446b0e76
]
}

Multiple checksums algorithms can be passed together:
{
"sha1": [
"b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
"27afff2610b5a94274a2311f8b15e514446b0e76
],
"md5": [
"e927df60b093456d4e611ae235c1aa5b"
]
}

{
"sha1": [
"b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
"27afff2610b5a94274a2311f8b15e514446b0e76
],
"md5": [
"e927df60b093456d4e611ae235c1aa5b"
]
}

This will return Resources whose sha1 or md5 matches those values.
"""
Expand Down Expand Up @@ -376,20 +379,30 @@ def get_enhanced_package_data(self, request, *args, **kwargs):
@action(detail=False, methods=['post'])
def index_packages(self, request, *args, **kwargs):
"""
Take a list of dictionary where each dictionary has either resolved PURL i.e. PURL with
version or version-less PURL along with vers range. Then return a mapping containing
Take a list of `packages` where each item is a dictionary containing either PURL
or versionless PURL along with vers range.
**Note:** When a versionless PURL is supplied without a vers range, then all the versions
of that package will be indexed.

**Input example:**

Input example:
[
{
"purl": "pkg:npm/foobar@12.3.1",
},
{
"purl": "pkg:npm/foobar",
"vers": "vers:npm/>=1.0.0|<=4.1.0"
"packages": [
{
"purl": "pkg:npm/foobar@12.3.1",
},
{
"purl": "pkg:npm/foobar",
"vers": "vers:npm/>=1.0.0|<=4.1.0"
},
{
"purl": "pkg:npm/foobar2",
}
...
]
}
...
]

Then return a mapping containing:

- queued_packages_count
- The number of package urls placed on the queue.
Expand All @@ -413,7 +426,6 @@ def index_packages(self, request, *args, **kwargs):
- unsupported_vers
- A list of vers range that are not supported by the univers or package_manager.
"""

packages = request.data.get('packages') or []
queued_packages = []
unqueued_packages = []
Expand Down Expand Up @@ -465,6 +477,7 @@ def reindex_packages(self, request, *args, **kwargs):
package set as the packages from `package_urls` will be reindexed.

Then return a mapping containing:

- requeued_packages_count
- The number of package urls placed on the queue.
- requeued_packages
Expand Down Expand Up @@ -517,29 +530,32 @@ def filter_by_checksums(self, request, *args, **kwargs):
against the packagedb.

Supported checksum fields are:

- md5
- sha1
- sha256
- sha512

Example:
{
"sha1": [
"b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
"27afff2610b5a94274a2311f8b15e514446b0e76
]
}

{
"sha1": [
"b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
"27afff2610b5a94274a2311f8b15e514446b0e76
]
}

Multiple checksums algorithms can be passed together:
{
"sha1": [
"b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
"27afff2610b5a94274a2311f8b15e514446b0e76
],
"md5": [
"e927df60b093456d4e611ae235c1aa5b"
]
}

{
"sha1": [
"b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
"27afff2610b5a94274a2311f8b15e514446b0e76
],
"md5": [
"e927df60b093456d4e611ae235c1aa5b"
]
}

This will return Packages whose sha1 or md5 matches those values.
"""
Expand Down Expand Up @@ -735,12 +751,18 @@ def get_resolved_purls(packages, supported_ecosystems):
unsupported_purls.add(purl)
continue

if parsed_purl.type not in supported_ecosystems:
unsupported_purls.add(purl)
continue

if parsed_purl.version:
unique_resolved_purls.add(purl)
continue

if not vers or parsed_purl.type not in supported_ecosystems:
unsupported_purls.add(purl)
# Versionless PURL without any vers-range should give all versions.
if not vers and not parsed_purl.version:
if resolved:= resolve_all_versions(parsed_purl):
unique_resolved_purls.update(resolved)
continue

if resolved:= resolve_versions(parsed_purl, vers):
Expand All @@ -750,7 +772,23 @@ def get_resolved_purls(packages, supported_ecosystems):

return list(unique_resolved_purls), list(unsupported_purls), list(unsupported_vers)

def resolve_all_versions(parsed_purl):
"""
Take versionless and return a list of PURLs for all the released versions.
"""
all_versions = get_all_versions(parsed_purl) or []

return [
str(
PackageURL(
type=parsed_purl.type,
namespace=parsed_purl.namespace,
name=parsed_purl.name,
version=version.string,
)
)
for version in all_versions
]

def resolve_versions(parsed_purl, vers):
"""
Expand Down
56 changes: 56 additions & 0 deletions packagedb/tests/test_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,62 @@ def test_package_api_index_packages_endpoint_with_vers(self, mock_get_all_versio
priority_resource_uris_count = PriorityResourceURI.objects.all().count()
self.assertEqual(9, priority_resource_uris_count)

@mock.patch("packagedb.api.get_all_versions")
def test_package_api_index_packages_endpoint_all_version_index(self, mock_get_all_versions):
priority_resource_uris_count = PriorityResourceURI.objects.all().count()
self.assertEqual(0, priority_resource_uris_count)
packages = [
{
"purl": "pkg:maven/ch.qos.reload4j/reload4j",
},
]
data = {"packages": packages}

mock_get_all_versions.return_value = [
MavenVersion("1.2.18.0"),
MavenVersion("1.2.18.1"),
MavenVersion("1.2.18.2"),
MavenVersion("1.2.18.3"),
MavenVersion("1.2.18.4"),
MavenVersion("1.2.18.5"),
MavenVersion("1.2.19"),
MavenVersion("1.2.20"),
MavenVersion("1.2.21"),
MavenVersion("1.2.22"),
MavenVersion("1.2.23"),
MavenVersion("1.2.24"),
MavenVersion("1.2.25"),
]

response = self.client.post(
"/api/packages/index_packages/", data=data, format="json"
)
self.assertEqual(13, response.data["queued_packages_count"])

expected_queued_packages = [
"pkg:maven/ch.qos.reload4j/reload4j@1.2.18.0",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.18.1",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.18.2",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.18.3",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.18.4",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.18.5",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.19",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.20",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.21",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.22",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.23",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.24",
"pkg:maven/ch.qos.reload4j/reload4j@1.2.25",
]
self.assertEqual(
sorted(expected_queued_packages), sorted(response.data["queued_packages"])
)
self.assertEqual(0, response.data["unqueued_packages_count"])
self.assertEqual([], response.data["unqueued_packages"])
self.assertEqual(0, response.data["unsupported_packages_count"])
priority_resource_uris_count = PriorityResourceURI.objects.all().count()
self.assertEqual(13, priority_resource_uris_count)

def test_package_api_filter_by_checksums(self):
sha1s = [
'testsha1',
Expand Down
1 change: 1 addition & 0 deletions setup.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ install_requires =
gunicorn == 20.1.0
ftputil == 5.0.4
jawa == 2.2.0
markdown == 3.5.1
natsort == 8.2.0
packageurl-python == 0.11.2
psycopg2-binary == 2.9.3
Expand Down