From ad5f579b82f4494e8ad0fa0783eba6dcd9235c39 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 2 Dec 2024 11:53:37 -0800 Subject: [PATCH 1/4] Remove ordering from Package and Resource #551 Signed-off-by: Jono Yang --- ..._package_options_alter_resource_options.py | 21 +++++++++++++++++++ packagedb/models.py | 2 -- 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 packagedb/migrations/0088_alter_package_options_alter_resource_options.py diff --git a/packagedb/migrations/0088_alter_package_options_alter_resource_options.py b/packagedb/migrations/0088_alter_package_options_alter_resource_options.py new file mode 100644 index 00000000..838a2b87 --- /dev/null +++ b/packagedb/migrations/0088_alter_package_options_alter_resource_options.py @@ -0,0 +1,21 @@ +# Generated by Django 5.1.2 on 2024-12-02 19:40 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("packagedb", "0087_rename_discovered_dependency_attribute"), + ] + + operations = [ + migrations.AlterModelOptions( + name="package", + options={}, + ), + migrations.AlterModelOptions( + name="resource", + options={}, + ), + ] diff --git a/packagedb/models.py b/packagedb/models.py index 1ce166bd..94af6ed5 100644 --- a/packagedb/models.py +++ b/packagedb/models.py @@ -552,7 +552,6 @@ class Package( # TODO: Think about ordering, unique together, indexes, etc. class Meta: - ordering = ["id"] unique_together = [ ( "download_url", @@ -1144,7 +1143,6 @@ class Resource( class Meta: unique_together = (("package", "path"),) - ordering = ("id",) indexes = [ models.Index(fields=["md5"]), models.Index(fields=["sha1"]), From b7bb053d633926e33f747d17351af48c2834a062 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 2 Dec 2024 18:02:10 -0800 Subject: [PATCH 2/4] Update Resource query in filter_by_checksums #551 Signed-off-by: Jono Yang --- Dockerfile | 2 +- packagedb/api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8cb9bcd9..08afc9cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -FROM --platform=linux/amd64 python:3.12-slim +FROM python:3.12-slim LABEL org.opencontainers.image.source="https://github.com/aboutcode-org/purldb" LABEL org.opencontainers.image.description="PurlDB" diff --git a/packagedb/api.py b/packagedb/api.py index 24621ee7..01092362 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -258,7 +258,7 @@ def filter_by_checksums(self, request, *args, **kwargs): d = {f"{field}__in": value} lookups |= Q(**d) - qs = Resource.objects.filter(lookups).prefetch_related("package") + qs = Resource.objects.filter(lookups).select_related("package").defer("package__history") paginated_qs = self.paginate_queryset(qs) serializer = ResourceAPISerializer( paginated_qs, many=True, context={"request": request} From af6a63c715c69075db6be3c276191f704284475d Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 4 Dec 2024 16:19:24 -0800 Subject: [PATCH 3/4] Do not order results in filter_by_checksums #551 Signed-off-by: Jono Yang --- packagedb/api.py | 12 +- ...package_md5_alter_package_sha1_and_more.py | 109 ++++++++++++++++++ ..._package_options_alter_resource_options.py | 21 ---- packagedb/models.py | 6 + 4 files changed, 124 insertions(+), 24 deletions(-) create mode 100644 packagedb/migrations/0088_alter_package_md5_alter_package_sha1_and_more.py delete mode 100644 packagedb/migrations/0088_alter_package_options_alter_resource_options.py diff --git a/packagedb/api.py b/packagedb/api.py index 01092362..01ccfabe 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -258,12 +258,18 @@ def filter_by_checksums(self, request, *args, **kwargs): d = {f"{field}__in": value} lookups |= Q(**d) - qs = Resource.objects.filter(lookups).select_related("package").defer("package__history") + qs = ( + Resource.objects.filter(lookups) + .select_related("package") + .only("package__uuid") + .order_by() + ) paginated_qs = self.paginate_queryset(qs) serializer = ResourceAPISerializer( paginated_qs, many=True, context={"request": request} ) - return self.get_paginated_response(serializer.data) + paginated_response = self.get_paginated_response(serializer.data) + return paginated_response class MultiplePackageURLFilter(MultipleCharFilter): @@ -485,7 +491,7 @@ def filter_by_checksums(self, request, *args, **kwargs): ) # Query to get the full Package objects with the earliest release_date for each sha1 - qs = Package.objects.filter(lookups) + qs = Package.objects.filter(lookups).order_by() paginated_qs = self.paginate_queryset(qs) if enhance_package_data: serialized_package_data = [ diff --git a/packagedb/migrations/0088_alter_package_md5_alter_package_sha1_and_more.py b/packagedb/migrations/0088_alter_package_md5_alter_package_sha1_and_more.py new file mode 100644 index 00000000..b0834fb9 --- /dev/null +++ b/packagedb/migrations/0088_alter_package_md5_alter_package_sha1_and_more.py @@ -0,0 +1,109 @@ +# Generated by Django 5.1.2 on 2024-12-05 00:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("packagedb", "0087_rename_discovered_dependency_attribute"), + ] + + operations = [ + migrations.AlterField( + model_name="package", + name="md5", + field=models.CharField( + blank=True, + db_index=True, + help_text="MD5 checksum hex-encoded, as in md5sum.", + max_length=32, + null=True, + verbose_name="MD5", + ), + ), + migrations.AlterField( + model_name="package", + name="sha1", + field=models.CharField( + blank=True, + db_index=True, + help_text="SHA1 checksum hex-encoded, as in sha1sum.", + max_length=40, + null=True, + verbose_name="SHA1", + ), + ), + migrations.AlterField( + model_name="package", + name="sha256", + field=models.CharField( + blank=True, + db_index=True, + help_text="SHA256 checksum hex-encoded, as in sha256sum.", + max_length=64, + null=True, + verbose_name="SHA256", + ), + ), + migrations.AlterField( + model_name="package", + name="sha512", + field=models.CharField( + blank=True, + db_index=True, + help_text="SHA512 checksum hex-encoded, as in sha512sum.", + max_length=128, + null=True, + verbose_name="SHA512", + ), + ), + migrations.AlterField( + model_name="resource", + name="md5", + field=models.CharField( + blank=True, + db_index=True, + help_text="MD5 checksum hex-encoded, as in md5sum.", + max_length=32, + null=True, + verbose_name="MD5", + ), + ), + migrations.AlterField( + model_name="resource", + name="sha1", + field=models.CharField( + blank=True, + db_index=True, + help_text="SHA1 checksum hex-encoded, as in sha1sum.", + max_length=40, + null=True, + verbose_name="SHA1", + ), + ), + migrations.AlterField( + model_name="resource", + name="sha256", + field=models.CharField( + blank=True, + db_index=True, + help_text="SHA256 checksum hex-encoded, as in sha256sum.", + max_length=64, + null=True, + verbose_name="SHA256", + ), + ), + migrations.AlterField( + model_name="resource", + name="sha512", + field=models.CharField( + blank=True, + db_index=True, + help_text="SHA512 checksum hex-encoded, as in sha512sum.", + max_length=128, + null=True, + verbose_name="SHA512", + ), + ), + ] diff --git a/packagedb/migrations/0088_alter_package_options_alter_resource_options.py b/packagedb/migrations/0088_alter_package_options_alter_resource_options.py deleted file mode 100644 index 838a2b87..00000000 --- a/packagedb/migrations/0088_alter_package_options_alter_resource_options.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 5.1.2 on 2024-12-02 19:40 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("packagedb", "0087_rename_discovered_dependency_attribute"), - ] - - operations = [ - migrations.AlterModelOptions( - name="package", - options={}, - ), - migrations.AlterModelOptions( - name="resource", - options={}, - ), - ] diff --git a/packagedb/models.py b/packagedb/models.py index 94af6ed5..60493a29 100644 --- a/packagedb/models.py +++ b/packagedb/models.py @@ -174,6 +174,7 @@ class Meta: max_length=32, blank=True, null=True, + db_index=True, help_text=_("MD5 checksum hex-encoded, as in md5sum."), ) sha1 = models.CharField( @@ -181,6 +182,7 @@ class Meta: max_length=40, blank=True, null=True, + db_index=True, help_text=_("SHA1 checksum hex-encoded, as in sha1sum."), ) sha256 = models.CharField( @@ -188,6 +190,7 @@ class Meta: max_length=64, blank=True, null=True, + db_index=True, help_text=_("SHA256 checksum hex-encoded, as in sha256sum."), ) sha512 = models.CharField( @@ -195,6 +198,7 @@ class Meta: max_length=128, blank=True, null=True, + db_index=True, help_text=_("SHA512 checksum hex-encoded, as in sha512sum."), ) @@ -552,6 +556,7 @@ class Package( # TODO: Think about ordering, unique together, indexes, etc. class Meta: + ordering = ["id"] unique_together = [ ( "download_url", @@ -1143,6 +1148,7 @@ class Resource( class Meta: unique_together = (("package", "path"),) + ordering = ("id",) indexes = [ models.Index(fields=["md5"]), models.Index(fields=["sha1"]), From d57cc24fb29197e83ceee38a8148674b7a1d7e0c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 5 Dec 2024 10:45:41 -0800 Subject: [PATCH 4/4] Use f-string Signed-off-by: Jono Yang --- minecode/ls.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/minecode/ls.py b/minecode/ls.py index 14a9ad71..6da1a25c 100644 --- a/minecode/ls.py +++ b/minecode/ls.py @@ -182,10 +182,7 @@ def parse_directory_listing(dir_listing, from_find=False): # this is likely a directory line from an ls -LR listing. Strip # trailing colon and keep track of the base directory if not line.endswith(":"): - raise Exception( - "Unknown directory listing line format: #%(ln)d: %(line)r" - % locals() - ) + raise Exception(f"Unknown directory listing line format: #{ln}: {line}") base_dir = line.strip(":") continue