From 0d35f51a7d5c1667cf15496dd64bc5ee1b40e7cd Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 8 Aug 2023 17:15:48 -0700 Subject: [PATCH 1/2] Update spdx fields on packages #166 * declared_license_expression_spdx and other_license_expression_spdx on package is now a property that is generated from the main license fields Signed-off-by: Jono Yang --- ...clared_license_expression_spdx_and_more.py | 20 ++++++++++++ packagedb/models.py | 31 +++++++++---------- 2 files changed, 34 insertions(+), 17 deletions(-) create mode 100644 packagedb/migrations/0077_remove_package_declared_license_expression_spdx_and_more.py diff --git a/packagedb/migrations/0077_remove_package_declared_license_expression_spdx_and_more.py b/packagedb/migrations/0077_remove_package_declared_license_expression_spdx_and_more.py new file mode 100644 index 00000000..ed8a4a5d --- /dev/null +++ b/packagedb/migrations/0077_remove_package_declared_license_expression_spdx_and_more.py @@ -0,0 +1,20 @@ +# Generated by Django 4.1.2 on 2023-08-08 23:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("packagedb", "0076_rename_history_json_package_history"), + ] + + operations = [ + migrations.RemoveField( + model_name="package", + name="declared_license_expression_spdx", + ), + migrations.RemoveField( + model_name="package", + name="other_license_expression_spdx", + ), + ] diff --git a/packagedb/models.py b/packagedb/models.py index f4253f10..e4e43816 100644 --- a/packagedb/models.py +++ b/packagedb/models.py @@ -21,6 +21,7 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from licensedcode.cache import build_spdx_license_expression from packageurl import PackageURL from packageurl.contrib.django.models import PackageURLMixin from packageurl.contrib.django.models import PackageURLQuerySetMixin @@ -57,7 +58,7 @@ def get_or_none(self, *args, **kwargs): return self.get(*args, **kwargs) except self.DoesNotExist: return - + def paginated(self, per_page=5000): """ Iterate over a (large) QuerySet by chunks of ``per_page`` items. @@ -359,14 +360,6 @@ class AbstractPackage(models.Model): "routine or convention." ), ) - declared_license_expression_spdx = models.TextField( - blank=True, - null=True, - help_text=_( - "The SPDX license expression for this package converted " - "from its declared_license_expression." - ), - ) license_detections = models.JSONField( default=list, blank=True, @@ -386,14 +379,6 @@ class AbstractPackage(models.Model): "routine or convention." ), ) - other_license_expression_spdx = models.TextField( - blank=True, - null=True, - help_text=_( - "The other SPDX license expression for this package converted " - "from its other_license_expression." - ), - ) other_license_detections = models.JSONField( default=list, blank=True, @@ -442,6 +427,18 @@ class AbstractPackage(models.Model): class Meta: abstract = True + @property + def declared_license_expression_spdx(self): + declared_license_expression = self.declared_license_expression + if declared_license_expression: + return build_spdx_license_expression(declared_license_expression) + + @property + def other_license_expression_spdx(self): + other_license_expression = self.other_license_expression + if other_license_expression: + return build_spdx_license_expression(other_license_expression) + class PackageContentType(models.IntegerChoices): """List of Package content types.""" From 7d90c9d408b2a17103a65a897cbc1e7d257c29d1 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 8 Aug 2023 17:55:41 -0700 Subject: [PATCH 2/2] Update test expectations #166 Signed-off-by: Jono Yang --- minecode/model_utils.py | 8 +++++--- minecode/tests/test_housekeeping.py | 6 +++--- minecode/tests/test_maven.py | 2 +- .../housekeeping/declared_license_search_expected.json | 4 ++-- .../tests/testfiles/housekeeping/example_expected.json | 4 ++-- .../housekeeping/ignore_upper_case_search_expected.json | 2 +- .../housekeeping/license_expression_search_expected.json | 4 ++-- .../maven/pom/pulsar-client-merged-ancestor-package.json | 2 +- .../testfiles/model_utils/expected_updated_fields.json | 5 ----- packagedb/serializers.py | 4 ++++ packagedb/tests/testfiles/api/enhanced_package.json | 2 +- packagedb/tests/testfiles/api/twill-core-0.12.0.json | 2 +- 12 files changed, 23 insertions(+), 22 deletions(-) diff --git a/minecode/model_utils.py b/minecode/model_utils.py index cfa2d032..a600b104 100644 --- a/minecode/model_utils.py +++ b/minecode/model_utils.py @@ -67,7 +67,11 @@ def merge_packages(existing_package, new_package_data, replace=False): new_mapping = new_package_data - fields_to_skip = ('package_uid',) + fields_to_skip = ( + 'package_uid', + 'declared_license_expression_spdx', + 'other_license_expression_spdx', + ) updated_fields = [] for existing_field, existing_value in existing_mapping.items(): @@ -327,10 +331,8 @@ def merge_or_create_package(scanned_package, visit_level): copyright=scanned_package.copyright, holder=scanned_package.holder, declared_license_expression=scanned_package.declared_license_expression, - declared_license_expression_spdx=scanned_package.declared_license_expression_spdx, license_detections=scanned_package.license_detections, other_license_expression=scanned_package.other_license_expression, - other_license_expression_spdx=scanned_package.other_license_expression_spdx, other_license_detections=scanned_package.other_license_detections, extracted_license_statement=scanned_package.extracted_license_statement, notice_text=scanned_package.notice_text, diff --git a/minecode/tests/test_housekeeping.py b/minecode/tests/test_housekeeping.py index 7916891c..8796a4c1 100644 --- a/minecode/tests/test_housekeeping.py +++ b/minecode/tests/test_housekeeping.py @@ -36,7 +36,7 @@ def test_find_ambiguous_packages_declared_license(self): packagedb.models.Package.objects.create( download_url='http://example.com', name='Foo', - declared_license_expression='apache and unknown', + declared_license_expression='apache-2.0 and unknown', type='maven' ) packages = [p.to_dict() for p in find_ambiguous_packages()] @@ -47,7 +47,7 @@ def test_find_ambiguous_packages_license_expression(self): packagedb.models.Package.objects.create( download_url='http://example.com', name='Foo', - declared_license_expression='apache and unknown', + declared_license_expression='apache-2.0 and unknown', type='maven' ) packages = [p.to_dict() for p in find_ambiguous_packages()] @@ -72,7 +72,7 @@ def test_run_check_licenses_command(self): packagedb.models.Package.objects.create( download_url='http://example.com', name='Foo', - declared_license_expression='apache and unknown', + declared_license_expression='apache-2.0 and unknown', type='maven' ) results_loc = self.get_temp_file() diff --git a/minecode/tests/test_maven.py b/minecode/tests/test_maven.py index a6500441..a861d00b 100644 --- a/minecode/tests/test_maven.py +++ b/minecode/tests/test_maven.py @@ -830,7 +830,7 @@ def test_merge_ancestors(self, regen=False): maven_visitor.merge_ancestors(ancestor_pom_texts, package) expected_after_loc = self.get_test_loc('maven/pom/pulsar-client-1x-2.5.1-package_after.json') self.check_expected_results(package.to_dict(), expected_after_loc, regen=regen) - + @mock.patch("minecode.visitors.maven.get_pom_text") def test_get_merged_ancestor_package_from_maven_package(self, get_pom_text_mock, regen=False): get_pom_text_mock.return_value = "" diff --git a/minecode/tests/testfiles/housekeeping/declared_license_search_expected.json b/minecode/tests/testfiles/housekeeping/declared_license_search_expected.json index 8e1128c6..3f6e5105 100644 --- a/minecode/tests/testfiles/housekeeping/declared_license_search_expected.json +++ b/minecode/tests/testfiles/housekeeping/declared_license_search_expected.json @@ -25,8 +25,8 @@ "vcs_url":null, "copyright":null, "holder":null, - "declared_license_expression":"apache and unknown", - "declared_license_expression_spdx":null, + "declared_license_expression":"apache-2.0 and unknown", + "declared_license_expression_spdx":"Apache-2.0 AND LicenseRef-scancode-unknown", "license_detections":[], "other_license_expression":null, "other_license_expression_spdx":null, diff --git a/minecode/tests/testfiles/housekeeping/example_expected.json b/minecode/tests/testfiles/housekeeping/example_expected.json index 8e1128c6..3f6e5105 100644 --- a/minecode/tests/testfiles/housekeeping/example_expected.json +++ b/minecode/tests/testfiles/housekeeping/example_expected.json @@ -25,8 +25,8 @@ "vcs_url":null, "copyright":null, "holder":null, - "declared_license_expression":"apache and unknown", - "declared_license_expression_spdx":null, + "declared_license_expression":"apache-2.0 and unknown", + "declared_license_expression_spdx":"Apache-2.0 AND LicenseRef-scancode-unknown", "license_detections":[], "other_license_expression":null, "other_license_expression_spdx":null, diff --git a/minecode/tests/testfiles/housekeeping/ignore_upper_case_search_expected.json b/minecode/tests/testfiles/housekeeping/ignore_upper_case_search_expected.json index 3552ae7e..5d00d875 100644 --- a/minecode/tests/testfiles/housekeeping/ignore_upper_case_search_expected.json +++ b/minecode/tests/testfiles/housekeeping/ignore_upper_case_search_expected.json @@ -26,7 +26,7 @@ "copyright":null, "holder":null, "declared_license_expression":"Unknown", - "declared_license_expression_spdx":null, + "declared_license_expression_spdx":"LicenseRef-scancode-unknown", "license_detections":[], "other_license_expression":null, "other_license_expression_spdx":null, diff --git a/minecode/tests/testfiles/housekeeping/license_expression_search_expected.json b/minecode/tests/testfiles/housekeeping/license_expression_search_expected.json index 8e1128c6..3f6e5105 100644 --- a/minecode/tests/testfiles/housekeeping/license_expression_search_expected.json +++ b/minecode/tests/testfiles/housekeeping/license_expression_search_expected.json @@ -25,8 +25,8 @@ "vcs_url":null, "copyright":null, "holder":null, - "declared_license_expression":"apache and unknown", - "declared_license_expression_spdx":null, + "declared_license_expression":"apache-2.0 and unknown", + "declared_license_expression_spdx":"Apache-2.0 AND LicenseRef-scancode-unknown", "license_detections":[], "other_license_expression":null, "other_license_expression_spdx":null, diff --git a/minecode/tests/testfiles/maven/pom/pulsar-client-merged-ancestor-package.json b/minecode/tests/testfiles/maven/pom/pulsar-client-merged-ancestor-package.json index de037417..1966ac59 100644 --- a/minecode/tests/testfiles/maven/pom/pulsar-client-merged-ancestor-package.json +++ b/minecode/tests/testfiles/maven/pom/pulsar-client-merged-ancestor-package.json @@ -25,7 +25,7 @@ "copyright":null, "holder":null, "declared_license_expression":"apache-2.0", - "declared_license_expression_spdx":null, + "declared_license_expression_spdx":"Apache-2.0", "license_detections":[], "other_license_expression":null, "other_license_expression_spdx":null, diff --git a/minecode/tests/testfiles/model_utils/expected_updated_fields.json b/minecode/tests/testfiles/model_utils/expected_updated_fields.json index 0ef6a17d..2a49eb5d 100644 --- a/minecode/tests/testfiles/model_utils/expected_updated_fields.json +++ b/minecode/tests/testfiles/model_utils/expected_updated_fields.json @@ -79,11 +79,6 @@ "old_value":null, "new_value":"apache-2.0" }, - { - "field":"declared_license_expression_spdx", - "old_value":null, - "new_value":"Apache-2.0" - }, { "field":"license_detections", "old_value":[], diff --git a/packagedb/serializers.py b/packagedb/serializers.py index 2b8a9afd..7dbef5f5 100644 --- a/packagedb/serializers.py +++ b/packagedb/serializers.py @@ -167,6 +167,8 @@ class PackageAPISerializer(HyperlinkedModelSerializer): url = HyperlinkedIdentityField(view_name='api:package-detail', lookup_field='uuid') package_sets = PackageSetAPISerializer(many=True) package_content = SerializerMethodField() + declared_license_expression_spdx = CharField() + other_license_expression_spdx = CharField() class Meta: model = Package @@ -262,6 +264,8 @@ class PackageMetadataSerializer(ModelSerializer): parties = PartySerializer(many=True) package_sets = PackageSetMetadataSerializer(many=True) package_content = SerializerMethodField() + declared_license_expression_spdx = CharField() + other_license_expression_spdx = CharField() class Meta: model = Package diff --git a/packagedb/tests/testfiles/api/enhanced_package.json b/packagedb/tests/testfiles/api/enhanced_package.json index 65c3f279..f47d8637 100644 --- a/packagedb/tests/testfiles/api/enhanced_package.json +++ b/packagedb/tests/testfiles/api/enhanced_package.json @@ -24,7 +24,7 @@ "copyright":"Copyright (c) example corp.", "holder":null, "declared_license_expression":"apache-2.0", - "declared_license_expression_spdx":null, + "declared_license_expression_spdx":"Apache-2.0", "license_detections":[], "other_license_expression":null, "other_license_expression_spdx":null, diff --git a/packagedb/tests/testfiles/api/twill-core-0.12.0.json b/packagedb/tests/testfiles/api/twill-core-0.12.0.json index 8df2a560..0c1c60fa 100644 --- a/packagedb/tests/testfiles/api/twill-core-0.12.0.json +++ b/packagedb/tests/testfiles/api/twill-core-0.12.0.json @@ -37,7 +37,7 @@ "copyright":null, "holder":null, "declared_license_expression":"apache-2.0", - "declared_license_expression_spdx":null, + "declared_license_expression_spdx":"Apache-2.0", "license_detections":[], "other_license_expression":null, "other_license_expression_spdx":null,