Update spdx fields on packages #166

minecode/model_utils.py

@@ -67,7 +67,11 @@ def merge_packages(existing_package, new_package_data, replace=False):
 
     new_mapping = new_package_data
 
-    fields_to_skip = ('package_uid',)
+    fields_to_skip = (
+        'package_uid',
+        'declared_license_expression_spdx',
+        'other_license_expression_spdx',
+    )
     updated_fields = []
 
     for existing_field, existing_value in existing_mapping.items():
@@ -327,10 +331,8 @@ def merge_or_create_package(scanned_package, visit_level):
             copyright=scanned_package.copyright,
             holder=scanned_package.holder,
             declared_license_expression=scanned_package.declared_license_expression,
-            declared_license_expression_spdx=scanned_package.declared_license_expression_spdx,
             license_detections=scanned_package.license_detections,
             other_license_expression=scanned_package.other_license_expression,
-            other_license_expression_spdx=scanned_package.other_license_expression_spdx,
             other_license_detections=scanned_package.other_license_detections,
             extracted_license_statement=scanned_package.extracted_license_statement,
             notice_text=scanned_package.notice_text,

minecode/tests/test_housekeeping.py

@@ -36,7 +36,7 @@ def test_find_ambiguous_packages_declared_license(self):
         packagedb.models.Package.objects.create(
             download_url='http://example.com',
             name='Foo',
-            declared_license_expression='apache and unknown',
+            declared_license_expression='apache-2.0 and unknown',
             type='maven'
         )
         packages = [p.to_dict() for p in find_ambiguous_packages()]
@@ -47,7 +47,7 @@ def test_find_ambiguous_packages_license_expression(self):
         packagedb.models.Package.objects.create(
             download_url='http://example.com',
             name='Foo',
-            declared_license_expression='apache and unknown',
+            declared_license_expression='apache-2.0 and unknown',
             type='maven'
         )
         packages = [p.to_dict() for p in find_ambiguous_packages()]
@@ -72,7 +72,7 @@ def test_run_check_licenses_command(self):
         packagedb.models.Package.objects.create(
             download_url='http://example.com',
             name='Foo',
-            declared_license_expression='apache and unknown',
+            declared_license_expression='apache-2.0 and unknown',
             type='maven'
         )
         results_loc = self.get_temp_file()

minecode/tests/test_maven.py

@@ -830,7 +830,7 @@ def test_merge_ancestors(self, regen=False):
         maven_visitor.merge_ancestors(ancestor_pom_texts, package)
         expected_after_loc = self.get_test_loc('maven/pom/pulsar-client-1x-2.5.1-package_after.json')
         self.check_expected_results(package.to_dict(), expected_after_loc, regen=regen)
-    
+
     @mock.patch("minecode.visitors.maven.get_pom_text")
     def test_get_merged_ancestor_package_from_maven_package(self, get_pom_text_mock, regen=False):
         get_pom_text_mock.return_value = ""

minecode/tests/testfiles/housekeeping/declared_license_search_expected.json

@@ -25,8 +25,8 @@
     "vcs_url":null,
     "copyright":null,
     "holder":null,
-    "declared_license_expression":"apache and unknown",
-    "declared_license_expression_spdx":null,
+    "declared_license_expression":"apache-2.0 and unknown",
+    "declared_license_expression_spdx":"Apache-2.0 AND LicenseRef-scancode-unknown",
     "license_detections":[],
     "other_license_expression":null,
     "other_license_expression_spdx":null,

minecode/tests/testfiles/housekeeping/example_expected.json

@@ -25,8 +25,8 @@
     "vcs_url":null,
     "copyright":null,
     "holder":null,
-    "declared_license_expression":"apache and unknown",
-    "declared_license_expression_spdx":null,
+    "declared_license_expression":"apache-2.0 and unknown",
+    "declared_license_expression_spdx":"Apache-2.0 AND LicenseRef-scancode-unknown",
     "license_detections":[],
     "other_license_expression":null,
     "other_license_expression_spdx":null,

minecode/tests/testfiles/housekeeping/ignore_upper_case_search_expected.json

@@ -26,7 +26,7 @@
     "copyright":null,
     "holder":null,
     "declared_license_expression":"Unknown",
-    "declared_license_expression_spdx":null,
+    "declared_license_expression_spdx":"LicenseRef-scancode-unknown",
     "license_detections":[],
     "other_license_expression":null,
     "other_license_expression_spdx":null,

minecode/tests/testfiles/housekeeping/license_expression_search_expected.json

@@ -25,8 +25,8 @@
     "vcs_url":null,
     "copyright":null,
     "holder":null,
-    "declared_license_expression":"apache and unknown",
-    "declared_license_expression_spdx":null,
+    "declared_license_expression":"apache-2.0 and unknown",
+    "declared_license_expression_spdx":"Apache-2.0 AND LicenseRef-scancode-unknown",
     "license_detections":[],
     "other_license_expression":null,
     "other_license_expression_spdx":null,

minecode/tests/testfiles/maven/pom/pulsar-client-merged-ancestor-package.json

@@ -25,7 +25,7 @@
   "copyright":null,
   "holder":null,
   "declared_license_expression":"apache-2.0",
-  "declared_license_expression_spdx":null,
+  "declared_license_expression_spdx":"Apache-2.0",
   "license_detections":[],
   "other_license_expression":null,
   "other_license_expression_spdx":null,

minecode/tests/testfiles/model_utils/expected_updated_fields.json

@@ -79,11 +79,6 @@
     "old_value":null,
     "new_value":"apache-2.0"
   },
-  {
-    "field":"declared_license_expression_spdx",
-    "old_value":null,
-    "new_value":"Apache-2.0"
-  },
   {
     "field":"license_detections",
     "old_value":[],

packagedb/migrations/0077_remove_package_declared_license_expression_spdx_and_more.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.2 on 2023-08-08 23:55
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("packagedb", "0076_rename_history_json_package_history"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="package",
+            name="declared_license_expression_spdx",
+        ),
+        migrations.RemoveField(
+            model_name="package",
+            name="other_license_expression_spdx",
+        ),
+    ]

packagedb/models.py

@@ -21,6 +21,7 @@
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
+from licensedcode.cache import build_spdx_license_expression
 from packageurl import PackageURL
 from packageurl.contrib.django.models import PackageURLMixin
 from packageurl.contrib.django.models import PackageURLQuerySetMixin
@@ -57,7 +58,7 @@ def get_or_none(self, *args, **kwargs):
             return self.get(*args, **kwargs)
         except self.DoesNotExist:
             return
-    
+
     def paginated(self, per_page=5000):
         """
         Iterate over a (large) QuerySet by chunks of ``per_page`` items.
@@ -359,14 +360,6 @@ class AbstractPackage(models.Model):
             "routine or convention."
         ),
     )
-    declared_license_expression_spdx = models.TextField(
-        blank=True,
-        null=True,
-        help_text=_(
-            "The SPDX license expression for this package converted "
-            "from its declared_license_expression."
-        ),
-    )
     license_detections = models.JSONField(
         default=list,
         blank=True,
@@ -386,14 +379,6 @@ class AbstractPackage(models.Model):
             "routine or convention."
         ),
     )
-    other_license_expression_spdx = models.TextField(
-        blank=True,
-        null=True,
-        help_text=_(
-            "The other SPDX license expression for this package converted "
-            "from its other_license_expression."
-        ),
-    )
     other_license_detections = models.JSONField(
         default=list,
         blank=True,
@@ -442,6 +427,18 @@ class AbstractPackage(models.Model):
     class Meta:
         abstract = True
 
+    @property
+    def declared_license_expression_spdx(self):
+        declared_license_expression = self.declared_license_expression
+        if declared_license_expression:
+            return build_spdx_license_expression(declared_license_expression)
+
+    @property
+    def other_license_expression_spdx(self):
+        other_license_expression = self.other_license_expression
+        if other_license_expression:
+            return build_spdx_license_expression(other_license_expression)
+
 
 class PackageContentType(models.IntegerChoices):
     """List of Package content types."""

packagedb/serializers.py

@@ -167,6 +167,8 @@ class PackageAPISerializer(HyperlinkedModelSerializer):
     url = HyperlinkedIdentityField(view_name='api:package-detail', lookup_field='uuid')
     package_sets = PackageSetAPISerializer(many=True)
     package_content = SerializerMethodField()
+    declared_license_expression_spdx = CharField()
+    other_license_expression_spdx = CharField()
 
     class Meta:
         model = Package
@@ -262,6 +264,8 @@ class PackageMetadataSerializer(ModelSerializer):
     parties = PartySerializer(many=True)
     package_sets = PackageSetMetadataSerializer(many=True)
     package_content = SerializerMethodField()
+    declared_license_expression_spdx = CharField()
+    other_license_expression_spdx = CharField()
 
     class Meta:
         model = Package

packagedb/tests/testfiles/api/enhanced_package.json

@@ -24,7 +24,7 @@
   "copyright":"Copyright (c) example corp.",
   "holder":null,
   "declared_license_expression":"apache-2.0",
-  "declared_license_expression_spdx":null,
+  "declared_license_expression_spdx":"Apache-2.0",
   "license_detections":[],
   "other_license_expression":null,
   "other_license_expression_spdx":null,

packagedb/tests/testfiles/api/twill-core-0.12.0.json

@@ -37,7 +37,7 @@
   "copyright":null,
   "holder":null,
   "declared_license_expression":"apache-2.0",
-  "declared_license_expression_spdx":null,
+  "declared_license_expression_spdx":"Apache-2.0",
   "license_detections":[],
   "other_license_expression":null,
   "other_license_expression_spdx":null,