Generic on demand handler

matchcode/tests/testfiles/models/directory-matching/async-0.2.9-i-expected-content.json

@@ -28,6 +28,8 @@
           "version": "0.2.9",
           "qualifiers": "",
           "subpath": "",
+          "package_set": null,
+          "package_content": null,
           "primary_language": null,
           "description": null,
           "release_date": null,

matchcode/tests/testfiles/models/directory-matching/async-0.2.9-i-expected-structure.json

@@ -28,6 +28,8 @@
           "version": "0.2.9",
           "qualifiers": "",
           "subpath": "",
+          "package_set": null,
+          "package_content": null,
           "primary_language": null,
           "description": null,
           "release_date": null,

minecode/management/commands/process_scans.py

@@ -2,14 +2,13 @@
 # Copyright (c) 2018 by nexB, Inc. http://www.nexb.com/ - All rights reserved.
 #
 
-from collections import OrderedDict
 import logging
 import signal
 import sys
 
 from django.db import transaction
 
-from license_expression import Licensing
+from packagedcode.utils import combine_expressions
 
 from matchcode.models import ApproximateDirectoryContentIndex
 from matchcode.models import ApproximateDirectoryStructureIndex
@@ -80,20 +79,29 @@ def process_scan(cls, scannable_uri, get_scan_info_save_loc='', get_scan_data_sa
                 get_scan_data_save_loc=get_scan_data_save_loc
             )
 
-            package_updated = False
-            if summary:
-                package.summary = summary
-                package_updated = True
-
-            license_expression = summary.get('declared_license_expression')
-            if not package.declared_license_expression and license_expression:
-                package.declared_license_expression = license_expression
-                package_updated = True
+            other_license_expressions = summary.get('other_license_expressions', [])
+            other_license_expressions = [l['value'] for l in other_license_expressions]
+            other_license_expression = combine_expressions(other_license_expressions)
 
+            copyright = ''
             declared_holder = summary.get('declared_holder')
-            if not package.copyright:
-                if declared_holder:
-                    package.copyright = f'Copyright (c) {declared_holder}'
+            if declared_holder:
+                copyright = f'Copyright (c) {declared_holder}'
+
+            values_by_updateable_fields = {
+                'sha1': scan_info.sha1,
+                'sha256': scan_info.sha256,
+                'sha512': scan_info.sha512,
+                'summary': summary,
+                'declared_license_expression': summary.get('declared_license_expression'),
+                'other_license_expression': other_license_expression,
+                'copyright': copyright,
+            }
+
+            for field, value in values_by_updateable_fields.items():
+                p_val = getattr(package, field)
+                if not p_val and value:
+                    setattr(package, field, value)
                     package_updated = True
 
             if package_updated:
@@ -255,43 +263,3 @@ def index_package_files(package, scan_data):
         logger.error(msg)
 
     return scan_index_errors
-
-
-# TODO: Remove this when scancode-toolkit is upgraded. The current version of
-# scancode-toolkit in Minecode does not have this function
-# TODO: from packagedcode.utils import combine_expressions
-def combine_expressions(expressions, relation='AND', licensing=Licensing()):
-    """
-    Return a combined license expression string with relation, given a list of
-    license expressions strings.
-
-    For example:
-    >>> a = 'mit'
-    >>> b = 'gpl'
-    >>> combine_expressions([a, b])
-    'mit AND gpl'
-    >>> assert 'mit' == combine_expressions([a])
-    >>> combine_expressions([])
-    >>> combine_expressions(None)
-    >>> combine_expressions(('gpl', 'mit', 'apache',))
-    'gpl AND mit AND apache'
-    """
-    if not expressions:
-        return
-
-    if not isinstance(expressions, (list, tuple)):
-        raise TypeError(
-            'expressions should be a list or tuple and not: {}'.format(
-                type(expressions)))
-
-    # Remove duplicate element in the expressions list
-    expressions = list(OrderedDict((x, True) for x in expressions).keys())
-
-    if len(expressions) == 1:
-        return expressions[0]
-
-    expressions = [licensing.parse(le, simple=True) for le in expressions]
-    if relation == 'OR':
-        return str(licensing.OR(*expressions))
-    else:
-        return str(licensing.AND(*expressions))

minecode/management/scanning.py

@@ -74,9 +74,15 @@ class Scan(object):
     status = attr.ib(default=None)
     # as a time stamp
     execution_time = attr.ib(default=None)
+    md5 = attr.ib(default=None)
+    sha1 = attr.ib(default=None)
+    sha256 = attr.ib(default=None)
+    sha512 = attr.ib(default=None)
+    sha1_git = attr.ib(default=None)
+    filename = attr.ib(default=None)
 
     @classmethod
-    def from_response(cls, url, uuid, runs, input_sources, **kwargs):
+    def from_response(cls, url, uuid, runs, input_sources, extra_data={}, **kwargs):
         """
         Return a Scan object built from an API response data arguments.
         """
@@ -95,11 +101,20 @@ def from_response(cls, url, uuid, runs, input_sources, **kwargs):
         if len(input_sources) > 0:
             uri = input_sources[0]["source"]
 
+        md5 = extra_data.get('md5')
+        sha1 = extra_data.get('sha1')
+        sha256 = extra_data.get('sha256')
+        sha512 = extra_data.get('sha512')
+        sha1_git = extra_data.get('sha1_git')
+        filename = extra_data.get('filename')
+
         return Scan(
             url=url, uuid=uuid, run_uuid=run_uuid, uri=uri,
             created_date=created_date, task_start_date=task_start_date,
             task_end_date=task_end_date, task_exitcode=task_exitcode,
-            status=status, execution_time=execution_time
+            status=status, execution_time=execution_time,
+            md5=md5, sha1=sha1, sha256=sha256, sha512=sha512,
+            sha1_git=sha1_git, filename=filename
         )
 
     @property
@@ -243,20 +258,39 @@ def _call_scan_get_api(scan_uuid, endpoint='',
     return response.json()
 
 
-def get_scan_info(
+def _get_scan_info(
     scan_uuid,
     api_url=SCANCODEIO_API_URL_PROJECTS,
     api_auth_headers=SCANCODEIO_AUTH_HEADERS,
     get_scan_info_save_loc=''
 ):
     """
-    Return a Scan object for `scan_uuid` fetched from ScanCode.io or None.
+    Return a mapping of project info for `scan_uuid` fetched from ScanCode.io or None.
     Raise an exception on error.
     """
     results = _call_scan_get_api(scan_uuid, endpoint='', api_url=api_url, api_auth_headers=api_auth_headers)
     if get_scan_info_save_loc:
         with open(get_scan_info_save_loc, 'w') as f:
             json.dump(results, f)
+    return results
+
+
+def get_scan_info(
+    scan_uuid,
+    api_url=SCANCODEIO_API_URL_PROJECTS,
+    api_auth_headers=SCANCODEIO_AUTH_HEADERS,
+    get_scan_info_save_loc=''
+):
+    """
+    Return a Scan object for `scan_uuid` fetched from ScanCode.io or None.
+    Raise an exception on error.
+    """
+    results = _get_scan_info(
+        scan_uuid=scan_uuid,
+        api_url=api_url,
+        api_auth_headers=api_auth_headers,
+        get_scan_info_save_loc=get_scan_info_save_loc,
+    )
     return Scan.from_response(**results)
 
 

minecode/tests/test_generic.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import os
+
+from django.test import TestCase as DjangoTestCase
+
+from minecode.utils_test import JsonBasedTesting
+from minecode.visitors import generic
+from packagedb.models import Package
+
+from packagedcode.maven import _parse
+from packageurl import PackageURL
+
+
+class GenericPriorityQueueTests(JsonBasedTesting, DjangoTestCase):
+    def test_process_request(self):
+        package_count = Package.objects.all().count()
+        self.assertEqual(0, package_count)
+
+        purl = 'pkg:generic/test@1.0.0?download_url=http://example.com/test.tar.gz'
+        error_msg = generic.process_request(purl)
+
+        self.assertEqual(None, error_msg)
+        package_count = Package.objects.all().count()
+        self.assertEqual(1, package_count)
+
+        package = Package.objects.first()
+        self.assertEqual('test', package.name)
+        self.assertEqual('1.0.0', package.version)
+        self.assertEqual('http://example.com/test.tar.gz', package.download_url)
+
+    def test_process_request_no_download_url(self):
+        package_count = Package.objects.all().count()
+        self.assertEqual(0, package_count)
+
+        purl = 'pkg:generic/test@1.0.0'
+        error_msg = generic.process_request(purl)
+
+        self.assertEqual(
+            f'package_url {purl} does not contain a download_url qualifier',
+            error_msg
+        )
+        package_count = Package.objects.all().count()
+        self.assertEqual(0, package_count)
+
+    def test_map_generic_package(self):
+        package_count = Package.objects.all().count()
+        self.assertEqual(0, package_count)
+
+        purl = 'pkg:generic/test@1.0.0?download_url=http://example.com/test.tar.gz'
+        package_url = PackageURL.from_string(purl)
+        error_msg = generic.map_generic_package(package_url)
+
+        self.assertEqual('', error_msg)
+        package_count = Package.objects.all().count()
+        self.assertEqual(1, package_count)
+
+        package = Package.objects.first()
+        self.assertEqual('test', package.name)
+        self.assertEqual('1.0.0', package.version)
+        self.assertEqual('http://example.com/test.tar.gz', package.download_url)

minecode/tests/test_maven.py

@@ -312,7 +312,7 @@ def test_MavenNexusIndexVisitor_with_run_visit_then_map_end2end(self):
 
         package_results = list(pac.to_dict() for pac in newly_mapped)
         expected_loc = self.get_test_loc('maven/end2end/expected_mapped_packages.json')
-        self.check_expected_results(package_results, expected_loc, regen=False)
+        self.check_expected_results(package_results, expected_loc, fields_to_remove=['package_set'], regen=False)
 
         # check that the map status has been updated correctly
         visited_then_mapped = ResourceURI.objects.filter(uri__contains='maven-index://')
@@ -372,7 +372,7 @@ def test_visit_and_map_using_pom_with_unicode_multisteps(self):
 
         package_results = sorted((pac.to_dict() for pac in mapped), key=lambda d: list(d.keys()))
         expected_loc = self.get_test_loc('maven/end2end_multisteps/expected_mapped_commons-jaxrs-1.21-from-index.json')
-        self.check_expected_results(package_results, expected_loc, regen=False)
+        self.check_expected_results(package_results, expected_loc, fields_to_remove=['package_set'], regen=False)
 
         # Step 2: map a POM
 
@@ -389,7 +389,7 @@ def test_visit_and_map_using_pom_with_unicode_multisteps(self):
 
         package_results = sorted((pac.to_dict() for pac in mapped), key=lambda d: list(d.keys()))
         expected_loc = self.get_test_loc('maven/end2end_multisteps/expected_mapped_commons-jaxrs-1.21-from-pom.json')
-        self.check_expected_results(package_results, expected_loc, regen=False)
+        self.check_expected_results(package_results, expected_loc, fields_to_remove=['package_set'], regen=False)
 
     def test_visit_and_map_with_index(self):
         uri = 'https://repo1.maven.org/maven2/.index/nexus-maven-repository-index.properties'

minecode/tests/test_rubygems.py

@@ -299,4 +299,4 @@ def test_visit_and_map_end2end(self):
 
         package_results = [pac.to_dict() for pac in mapped]
         expected_loc = self.get_test_loc('rubygems/sprockets-vendor_gems-0.1.3.gem.mapped.json')
-        self.check_expected_results(package_results, expected_loc, regen=False)
+        self.check_expected_results(package_results, expected_loc, fields_to_remove=['package_set'], regen=False)

minecode/tests/test_scanning.py

@@ -144,6 +144,12 @@ def testscanning_get_scan_info(self, mock_get):
             task_exitcode=0,
             status='success',
             execution_time=9,
+            md5='57431f2f6d5841eebdb964b04091b8ed',
+            sha1='feff0d7bacd11d37a9c96daed87dc1db163065b1',
+            sha256='05155c2c588ac5922d930eeb1e8a1da896956f4696ae758d110708e9f095baba',
+            sha512='4431f237bcdfee5d2b86b1b3f01c8abaa160d5b7007c63e6281845a3f920d89fdb2e4044f97694ddef91e174d9dd30e5016bbad46eec2d68af200a47e9cedd85',
+            sha1_git='ad18d88bdae8449e7c170f8e7db1bfe336dbb4e0',
+            filename='wagon-api-20040705.181715.jar',
         )
         expected = attr.asdict(expected)
         result = attr.asdict(result)

minecode/tests/testfiles/directories/find-ls-expected.json

@@ -10,7 +10,7 @@
     "path":"groovy/.revision",
     "type":"f",
     "size":6,
-    "date":"2022-05",
+    "date":"2023-05",
     "target":null
   },
   {

minecode/tests/testfiles/housekeeping/declared_license_search_expected.json

@@ -6,6 +6,8 @@
     "version":"",
     "qualifiers":"",
     "subpath":"",
+    "package_set":null,
+    "package_content":null,
     "primary_language":null,
     "description":null,
     "release_date":null,

minecode/tests/testfiles/housekeeping/example_expected.json

@@ -6,6 +6,8 @@
     "version":"",
     "qualifiers":"",
     "subpath":"",
+    "package_set":null,
+    "package_content":null,
     "primary_language":null,
     "description":null,
     "release_date":null,

minecode/tests/testfiles/housekeeping/ignore_upper_case_search_expected.json

@@ -6,6 +6,8 @@
     "version":"",
     "qualifiers":"",
     "subpath":"",
+    "package_set":null,
+    "package_content":null,
     "primary_language":null,
     "description":null,
     "release_date":null,

minecode/tests/testfiles/housekeeping/license_expression_search_expected.json

@@ -6,6 +6,8 @@
     "version":"",
     "qualifiers":"",
     "subpath":"",
+    "package_set":null,
+    "package_content":null,
     "primary_language":null,
     "description":null,
     "release_date":null,