Add package LicenseDetection [WIP]

Return LicenseDetection in packages and rename license attribute
names.

Reference: #2065

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

src/licensedcode/detection.py

@@ -163,8 +163,6 @@ def from_matches(cls, matches):
         if license_expression == None:
             return cls(matches=matches)
 
-
-
         return cls(
             matches=matches,
             license_expression=str(license_expression),
@@ -894,20 +892,35 @@ def combine_matches_in_detections(matches):
     return detections
 
 
-def detect_licenses(location, min_score, deadline, **kwargs):
+def detect_licenses(min_score, deadline, location=None, query_string=None, as_expression=False, **kwargs):
     """
     Yield LicenseDetection objects for licenses detected in the file at
     `location`.
     """
+    if location and query_string:
+        raise Exception("Either location or query_string should be provided")
+
     from licensedcode import cache
     idx = cache.get_index()
 
-    matches = idx.match(
-        location=location,
-        min_score=min_score,
-        deadline=deadline,
-        **kwargs,
-    )
+    if location:
+        matches = idx.match(
+            location=location,
+            min_score=min_score,
+            deadline=deadline,
+            as_expression=as_expression,
+            **kwargs,
+        )
+    elif query_string:
+        matches = idx.match(
+            query_string=query_string,
+            min_score=min_score,
+            deadline=deadline,
+            as_expression=as_expression,
+            **kwargs,
+        )
+    else:
+        return
 
     if not matches:
         return

src/packagedcode/about.py

@@ -76,8 +76,7 @@ def parse(cls, location):
         download_url = package_data.get('download_url')
         copyright_statement = package_data.get('copyright')
 
-        license_expression = package_data.get('license_expression')
-        declared_license = license_expression
+        declared_license_expression = package_data.get('license_expression')
 
         owner = package_data.get('owner')
         if not isinstance(owner, str):
@@ -91,21 +90,31 @@ def parse(cls, location):
             file_references.append(models.FileReference(path=about_resource))
 
         # FIXME: we should put the unprocessed attributes in extra data
-        yield models.PackageData(
+        package_data = models.PackageData(
             datasource_id=cls.datasource_id,
             type=package_type,
             namespace=package_ns,
             name=name,
             version=version,
-            declared_license=declared_license,
-            license_expression=license_expression,
+            extracted_license_statement=declared_license_expression,
+            declared_license_expression=declared_license_expression,
             copyright=copyright_statement,
             parties=parties,
             homepage_url=homepage_url,
             download_url=download_url,
             file_references=file_references,
         )
 
+        package_data.declared_license_expression_spdx = package_data.get_declared_license_expression_spdx(
+            declared_license_expression=declared_license_expression,
+        )
+        package_data.license_detections = package_data.get_license_detections(
+            extracted_license_statement=declared_license_expression,
+            as_expression=True,
+        )
+
+        yield package_data
+
     @classmethod
     def assemble(cls, package_data, resource, codebase):
         """
@@ -125,8 +134,9 @@ def assemble(cls, package_data, resource, codebase):
             resource.for_packages.append(package_uid)
             resource.save(codebase)
 
-            if not package.license_expression:
-                package.license_expression = cls.compute_normalized_license(package)
+            if not package.declared_license_expression:
+                package.declared_license_expression = cls.compute_normalized_license(package)
+                package.license_detections = cls.detect_licenses(package)
 
             yield package
 

src/packagedcode/alpine.py

@@ -74,7 +74,7 @@ def assemble(cls, package_data, resource, codebase):
         )
         package_uid = package.package_uid
 
-        package.license_expression = cls.compute_normalized_license(package)
+        package.declared_license_expression = cls.compute_normalized_license(package)
 
         dependent_packages = package_data.dependencies
         if dependent_packages:

src/packagedcode/bower.py

@@ -39,14 +39,14 @@ def parse(cls, location):
 
         description = package_data.get('description')
         version = package_data.get('version')
-        declared_license = package_data.get('license')
-        if declared_license:
-            if isinstance(declared_license, str):
-                declared_license = [declared_license]
-            elif isinstance(declared_license, (list, tuple)):
-                declared_license = [l for l in declared_license if l and l.strip()]
+        extracted_license_statement = package_data.get('license')
+        if extracted_license_statement:
+            if isinstance(extracted_license_statement, str):
+                extracted_license_statement = [extracted_license_statement]
+            elif isinstance(extracted_license_statement, (list, tuple)):
+                extracted_license_statement = [l for l in extracted_license_statement if l and l.strip()]
             else:
-                declared_license = [repr(declared_license)]
+                extracted_license_statement = [repr(extracted_license_statement)]
 
         keywords = package_data.get('keywords') or []
 
@@ -100,20 +100,32 @@ def parse(cls, location):
                 )
             )
 
-        yield models.PackageData(
+        package_data = models.PackageData(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
             name=name,
             description=description,
             version=version,
-            declared_license=declared_license,
+            extracted_license_statement=extracted_license_statement,
             keywords=keywords,
             parties=parties,
             homepage_url=homepage_url,
             vcs_url=vcs_url,
             dependencies=dependencies
         )
 
+        package_data.declared_license_expression = models.compute_normalized_license(
+            extracted_license_statement=extracted_license_statement
+        )
+        package_data.declared_license_expression_spdx = package_data.get_declared_license_expression_spdx(
+            declared_license_expression=package_data.declared_license_expression
+        )
+        package_data.license_detections = package_data.get_license_detections(
+            extracted_license_statement=extracted_license_statement
+        )
+
+        yield package_data
+
 
 def compute_bower_normalized_license(declared_license):
     """

src/packagedcode/build.py

@@ -89,8 +89,8 @@ def assemble(cls, package_data, resource, codebase):
                 datafile_path=datafile_path,
             )
 
-            if not package.license_expression:
-                package.license_expression = compute_normalized_license(
+            if not package.declared_license_expression:
+                package.declared_license_expression = compute_normalized_license(
                     package=package,
                     resource=resource,
                     codebase=codebase,

src/packagedcode/cargo.py

@@ -75,7 +75,7 @@ def parse(cls, location):
             primary_language=cls.default_primary_language,
             description=description,
             parties=parties,
-            declared_license=declared_license,
+            extracted_license_statement=extracted_license_statement,
             vcs_url=vcs_url,
             homepage_url=homepage_url,
             repository_homepage_url=repository_homepage_url,

src/packagedcode/chef.py

@@ -269,8 +269,8 @@ def build_package(package_data, datasource_id):
         version=version,
         parties=parties,
         description=description.strip() or None,
-        declared_license=declared_license,
-        license_expression=license_expression,
+        extracted_license_statement=extracted_license_statement,
+        declared_license_expression=declared_license_expression,
         code_view_url=code_view_url.strip() or None,
         bug_tracking_url=bug_tracking_url.strip() or None,
         dependencies=dependencies,

src/packagedcode/cocoapods.py

@@ -247,8 +247,8 @@ def parse(cls, location):
             # FIXME: a source should be a PURL, not a list of URLs
             # source_packages=vcs_url.split('\n'),
             description=description,
-            declared_license=declared_license,
-            license_expression=license_expression,
+            extracted_license_statement=extracted_license_statement,
+            declared_license_expression=declared_license_expression,
             homepage_url=homepage_url,
             parties=parties,
             **urls,
@@ -416,8 +416,8 @@ def parse(cls, location):
             name=name,
             version=version,
             description=description,
-            declared_license=declared_license,
-            license_expression=license_expression,
+            extracted_license_statement=extracted_license_statement,
+            declared_license_expression=declared_license_expression,
             parties=parties,
             vcs_url=vcs_url,
             homepage_url=homepage_url,

src/packagedcode/conda.py

@@ -102,7 +102,7 @@ def parse(cls, location):
         package.homepage_url = about.get('home')
         package.declared_license = about.get('license')
         if package.declared_license:
-            package.license_expression = cls.compute_normalized_license(package)
+            package.declared_license_expression = cls.compute_normalized_license(package)
         package.description = about.get('summary')
         package.vcs_url = about.get('dev_url')
 

src/packagedcode/cran.py

@@ -103,8 +103,8 @@ def parse(cls, location):
             version=cran_desc.get('Version'),
             # TODO: combine both together
             description=cran_desc.get('Description', '') or cran_desc.get('Title', ''),
-            declared_license=declared_license,
-            license_expression=license_expression,
+            extracted_license_statement=extracted_license_statement,
+            declared_license_expression=declared_license_expression,
             parties=parties,
             dependencies=package_dependencies,
             repository_homepage_url=f'https://cran.r-project.org/package={name}',

src/packagedcode/freebsd.py

@@ -94,8 +94,8 @@ def parse(cls, location):
         # license_mapper needs multiple fields
         license_mapper(freebsd_manifest, package_data)
 
-        if package_data.declared_license:
-            package_data.license_expression = cls.compute_normalized_license(package_data)
+        if package_data.extracted_license_statement:
+            package_data.declared_license_expression = cls.compute_normalized_license(package_data)
 
         yield package_data
 
@@ -110,7 +110,7 @@ def compute_normalized_license(cls, package):
             return
 
         if not isinstance(declared_license, dict):
-            return models.compute_normalized_license(declared_license=declared_license)
+            return models.compute_normalized_license(extracted_license_statement=extracted_license_statement)
 
         licenses = declared_license.get('licenses')
         if not licenses:

src/packagedcode/haxe.py

@@ -80,8 +80,8 @@ def parse(cls, location):
             primary_language=cls.default_primary_language,
         )
 
-        if not package_data.license_expression and package_data.declared_license:
-            package_data.license_expression = cls.compute_normalized_license(package_data)
+        if not package_data.declared_license_expression and package_data.extracted_license_statement:
+            package_data.declared_license_expression = cls.compute_normalized_license(package_data)
 
         if name and version:
             download_url = f'https://lib.haxe.org/p/{name}/{version}/download/'

src/packagedcode/jar_manifest.py

@@ -40,8 +40,8 @@ def parse(cls, location):
             if manifest:
                 package_data = models.PackageData(**manifest,)
 
-                if not package_data.license_expression and package_data.declared_license:
-                    package_data.license_expression = cls.compute_normalized_license(package_data)
+                if not package_data.declared_license_expression and package_data.extracted_license_statement:
+                    package_data.declared_license_expression = cls.compute_normalized_license(package_data)
 
                 yield package_data
 

src/packagedcode/maven.py

@@ -1171,16 +1171,16 @@ def parse(
         qualifiers=qualifiers or None,
         description=description or None,
         homepage_url=pom.url or None,
-        declared_license=declared_license or None,
+        extracted_license_statement=extracted_license_statement or None,
         parties=get_parties(pom),
         dependencies=get_dependencies(pom),
         source_packages=source_packages,
         bug_tracking_url=bug_tracking_url,
         **urls,
     )
 
-    if not package_data.license_expression and package_data.declared_license:
-        package_data.license_expression = models.compute_normalized_license(package_data.declared_license)
+    if not package_data.declared_license_expression and package_data.extracted_license_statement:
+        package_data.declared_license_expression = models.compute_normalized_license(package_data.extracted_license_statement)
 
     yield package_data