Merge pull request #3299 from nexB/license-detection-follow-up

Fix misc license detection related bugs

src/licensedcode/detection.py

@@ -522,7 +522,7 @@ def detections_from_license_detection_mappings(
     return license_detections
 
 
-def get_new_identifier_from_detections(initial_detection, detections_added):
+def get_new_identifier_from_detections(initial_detection, detections_added, license_expression):
     """
     Return a new UUID based on two sets of detections: `initial_detection` is
     the detection being modified with a list of detections (from another file region)
@@ -533,7 +533,8 @@ def get_new_identifier_from_detections(initial_detection, detections_added):
         for detection_mapping in detections_added
     ]
     identifiers.append(initial_detection["identifier"])
-    return get_uuid_on_content(content=sorted(identifiers))
+    uuid = get_uuid_on_content(content=sorted(identifiers))
+    return f"{license_expression}-{uuid}"
 
 
 @attr.s

src/licensedcode/plugin_license.py

@@ -354,6 +354,7 @@ def add_referenced_filenames_license_matches_for_detections(resource, codebase):
         license_detection_mapping["identifier"] = get_new_identifier_from_detections(
             initial_detection=license_detection_mapping,
             detections_added=detections_added,
+            license_expression=license_expression,
         )
 
     if modified:

src/licensedcode/plugin_license_policy.py

@@ -49,7 +49,7 @@ class LicensePolicy(PostScanPlugin):
     detected license key that is found in the license_policy.yml file
     """
 
-    resource_attributes = dict(license_policy=attr.ib(default=attr.Factory(dict)))
+    resource_attributes = dict(license_policy=attr.ib(default=attr.Factory(list)))
 
     sort_order = 9
 
@@ -90,16 +90,19 @@ def process_codebase(self, codebase, license_policy, **kwargs):
 
             except AttributeError:
                 # add license_policy regardless if there is license info or not
-                resource.license_policy = {}
+                resource.license_policy = []
                 codebase.save_resource(resource)
                 continue
 
+            license_policies = []
             for key in resource_license_keys:
                 for policy in policies:
                     if key == policy.get('license_key'):
                         # Apply the policy to the Resource
-                        resource.license_policy = policy
-                        codebase.save_resource(resource)
+                        license_policies.append(policy)
+
+            resource.license_policy = sorted(license_policies, key=lambda d: d['license_key']) 
+            codebase.save_resource(resource)
 
 
 def has_policy_duplicates(license_policy_location):

src/packagedcode/licensing.py

@@ -136,6 +136,7 @@ def add_referenced_license_matches_for_package(resource, codebase, no_licenses):
             license_detection_mapping["identifier"] = get_new_identifier_from_detections(
                 initial_detection=license_detection_mapping,
                 detections_added=referenced_license_detections,
+                license_expression=license_expression,
             )
 
         if modified:
@@ -252,6 +253,7 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
         license_detection_mapping["identifier"] = get_new_identifier_from_detections(
             initial_detection=license_detection_mapping,
             detections_added=detections_added,
+            license_expression=license_expression,
         )
 
     if modified:

src/scancode_config.py

@@ -95,7 +95,7 @@ def _create_dir(location):
     from subprocess import CalledProcessError
 
     # this may fail with exceptions
-    cmd = 'git', 'describe',
+    cmd = 'git', 'describe', '--tags',
     try:
         output = check_output(cmd, stderr=STDOUT)
         __version__ = output.decode('utf-8').strip()

tests/formattedcode/data/csv/expressions/expected.csv

@@ -1,2 +1,2 @@
-path,type,declared_license_expression,scan_errors
-apache-2.0.LICENSE,file,apache-2.0 AND mit,
+path,type,declared_license_expression,declared_license_expression_spdx,scan_errors
+apache-2.0.LICENSE,file,apache-2.0 AND mit,Apache-2.0 AND MIT,

tests/formattedcode/data/csv/expressions/scan.json

@@ -5,6 +5,8 @@
       "type": "file",
       "license_detections": [],
       "declared_license_expression": "apache-2.0 AND mit",
+      "declared_license_expression_spdx": "Apache-2.0 AND MIT",
+      "license_clues": [],
       "scan_errors": []
     }
   ]

tests/formattedcode/test_output_csv.py

@@ -124,6 +124,8 @@ def test_can_process_live_scan_for_packages_with_root():
 
 
 def test_output_can_handle_non_ascii_paths():
+    # very small file with selective attributes just to test unicode, no need
+    # run scan again, update manually if required
     test_file = test_env.get_test_loc('csv/unicode.json')
     result_file = test_env.get_temp_file(extension='csv', file_name='test_csv')
     run_scan_click(['--from-json', test_file, '--csv', result_file])
@@ -250,6 +252,8 @@ def test_can_process_live_scan_for_packages_strip_root():
 
 @pytest.mark.scanslow
 def test_output_contains_license_expression():
+    # very small file with selective attributes just to test license_expressions,
+    # no need to run scan again, update manually if required
     test_file = test_env.get_test_loc('csv/expressions/scan.json')
     result_file = test_env.get_temp_file('csv')
     args = ['--from-json', test_file, '--csv', result_file]
@@ -260,6 +264,7 @@ def test_output_contains_license_expression():
 
 @pytest.mark.scanslow
 def test_output_handles_non_standard_data():
+    # non standard data from other tool, update manually if required
     test_file = test_env.get_test_loc('csv/non-standard/identified.json')
     result_file = test_env.get_temp_file('csv')
     args = ['--from-json', test_file, '--csv', result_file]

tests/licensedcode/data/plugin_license/license_reference/license-ref-see-copying.expected.json

@@ -84,7 +84,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "1c807a43-2040-70af-75aa-c343d5f2b90c"
+          "identifier": "apache-2.0-1c807a43-2040-70af-75aa-c343d5f2b90c"
         }
       ],
       "license_clues": [],

tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json

@@ -84,7 +84,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "20c01557-97bd-0022-052e-56c5ed8465ea"
+          "identifier": "mit-20c01557-97bd-0022-052e-56c5ed8465ea"
         }
       ],
       "license_clues": [],

tests/licensedcode/data/plugin_license/license_reference/scan-unknown-reference-copyright.expected.json

@@ -90,7 +90,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "61f804f6-d484-92ca-09b5-26be51ac974e"
+          "identifier": "x11-xconsortium-veillard-61f804f6-d484-92ca-09b5-26be51ac974e"
         }
       ],
       "license_clues": [],
@@ -136,7 +136,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "61f804f6-d484-92ca-09b5-26be51ac974e"
+          "identifier": "x11-xconsortium-veillard-61f804f6-d484-92ca-09b5-26be51ac974e"
         }
       ],
       "license_clues": [],
@@ -202,7 +202,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "50c015bd-e4e1-c6fe-eb82-9551473dd8e1"
+          "identifier": "x11-xconsortium-veillard-50c015bd-e4e1-c6fe-eb82-9551473dd8e1"
         }
       ],
       "license_clues": [],

tests/licensedcode/data/plugin_license/license_reference/unknown-ref-to-key-file-root.expected.json

@@ -165,7 +165,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "e46a912c-b32a-30d5-dc27-c13824253230"
+          "identifier": "mit-e46a912c-b32a-30d5-dc27-c13824253230"
         }
       ],
       "license_clues": [],
@@ -224,7 +224,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "e46a912c-b32a-30d5-dc27-c13824253230"
+          "identifier": "mit-e46a912c-b32a-30d5-dc27-c13824253230"
         }
       ],
       "license_clues": [],
@@ -283,7 +283,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "e46a912c-b32a-30d5-dc27-c13824253230"
+          "identifier": "mit-e46a912c-b32a-30d5-dc27-c13824253230"
         }
       ],
       "license_clues": [],
@@ -404,7 +404,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "ad99a349-2a14-9fe5-c6a6-366fd3b9067b"
+          "identifier": "mit-ad99a349-2a14-9fe5-c6a6-366fd3b9067b"
         }
       ],
       "license_clues": [],
@@ -473,7 +473,7 @@
           "detection_log": [
             "unknown-reference-to-local-file"
           ],
-          "identifier": "e46a912c-b32a-30d5-dc27-c13824253230"
+          "identifier": "mit-e46a912c-b32a-30d5-dc27-c13824253230"
         }
       ],
       "license_clues": [],

tests/licensedcode/data/plugin_license_policy/file_with_multiple_licenses.expected.json

@@ -0,0 +1,85 @@
+{
+  "license_detections": [
+    {
+      "identifier": "apache_2_0_and_gpl_3_0-494ca0ae-1282-09a2-139f-a52c04fde6dc",
+      "license_expression": "apache-2.0 AND gpl-3.0",
+      "detection_count": 1
+    }
+  ],
+  "files": [
+    {
+      "path": "file_with_multiple_licenses.txt",
+      "type": "file",
+      "name": "file_with_multiple_licenses.txt",
+      "base_name": "file_with_multiple_licenses",
+      "extension": ".txt",
+      "size": 38,
+      "sha1": "92bb9a9fbe881166541fb395dc29501851405892",
+      "md5": "1dcb636e0d184b1c85a47245ec6ef8ca",
+      "sha256": "1f9e9d483e71691f5d0a5aa2ad89b07cc7f52ed0ff18b9ab641527c6c2974203",
+      "mime_type": "text/plain",
+      "file_type": "ASCII text",
+      "programming_language": null,
+      "is_binary": false,
+      "is_text": true,
+      "is_archive": false,
+      "is_media": false,
+      "is_source": false,
+      "is_script": false,
+      "detected_license_expression": "apache-2.0 AND gpl-3.0",
+      "detected_license_expression_spdx": "Apache-2.0 AND GPL-3.0-only",
+      "license_detections": [
+        {
+          "license_expression": "apache-2.0 AND gpl-3.0",
+          "matches": [
+            {
+              "score": 100.0,
+              "start_line": 1,
+              "end_line": 1,
+              "matched_length": 4,
+              "match_coverage": 100.0,
+              "matcher": "2-aho",
+              "license_expression": "apache-2.0",
+              "rule_identifier": "apache-2.0_65.RULE",
+              "rule_relevance": 100,
+              "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_65.RULE"
+            },
+            {
+              "score": 100.0,
+              "start_line": 3,
+              "end_line": 3,
+              "matched_length": 4,
+              "match_coverage": 100.0,
+              "matcher": "2-aho",
+              "license_expression": "gpl-3.0",
+              "rule_identifier": "gpl-3.0_173.RULE",
+              "rule_relevance": 100,
+              "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/gpl-3.0_173.RULE"
+            }
+          ],
+          "identifier": "apache_2_0_and_gpl_3_0-494ca0ae-1282-09a2-139f-a52c04fde6dc"
+        }
+      ],
+      "license_clues": [],
+      "percentage_of_license_text": 100.0,
+      "license_policy": [
+        {
+          "license_key": "apache-2.0",
+          "label": "Approved License",
+          "color_code": "#008000",
+          "icon": "icon-ok-circle"
+        },
+        {
+          "license_key": "gpl-3.0",
+          "label": "Restricted License",
+          "color_code": "#FFcc33",
+          "icon": "icon-warning-sign"
+        }
+      ],
+      "files_count": 0,
+      "dirs_count": 0,
+      "size_count": 0,
+      "scan_errors": []
+    }
+  ]
+}

tests/licensedcode/data/plugin_license_policy/file_with_multiple_licenses.txt

@@ -0,0 +1,3 @@
+License: apache-2.0
+
+License: gpl-3.0