Skip to content

Commit

Permalink
define process_codebase function for matching references
Browse files Browse the repository at this point in the history 
Signed-off-by: akugarg <akanksha.garg2k@gmail.com>
  • Loading branch information
@akugarg
akugarg committed Aug 14, 2021
1 parent 6f751e9 commit d4f1436
Show file tree
Hide file tree
Showing 8 changed files with 381 additions and 79 deletions.
49 changes: 23 additions & 26 deletions src/licensedcode/plugin_license.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,6 @@ def match_reference_license(resource, codebase):
"""
Find instances for any licenses in referenced filenames
"""

if not resource.is_file:
return

Expand All @@ -142,37 +141,35 @@ def match_reference_license(resource, codebase):
if not licenses:
return

location = resource.location
from licensedcode import cache
from scancode.api import get_licenses
idx = cache.get_index()
matches = idx.match(
location=location, min_score=0)
reference_licenses=[]
reference_license_expressions=[]
unique_files = []

for match in matches:
ref_files=match.rule.referenced_filenames
for license in licenses:
ref_files=license['matched_rule']['referenced_filenames']
if len(ref_files) != 0:
for i in range(len(ref_files)):
if not ref_files[i].startswith('usr/share/common-licenses'):
new_loc=find_reference_file(location,ref_files[i])
if new_loc:
new_lic=get_licenses(new_loc, min_score=0)
licenses.extend(new_lic['licenses'])
license_expressions.extend(new_lic['license_expressions'])

if ref_files[i].startswith('usr/share/common-licenses') or ref_files[i] in unique_files:
continue
unique_files.append(ref_files[i])
new_resource = find_reference_licenses(ref_files[i], resource, codebase)
if new_resource:
reference_licenses.extend(new_resource.licenses)
reference_license_expressions.extend(new_resource.license_expressions)

licenses.extend(reference_licenses)
license_expressions.extend(reference_license_expressions)
codebase.save_resource(resource)
return resource


def find_reference_file(location,referenced_filename):
def find_reference_licenses(referenced_filename, resource, codebase):
"""
Searches for the referenced filename in the same directory and
returns the found location
Returns the resource which references to referenced_filename if found
"""
file_name=referenced_filename
par_dir=fileutils.parent_directory(location)
for root, dirs, files in os.walk(par_dir):
if file_name in files:
path_file = os.path.join(root,file_name)
return path_file

par = resource.parent(codebase)

for child in par.children(codebase):
file_name = fileutils.file_name(child.path)
if referenced_filename == file_name:
return child
14 changes: 13 additions & 1 deletion tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"short_name": "MIT License",
"category": "Permissive",
"is_exception": false,
"is_unknown": false,
"owner": "MIT",
"homepage_url": "http://opensource.org/licenses/mit-license.php",
"text_url": "http://opensource.org/licenses/mit-license.php",
Expand All @@ -46,11 +47,13 @@
"licenses": [
"mit"
],
"referenced_filenames": [],
"is_license_text": false,
"is_license_notice": true,
"is_license_reference": false,
"is_license_tag": false,
"is_license_intro": false,
"has_unknown": false,
"matcher": "1-hash",
"rule_length": 10,
"matched_length": 10,
Expand All @@ -77,6 +80,7 @@
"short_name": "Unknown License reference",
"category": "Unstated License",
"is_exception": false,
"is_unknown": true,
"owner": "Unspecified",
"homepage_url": null,
"text_url": "",
Expand All @@ -93,11 +97,15 @@
"licenses": [
"unknown-license-reference"
],
"referenced_filenames": [
"LICENSE"
],
"is_license_text": false,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": true,
"is_license_intro": false,
"has_unknown": true,
"matcher": "2-aho",
"rule_length": 5,
"matched_length": 5,
Expand All @@ -113,6 +121,7 @@
"short_name": "MIT License",
"category": "Permissive",
"is_exception": false,
"is_unknown": false,
"owner": "MIT",
"homepage_url": "http://opensource.org/licenses/mit-license.php",
"text_url": "http://opensource.org/licenses/mit-license.php",
Expand All @@ -129,17 +138,20 @@
"licenses": [
"mit"
],
"referenced_filenames": [],
"is_license_text": false,
"is_license_notice": true,
"is_license_reference": false,
"is_license_tag": false,
"is_license_intro": false,
"has_unknown": false,
"matcher": "1-hash",
"rule_length": 10,
"matched_length": 10,
"match_coverage": 100.0,
"rule_relevance": 100
}
},
"matched_text": "that is licensed under [MIT](http://opensource.org/licenses/MIT)."
}
],
"license_expressions": [
Expand Down
5 changes: 5 additions & 0 deletions tests/licensedcode/data/plugin_license/license_reference/scan-wref.expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"short_name": "Unknown License reference",
"category": "Unstated License",
"is_exception": false,
"is_unknown": true,
"owner": "Unspecified",
"homepage_url": null,
"text_url": "",
Expand All @@ -46,11 +47,15 @@
"licenses": [
"unknown-license-reference"
],
"referenced_filenames": [
"LICENSE"
],
"is_license_text": false,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": true,
"is_license_intro": false,
"has_unknown": true,
"matcher": "2-aho",
"rule_length": 5,
"matched_length": 5,
Expand Down
95 changes: 94 additions & 1 deletion tests/summarycode/data/full_summary/summary.expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"license_expressions": [
{
"value": "zlib",
"count": 12
"count": 19
},
{
"value": null,
Expand Down Expand Up @@ -3725,6 +3725,7 @@
"short_name": "ZLIB License",
"category": "Permissive",
"is_exception": false,
"is_unknown": false,
"owner": "zlib",
"homepage_url": "http://www.zlib.net/",
"text_url": "http://www.gzip.org/zlib/zlib_license.html",
Expand All @@ -3741,11 +3742,13 @@
"licenses": [
"zlib"
],
"referenced_filenames": [],
"is_license_text": true,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": false,
"is_license_intro": false,
"has_unknown": false,
"matcher": "2-aho",
"rule_length": 144,
"matched_length": 144,
Expand Down Expand Up @@ -3931,6 +3934,7 @@
"short_name": "ZLIB License",
"category": "Permissive",
"is_exception": false,
"is_unknown": false,
"owner": "zlib",
"homepage_url": "http://www.zlib.net/",
"text_url": "http://www.gzip.org/zlib/zlib_license.html",
Expand All @@ -3947,11 +3951,13 @@
"licenses": [
"zlib"
],
"referenced_filenames": [],
"is_license_text": true,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": false,
"is_license_intro": false,
"has_unknown": false,
"matcher": "2-aho",
"rule_length": 144,
"matched_length": 144,
Expand Down Expand Up @@ -7234,6 +7240,7 @@
"short_name": "ZLIB License",
"category": "Permissive",
"is_exception": false,
"is_unknown": false,
"owner": "zlib",
"homepage_url": "http://www.zlib.net/",
"text_url": "http://www.gzip.org/zlib/zlib_license.html",
Expand All @@ -7250,11 +7257,13 @@
"licenses": [
"zlib"
],
"referenced_filenames": [],
"is_license_text": true,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": false,
"is_license_intro": false,
"has_unknown": false,
"matcher": "2-aho",
"rule_length": 144,
"matched_length": 144,
Expand Down Expand Up @@ -7348,9 +7357,48 @@
"match_coverage": 100.0,
"rule_relevance": 100
}
},
{
"key": "zlib",
"score": 100.0,
"name": "ZLIB License",
"short_name": "ZLIB License",
"category": "Permissive",
"is_exception": false,
"is_unknown": false,
"owner": "zlib",
"homepage_url": "http://www.zlib.net/",
"text_url": "http://www.gzip.org/zlib/zlib_license.html",
"reference_url": "https://scancode-licensedb.aboutcode.org/zlib",
"scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/zlib.LICENSE",
"scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/zlib.yml",
"spdx_license_key": "Zlib",
"spdx_url": "https://spdx.org/licenses/Zlib",
"start_line": 6,
"end_line": 23,
"matched_rule": {
"identifier": "zlib_17.RULE",
"license_expression": "zlib",
"licenses": [
"zlib"
],
"referenced_filenames": [],
"is_license_text": true,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": false,
"is_license_intro": false,
"has_unknown": false,
"matcher": "2-aho",
"rule_length": 144,
"matched_length": 144,
"match_coverage": 100.0,
"rule_relevance": 100
}
}
],
"license_expressions": [
"zlib",
"zlib"
],
"percentage_of_license_text": 40.0,
Expand Down Expand Up @@ -7434,9 +7482,48 @@
"match_coverage": 100.0,
"rule_relevance": 100
}
},
{
"key": "zlib",
"score": 100.0,
"name": "ZLIB License",
"short_name": "ZLIB License",
"category": "Permissive",
"is_exception": false,
"is_unknown": false,
"owner": "zlib",
"homepage_url": "http://www.zlib.net/",
"text_url": "http://www.gzip.org/zlib/zlib_license.html",
"reference_url": "https://scancode-licensedb.aboutcode.org/zlib",
"scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/zlib.LICENSE",
"scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/zlib.yml",
"spdx_license_key": "Zlib",
"spdx_url": "https://spdx.org/licenses/Zlib",
"start_line": 6,
"end_line": 23,
"matched_rule": {
"identifier": "zlib_17.RULE",
"license_expression": "zlib",
"licenses": [
"zlib"
],
"referenced_filenames": [],
"is_license_text": true,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": false,
"is_license_intro": false,
"has_unknown": false,
"matcher": "2-aho",
"rule_length": 144,
"matched_length": 144,
"match_coverage": 100.0,
"rule_relevance": 100
}
}
],
"license_expressions": [
"zlib",
"zlib"
],
"percentage_of_license_text": 50.0,
Expand Down Expand Up @@ -8211,6 +8298,7 @@
"short_name": "ZLIB License",
"category": "Permissive",
"is_exception": false,
"is_unknown": false,
"owner": "zlib",
"homepage_url": "http://www.zlib.net/",
"text_url": "http://www.gzip.org/zlib/zlib_license.html",
Expand All @@ -8227,11 +8315,13 @@
"licenses": [
"zlib"
],
"referenced_filenames": [],
"is_license_text": true,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": false,
"is_license_intro": false,
"has_unknown": false,
"matcher": "2-aho",
"rule_length": 144,
"matched_length": 144,
Expand Down Expand Up @@ -8333,6 +8423,7 @@
"short_name": "ZLIB License",
"category": "Permissive",
"is_exception": false,
"is_unknown": false,
"owner": "zlib",
"homepage_url": "http://www.zlib.net/",
"text_url": "http://www.gzip.org/zlib/zlib_license.html",
Expand All @@ -8349,11 +8440,13 @@
"licenses": [
"zlib"
],
"referenced_filenames": [],
"is_license_text": true,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": false,
"is_license_intro": false,
"has_unknown": false,
"matcher": "2-aho",
"rule_length": 144,
"matched_length": 144,
Expand Down
Loading

0 comments on commit d4f1436

Please sign in to comment.