Skip to content

Commit

Permalink
Remove references to licenses keys & choices #74
Browse files Browse the repository at this point in the history 
 * use license_expression instead everywhere
 * streamline tests (remove negative tests that was useless and not
   working)

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
  • Loading branch information
@pombredanne
pombredanne committed May 3, 2018
1 parent 5b8ca2e commit e10e390
Show file tree
Hide file tree
Showing 72 changed files with 411 additions and 631 deletions.
4 changes: 0 additions & 4 deletions etc/scripts/json2csv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,6 @@ def collect_keys(mapping, key_group):
elif k == 'matched_rule':
for mrk, mrv in val.items():
mrk = 'matched_rule__' + mrk
if mrk == 'license_choice':
mrv = 'y' if mrv else ''
if mrk == 'licenses':
mrv = ' '.join(mrv)
if mrk in ('match_coverage', 'rule_relevance'):
# normalize the string representation of this number
mrv = '{:.2f}'.format(mrv)
Expand Down
9 changes: 5 additions & 4 deletions etc/scripts/synclic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,11 +266,12 @@ def get_match(text):
query = match.query
query_len = len(query.whole_query_run().tokens)
rule = match.rule
key = rule.licenses[0]
rule_licenses = rule.license_keys()
key = rule_licenses[0]

is_exact = (
len(matches) == 1
and rule.is_license and len(rule.licenses) == 1
and rule.is_license and len(rule_licenses) == 1
and match.matcher == '1-hash'
and match.score() == 100
and match.qlen == query_len
Expand All @@ -280,14 +281,14 @@ def get_match(text):
return key, True, 100, matched_text

is_ok = (
len(rule.licenses) == 1
len(rule_licenses) == 1
and match.coverage() > 95
and match.score() > 95)
if is_ok:
return key, False, match.score(), matched_text

is_weak = (
len(rule.licenses) == 1
len(rule_licenses) == 1
and match.coverage() > 90
and match.score() > 90)
if is_weak:
Expand Down
32 changes: 0 additions & 32 deletions etc/scripts/testdata/json2csv/full.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@
"end_line": 3,
"matched_rule": {
"identifier": "proprietary_8.RULE",
"license_choice": false,
"licenses": [
"proprietary"
],
Expand Down Expand Up @@ -498,7 +497,6 @@
"end_line": 502,
"matched_rule": {
"identifier": "lgpl-2.1-plus_2.RULE",
"license_choice": false,
"licenses": [
"lgpl-2.1-plus"
],
Expand Down Expand Up @@ -611,7 +609,6 @@
"end_line": 108,
"matched_rule": {
"identifier": "jboss-eula.LICENSE",
"license_choice": false,
"licenses": [
"jboss-eula"
],
Expand Down Expand Up @@ -757,7 +754,6 @@
"end_line": 11,
"matched_rule": {
"identifier": "cc-by-2.5_4.RULE",
"license_choice": false,
"licenses": [
"cc-by-2.5"
],
Expand Down Expand Up @@ -843,7 +839,6 @@
"end_line": 20,
"matched_rule": {
"identifier": "lgpl-2.1-plus_59.RULE",
"license_choice": false,
"licenses": [
"lgpl-2.1-plus"
],
Expand Down Expand Up @@ -967,7 +962,6 @@
"end_line": 1649,
"matched_rule": {
"identifier": "public-domain.LICENSE",
"license_choice": false,
"licenses": [
"public-domain"
],
Expand All @@ -993,7 +987,6 @@
"end_line": 1692,
"matched_rule": {
"identifier": "public-domain.LICENSE",
"license_choice": false,
"licenses": [
"public-domain"
],
Expand Down Expand Up @@ -1084,7 +1077,6 @@
"end_line": 20,
"matched_rule": {
"identifier": "lgpl-2.1-plus_59.RULE",
"license_choice": false,
"licenses": [
"lgpl-2.1-plus"
],
Expand Down Expand Up @@ -1171,7 +1163,6 @@
"end_line": 20,
"matched_rule": {
"identifier": "lgpl-2.1-plus_59.RULE",
"license_choice": false,
"licenses": [
"lgpl-2.1-plus"
],
Expand Down Expand Up @@ -1280,7 +1271,6 @@
"end_line": 212,
"matched_rule": {
"identifier": "cpl-1.0.SPDX.RULE",
"license_choice": false,
"licenses": [
"cpl-1.0"
],
Expand Down Expand Up @@ -1334,7 +1324,6 @@
"end_line": 18,
"matched_rule": {
"identifier": "mit.LICENSE",
"license_choice": false,
"licenses": [
"mit"
],
Expand Down Expand Up @@ -1406,7 +1395,6 @@
"end_line": 502,
"matched_rule": {
"identifier": "lgpl-2.1-plus_2.RULE",
"license_choice": false,
"licenses": [
"lgpl-2.1-plus"
],
Expand Down Expand Up @@ -1492,7 +1480,6 @@
"end_line": 202,
"matched_rule": {
"identifier": "apache-2.0_easyeclipse.RULE",
"license_choice": false,
"licenses": [
"apache-2.0"
],
Expand Down Expand Up @@ -1557,7 +1544,6 @@
"end_line": 56,
"matched_rule": {
"identifier": "apache-1.1.SPDX.RULE",
"license_choice": false,
"licenses": [
"apache-1.1"
],
Expand Down Expand Up @@ -1644,7 +1630,6 @@
"end_line": 13,
"matched_rule": {
"identifier": "mit_13.RULE",
"license_choice": false,
"licenses": [
"mit"
],
Expand Down Expand Up @@ -1787,7 +1772,6 @@
"end_line": 3,
"matched_rule": {
"identifier": "zlib_5.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand All @@ -1813,7 +1797,6 @@
"end_line": 94,
"matched_rule": {
"identifier": "zlib_21.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand Down Expand Up @@ -1879,7 +1862,6 @@
"end_line": 3,
"matched_rule": {
"identifier": "zlib_5.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand Down Expand Up @@ -1945,7 +1927,6 @@
"end_line": 23,
"matched_rule": {
"identifier": "zlib_17.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand Down Expand Up @@ -2028,7 +2009,6 @@
"end_line": 3,
"matched_rule": {
"identifier": "zlib_5.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand Down Expand Up @@ -2120,7 +2100,6 @@
"end_line": 3,
"matched_rule": {
"identifier": "zlib_5.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand Down Expand Up @@ -2186,7 +2165,6 @@
"end_line": 3,
"matched_rule": {
"identifier": "zlib_5.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand Down Expand Up @@ -2426,7 +2404,6 @@
"end_line": 23,
"matched_rule": {
"identifier": "boost-1.0.LICENSE",
"license_choice": false,
"licenses": [
"boost-1.0"
],
Expand Down Expand Up @@ -2480,7 +2457,6 @@
"end_line": 58,
"matched_rule": {
"identifier": "boost-1.0.RULE",
"license_choice": false,
"licenses": [
"boost-1.0"
],
Expand Down Expand Up @@ -2552,7 +2528,6 @@
"end_line": 5,
"matched_rule": {
"identifier": "boost-1.0_1.RULE",
"license_choice": false,
"licenses": [
"boost-1.0"
],
Expand Down Expand Up @@ -2651,7 +2626,6 @@
"end_line": 15,
"matched_rule": {
"identifier": "cmr-no.LICENSE",
"license_choice": false,
"licenses": [
"cmr-no"
],
Expand Down Expand Up @@ -2724,7 +2698,6 @@
"end_line": 3,
"matched_rule": {
"identifier": "zlib_5.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand Down Expand Up @@ -2790,7 +2763,6 @@
"end_line": 3,
"matched_rule": {
"identifier": "zlib_5.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand Down Expand Up @@ -2856,7 +2828,6 @@
"end_line": 31,
"matched_rule": {
"identifier": "zlib_4.RULE",
"license_choice": false,
"licenses": [
"zlib"
],
Expand Down Expand Up @@ -2962,7 +2933,6 @@
"end_line": 25,
"matched_rule": {
"identifier": "gpl-2.0-plus-ada.LICENSE",
"license_choice": false,
"licenses": [
"gpl-2.0-plus-ada"
],
Expand Down Expand Up @@ -3081,7 +3051,6 @@
"end_line": 48,
"matched_rule": {
"identifier": "pypi_apache_software_license.RULE",
"license_choice": false,
"licenses": [
"apache-2.0"
],
Expand All @@ -3107,7 +3076,6 @@
"end_line": 49,
"matched_rule": {
"identifier": "pypi_osi_approved.RULE",
"license_choice": false,
"licenses": [
"free-unknown"
],
Expand Down
4 changes: 0 additions & 4 deletions etc/scripts/testdata/json2csv/minimal.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
"end_line": 5,
"matched_rule": {
"identifier": "apache-2.0_23.RULE",
"license_choice": false,
"licenses": [
"apache-2.0"
]
Expand All @@ -43,7 +42,6 @@
"end_line": 4,
"matched_rule": {
"identifier": "openssl-ssleay_2.RULE",
"license_choice": false,
"licenses": [
"openssl-ssleay"
]
Expand Down Expand Up @@ -84,7 +82,6 @@
"end_line": 5,
"matched_rule": {
"identifier": "apache-2.0_23.RULE",
"license_choice": false,
"licenses": [
"apache-2.0"
]
Expand All @@ -105,7 +102,6 @@
"end_line": 4,
"matched_rule": {
"identifier": "openssl-ssleay_2.RULE",
"license_choice": false,
"licenses": [
"openssl-ssleay"
]
Expand Down
2 changes: 0 additions & 2 deletions etc/scripts/testdata/json2csv/package_license_value_null.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
"end_line": 5,
"matched_rule": {
"identifier": "apache-2.0_23.RULE",
"license_choice": false,
"licenses": [
"apache-2.0"
]
Expand All @@ -43,7 +42,6 @@
"end_line": 4,
"matched_rule": {
"identifier": "openssl-ssleay_2.RULE",
"license_choice": false,
"licenses": [
"openssl-ssleay"
]
Expand Down
18 changes: 9 additions & 9 deletions etc/scripts/testdata/livescan/expected.csv
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
Resource,type,name,base_name,extension,size,date,sha1,md5,mime_type,file_type,programming_language,is_binary,is_text,is_archive,is_media,is_source,is_script,package_manifest,files_count,dirs_count,size_count,scan_errors,license__key,license__score,license__short_name,license__category,license__owner,license__homepage_url,license__text_url,license__reference_url,license__spdx_license_key,license__spdx_url,start_line,end_line,matched_rule__identifier,matched_rule__license_choice,matched_rule__licenses,copyright,copyright_holder,email,url
/json2csv.rb,file,json2csv.rb,json2csv,.rb,1599,2018-04-27,6cfb0bd0fb0b784f57164d15bdfca2b734ad87a6,f18e519b77bc7f3e4213215033db3857,text/x-python,"Python script, ASCII text executable",Ruby,False,True,False,False,True,True,,0,0,0,,,,,,,,,,,,,,,,,,,,
/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,apache-2.0,98.45,Apache 2.0,Permissive,Apache Software Foundation,http://www.apache.org/licenses/,http://www.apache.org/licenses/LICENSE-2.0,https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0,Apache-2.0,https://spdx.org/licenses/Apache-2.0,5,24,apache-2.0_scancode.RULE,False,"[u'apache-2.0', u'scancode-acknowledgment']",,,,
/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,scancode-acknowledgment,98.45,ScanCode acknowledgment,Permissive,nexB,https://github.com/nexB/scancode-toolkit/,,https://enterprise.dejacode.com/urn/urn:dje:license:scancode-acknowledgment,,,5,24,apache-2.0_scancode.RULE,False,"[u'apache-2.0', u'scancode-acknowledgment']",,,,
Resource,type,name,base_name,extension,size,date,sha1,md5,mime_type,file_type,programming_language,is_binary,is_text,is_archive,is_media,is_source,is_script,package_manifest,files_count,dirs_count,size_count,scan_errors,license__key,license__score,license__short_name,license__category,license__owner,license__homepage_url,license__text_url,license__reference_url,license__spdx_license_key,license__spdx_url,start_line,end_line,matched_rule__identifier,matched_rule__license_expression,matched_rule__licenses,copyright,copyright_holder,email,url
/json2csv.rb,file,json2csv.rb,json2csv,.rb,1599,2018-04-11,6cfb0bd0fb0b784f57164d15bdfca2b734ad87a6,f18e519b77bc7f3e4213215033db3857,text/x-python,"Python script, ASCII text executable",Ruby,False,True,False,False,True,True,,0,0,0,,,,,,,,,,,,,,,,,,,,
/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,apache-2.0,98.45,Apache 2.0,Permissive,Apache Software Foundation,http://www.apache.org/licenses/,http://www.apache.org/licenses/LICENSE-2.0,https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0,Apache-2.0,https://spdx.org/licenses/Apache-2.0,5,24,apache-2.0_scancode.RULE,apache-2.0 AND scancode-acknowledgment,"[u'apache-2.0', u'scancode-acknowledgment']",,,,
/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,scancode-acknowledgment,98.45,ScanCode acknowledgment,Permissive,nexB,https://github.com/nexB/scancode-toolkit/,,https://enterprise.dejacode.com/urn/urn:dje:license:scancode-acknowledgment,,,5,24,apache-2.0_scancode.RULE,apache-2.0 AND scancode-acknowledgment,"[u'apache-2.0', u'scancode-acknowledgment']",,,,
/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,Copyright (c) 2017 nexB Inc. and others.,,,
/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,,,,,nexB Inc. and others.,,
/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,http://nexb.com/
/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4,4,,,,,,,https://github.com/nexB/scancode-toolkit/
/json2csv.rb,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,10,10,,,,,,,http://apache.org/licenses/LICENSE-2.0
/license,file,license,license,,679,2018-04-27,75c5490a718ddd45e40e0cc7ce0c756abc373123,b965a762efb9421cf1bf4405f336e278,text/plain,ASCII text,,False,True,False,False,False,False,,0,0,0,,,,,,,,,,,,,,,,,,,,
/license,,,,,,,,,,,,,,,,,,,,,,,gpl-2.0-plus,100.00,GPL 2.0 or later,Copyleft,Free Software Foundation (FSF),http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0-plus,GPL-2.0-or-later,https://spdx.org/licenses/GPL-2.0-or-later,1,12,gpl-2.0-plus.LICENSE,False,[u'gpl-2.0-plus'],,,,
/package.json,file,package.json,package,.json,2200,2018-04-27,918376afce796ef90eeda1d6695f2289c90491ac,1f66239a9b850c5e60a9382dbe2162d2,text/plain,"ASCII text, with very long lines",JSON,False,True,False,False,True,False,"OrderedDict([(u'type', u'npm'), (u'namespace', None), (u'name', u'cookie-signature'), (u'version', u'1.0.3'), (u'qualifiers', None), (u'subpath', None), (u'primary_language', u'JavaScript'), (u'code_type', None), (u'description', u'Sign and unsign cookies'), (u'size', None), (u'release_date', None), (u'parties', [OrderedDict([(u'type', u'person'), (u'role', u'author'), (u'name', u'TJ Holowaychuk'), (u'email', u'tj@learnboost.com'), (u'url', None)])]), (u'keywords', [u'cookie', u'sign', u'unsign']), (u'homepage_url', None), (u'download_url', u'https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.3.tgz'), (u'download_checksums', []), (u'bug_tracking_url', u'https://github.com/visionmedia/node-cookie-signature/issues'), (u'code_view_url', None), (u'vcs_tool', u'git'), (u'vcs_repository', u'https://github.com/visionmedia/node-cookie-signature.git'), (u'vcs_revision', None), (u'copyright', None), (u'license_expression', None), (u'declared_licensing', None), (u'notice_text', None), (u'dependencies', [OrderedDict([(u'purl', u'pkg:npm/mocha'), (u'requirement', u'*'), (u'scope', u'devDependencies'), (u'is_runtime', False), (u'is_optional', True), (u'is_resolved', False)]), OrderedDict([(u'purl', u'pkg:npm/should'), (u'requirement', u'*'), (u'scope', u'devDependencies'), (u'is_runtime', False), (u'is_optional', True), (u'is_resolved', False)])]), (u'related_packages', [])])",0,0,0,,,,,,,,,,,,,,,,,,,,
/package.json,,,,,,,,,,,,,,,,,,,,,,,mit,15.00,MIT License,Permissive,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit_27.RULE,False,[u'mit'],,,,
/package.json,,,,,,,,,,,,,,,,,,,,,,,mit,100.00,MIT License,Permissive,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit.LICENSE,False,[u'mit'],,,,
/license,file,license,license,,679,2018-04-11,75c5490a718ddd45e40e0cc7ce0c756abc373123,b965a762efb9421cf1bf4405f336e278,text/plain,ASCII text,,False,True,False,False,False,False,,0,0,0,,,,,,,,,,,,,,,,,,,,
/license,,,,,,,,,,,,,,,,,,,,,,,gpl-2.0-plus,100.00,GPL 2.0 or later,Copyleft,Free Software Foundation (FSF),http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,http://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html,https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0-plus,GPL-2.0-or-later,https://spdx.org/licenses/GPL-2.0-or-later,1,12,gpl-2.0-plus.LICENSE,gpl-2.0-plus,[u'gpl-2.0-plus'],,,,
/package.json,file,package.json,package,.json,2200,2018-04-11,918376afce796ef90eeda1d6695f2289c90491ac,1f66239a9b850c5e60a9382dbe2162d2,text/plain,"ASCII text, with very long lines",JSON,False,True,False,False,True,False,"OrderedDict([(u'type', u'npm'), (u'namespace', None), (u'name', u'cookie-signature'), (u'version', u'1.0.3'), (u'qualifiers', None), (u'subpath', None), (u'primary_language', u'JavaScript'), (u'code_type', None), (u'description', u'Sign and unsign cookies'), (u'size', None), (u'release_date', None), (u'parties', [OrderedDict([(u'type', u'person'), (u'role', u'author'), (u'name', u'TJ Holowaychuk'), (u'email', u'tj@learnboost.com'), (u'url', None)])]), (u'keywords', [u'cookie', u'sign', u'unsign']), (u'homepage_url', None), (u'download_url', u'https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.3.tgz'), (u'download_checksums', []), (u'bug_tracking_url', u'https://github.com/visionmedia/node-cookie-signature/issues'), (u'code_view_url', None), (u'vcs_tool', u'git'), (u'vcs_repository', u'https://github.com/visionmedia/node-cookie-signature.git'), (u'vcs_revision', None), (u'copyright', None), (u'license_expression', None), (u'declared_licensing', None), (u'notice_text', None), (u'dependencies', [OrderedDict([(u'purl', u'pkg:npm/mocha'), (u'requirement', u'*'), (u'scope', u'devDependencies'), (u'is_runtime', False), (u'is_optional', True), (u'is_resolved', False)]), OrderedDict([(u'purl', u'pkg:npm/should'), (u'requirement', u'*'), (u'scope', u'devDependencies'), (u'is_runtime', False), (u'is_optional', True), (u'is_resolved', False)])]), (u'related_packages', [])])",0,0,0,,,,,,,,,,,,,,,,,,,,
/package.json,,,,,,,,,,,,,,,,,,,,,,,mit,15.00,MIT License,Permissive,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit_27.RULE,mit,[u'mit'],,,,
/package.json,,,,,,,,,,,,,,,,,,,,,,,mit,100.00,MIT License,Permissive,MIT,http://opensource.org/licenses/mit-license.php,http://opensource.org/licenses/mit-license.php,https://enterprise.dejacode.com/urn/urn:dje:license:mit,MIT,https://spdx.org/licenses/MIT,24,24,mit.LICENSE,mit,[u'mit'],,,,
/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,23,26,,,,Copyright (c) 2012 LearnBoost <tj@learnboost.com>,,,
/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,23,26,,,,,LearnBoost,,
/package.json,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,12,12,,,,,,tj@learnboost.com,
Expand Down
Loading

0 comments on commit e10e390

Please sign in to comment.