Merge latest develop branch

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

azure-pipelines.yml

@@ -208,8 +208,10 @@ jobs:
             displayName: 'Build installable releases'
 
           - script: ./etc/release/pip-install.sh
-            displayName: 'Test pip installation'
+            displayName: 'Test pip wheel installation'
 
+          - script: ./etc/release/pip-install-editable.sh
+            displayName: 'Test pip editable installation'
 
 ################################################################################
 # These jobs are using VMs and Azure-provided Pythons 3.6

etc/release/pip-install-editable.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved.
+#
+
+# ScanCode release script
+# This script creates and tests release archives in the dist/ dir
+
+set -e
+
+# un-comment to trace execution
+set -x
+
+echo "###  Installing ScanCode release with pip editable###"
+
+mkdir -p tmp/pipe
+python -m venv tmp/pipe
+tmp/pipe/bin/pip install -e .
+
+# perform a minimal check of the results for https://github.com/nexB/scancode-toolkit/issues/2201
+if [ `tmp/pipe/bin/scancode -i --json-pp - NOTICE | grep -c "scan_timings"` == 1 ]; then
+   echo "Failed scan that includes timings"
+   exit 1
+else
+   echo "pass"
+fi
+
+set +e
+set +x

etc/release/pip-install.sh

@@ -15,8 +15,15 @@ echo "###  Installing ScanCode release with pip ###"
 
 mkdir -p tmp/pip
 python -m venv tmp/pip
-source tmp/pip/bin/activate
-pip install dist/scancode_toolkit*.whl
+tmp/pip/bin/pip install dist/scancode_toolkit*.whl
+
+# perform a minimal check of the results for https://github.com/nexB/scancode-toolkit/issues/2201
+if [ `tmp/pip/bin/scancode -i --json-pp - NOTICE | grep -c "scan_timings"` == 1 ]; then
+   echo "Failed scan that includes timings"
+   exit 1
+else
+   echo "pass"
+fi
 
 set +e
 set +x

etc/scripts/README.rst

@@ -1,3 +1,23 @@
+*Before generating requirement tools and uploading asset make sure you've installed the prerequisites properly. This mainly :*
+  - pip-tools
+  - github-release-retry
+
+Instruction for generating requirement tools:
+=============================================
+- For this you must run on your terminal:
+
+  ``pip pip install -r etc/scripts/req_tools.txt``
+
+- Then run ``python etc/scripts/freeze_and_update_reqs.py --help`` on terminal
+
+From this you get the guidelines how to generate requirement tools.
+
+Instruction for uploading assets:
+=================================
+
+- Just run ``python etc/scripts/github_release.py --help``
+- From this you get the guidelines how to upload asset to github repositotory as an asset.
+
 This directory contains miscellaneous scripts of some use with ScanCode.
 
-    - json2csv: convert a scan JSON to a CSV.
+- json2csv: convert a scan JSON to a CSV.

etc/scripts/freeze_and_update_reqs.py

@@ -124,8 +124,8 @@ def main_with_args(args: str) -> None:
     upgrade = args.upgrade or False
     generate_req_text(
         find_links=find_links,
-        requirement=requirement,
-        upgrade_package=upgrade_package,
+        req_file=requirement,
+        package_name=upgrade_package,
         upgrade=upgrade,
     )
 

etc/scripts/req_tools.txt

@@ -0,0 +1,2 @@
+pip-tools
+github-release-retry

setup.py

@@ -137,7 +137,7 @@ def read(*names, **kwargs):
         # cluecode
         # Some nltk version ranges are buggy
         'nltk >= 3.2, < 4.0',
-        'py2_ipaddress >= 2.0, <3.5; python_version<"3"',
+        'py2_ipaddress >= 2.0, <3.5; python_version < "3"',
         'urlpy',
         'publicsuffix2',
         'fingerprints >= 0.6.0, < 1.0.0',
@@ -146,12 +146,12 @@ def read(*names, **kwargs):
         'patch >= 1.15, < 1.20 ',
         # to work around bug http://bugs.python.org/issue19839
         # on multistream bzip2 files: this can removed in Python 3.
-        'bz2file >= 0.98; python_version<"3"',
+        'bz2file >= 0.98; python_version < "3"',
         'extractcode_libarchive',
         'extractcode_7z',
 
         # commoncode
-        'backports.os == 0.1.1; python_version<"3"',
+        'backports.os == 0.1.1; python_version < "3"',
         'future >= 0.16.0',
         'text_unidecode >= 1.0, < 2.0',
         'saneyaml',
@@ -189,18 +189,19 @@ def read(*names, **kwargs):
         'xmltodict >= 0.11.0',
         'javaproperties >= 0.5',
         'toml >= 0.10.0',
+        'gemfileparser >= 0.7.0',
         'pkginfo >= 1.5.0.1',
         'dparse2',
 
         # used to fix mojibake in Windows PE
-        'ftfy <  5.0.0; python_version == "2.7"',
-        'ftfy>=  5.0.0; python_version > "3"',
+        # for now we use the evrsion that works on both Python 2 and 3
+        'ftfy <  5.0.0',
 
         # scancode
         'click >= 6.0.0',
         'colorama >= 0.3.9',
         'pluggy >= 0.4.0, < 1.0',
-        'attrs >= 18.1',
+        'attrs >= 18.1, !=20.1.0',
         'typing >=3.6, < 3.7',
 
         # scancode outputs
@@ -212,8 +213,8 @@ def read(*names, **kwargs):
 
         # ScanCode caching and locking
         'yg.lockfile >= 2.3, < 3.0.0',
-            # used by yg.lockfile
-            'contextlib2', 'pytz', 'tempora', 'jaraco.functools',
+        # used by yg.lockfile
+        'contextlib2', 'pytz', 'tempora', 'jaraco.functools',
         'zc.lockfile >= 2.0.0, < 3.0.0',
     ],
 

src/packagedcode/__init__.py

@@ -25,20 +25,22 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+from packagedcode import about
+from packagedcode import bower
 from packagedcode import build
+from packagedcode import cargo
 from packagedcode import chef
 from packagedcode import debian
-from packagedcode import models
-from packagedcode import about
-from packagedcode import bower
 from packagedcode import conda
-from packagedcode import cargo
+from packagedcode import cocoapods
 from packagedcode import freebsd
 from packagedcode import golang
 from packagedcode import haxe
 from packagedcode import maven
+from packagedcode import models
 from packagedcode import npm
 from packagedcode import nuget
+from packagedcode import opam
 from packagedcode import phpcomposer
 from packagedcode import pypi
 from packagedcode import rpm
@@ -64,6 +66,8 @@
     phpcomposer.PHPComposerPackage,
     haxe.HaxePackage,
     cargo.RustCargoCrate,
+    cocoapods.CocoapodsPackage,
+    opam.OpamPackage,
     models.MeteorPackage,
     bower.BowerPackage,
     freebsd.FreeBSDPackage,

src/packagedcode/cargo.py

@@ -1,5 +1,5 @@
 
-# Copyright (c) 2019 nexB Inc. and others. All rights reserved.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/scancode-toolkit/
 # The ScanCode software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode require an acknowledgment.
@@ -31,6 +31,7 @@
 import re
 
 import attr
+from packageurl import PackageURL
 import toml
 
 from commoncode import filetype
@@ -54,7 +55,7 @@
 
 @attr.s()
 class RustCargoCrate(models.Package):
-    metafiles = ('Cargo.toml',)
+    metafiles = ('Cargo.toml', 'Cargo.lock')
     default_type = 'cargo'
     default_primary_language = 'Rust'
     default_web_baseurl = 'https://crates.io'
@@ -70,33 +71,32 @@ def get_package_root(cls, manifest_resource, codebase):
         return manifest_resource.parent(codebase)
 
     def repository_homepage_url(self, baseurl=default_web_baseurl):
-        return '{}/crates/{}'.format(baseurl, self.name)
+        if self.name:
+            return '{}/crates/{}'.format(baseurl, self.name)
 
     def repository_download_url(self, baseurl=default_download_baseurl):
-        return '{}/crates/{}/{}/download'.format(baseurl, self.name, self.version)
+        if self.name and self.version:
+            return '{}/crates/{}/{}/download'.format(baseurl, self.name, self.version)
 
     def api_data_url(self, baseurl=default_api_baseurl):
-        return '{}/crates/{}'.format(baseurl, self.name)
-
-
-def is_cargo_toml(location):
-    return (filetype.is_file(location) and fileutils.file_name(location).lower() == 'cargo.toml')
+        if self.name:
+            return '{}/crates/{}'.format(baseurl, self.name)
 
 
 def parse(location):
     """
-    Return a Package object from a Cargo.toml file or None.
+    Return a Package object from a Cargo.toml/Cargo.lock file.
     """
-    if not is_cargo_toml(location):
-        return
+    handlers = {'cargo.toml': build_cargo_toml_package, 'cargo.lock': build_cargo_lock_package}
+    filename = filetype.is_file(location) and fileutils.file_name(location).lower()
+    handler = handlers.get(filename)
+    if handler:
+        return handler and handler(toml.load(location, _dict=OrderedDict))
 
-    package_data = toml.load(location, _dict=OrderedDict)
-    return build_package(package_data)
 
-
-def build_package(package_data):
+def build_cargo_toml_package(package_data):
     """
-    Return a Pacakge object from a package data mapping or None.
+    Return a Package object from a Cargo.toml package data mapping or None.
     """
 
     core_package_data = package_data.get('package', {})
@@ -179,3 +179,29 @@ def parse_person(person):
 person_parser_no_name = re.compile(
     r'(?P<email><([^>]+)>)?'
 ).match
+
+
+def build_cargo_lock_package(package_data):
+    """
+    Return a Package object from a Cargo.lock package data mapping or None.
+    """
+
+    package_dependencies = []
+    core_package_data = package_data.get('package', [])
+    for dep in core_package_data:
+        package_dependencies.append(
+            models.DependentPackage(
+                purl=PackageURL(
+                    type='crates',
+                    name=dep.get('name'),
+                    version=dep.get('version')
+                ).to_string(),
+                requirement=dep.get('version'),
+                scope='dependency',
+                is_runtime=True,
+                is_optional=False,
+                is_resolved=True,
+            )
+        )
+
+    return RustCargoCrate(dependencies=package_dependencies)