diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d21feaf7c9f..5f4542de849 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -208,8 +208,10 @@ jobs: displayName: 'Build installable releases' - script: ./etc/release/pip-install.sh - displayName: 'Test pip installation' + displayName: 'Test pip wheel installation' + - script: ./etc/release/pip-install-editable.sh + displayName: 'Test pip editable installation' ################################################################################ # These jobs are using VMs and Azure-provided Pythons 3.6 diff --git a/etc/release/pip-install-editable.sh b/etc/release/pip-install-editable.sh new file mode 100755 index 00000000000..b631711e6c1 --- /dev/null +++ b/etc/release/pip-install-editable.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. +# + +# ScanCode release script +# This script creates and tests release archives in the dist/ dir + +set -e + +# un-comment to trace execution +set -x + +echo "### Installing ScanCode release with pip editable###" + +mkdir -p tmp/pipe +python -m venv tmp/pipe +tmp/pipe/bin/pip install -e . + +# perform a minimal check of the results for https://github.com/nexB/scancode-toolkit/issues/2201 +if [ `tmp/pipe/bin/scancode -i --json-pp - NOTICE | grep -c "scan_timings"` == 1 ]; then + echo "Failed scan that includes timings" + exit 1 +else + echo "pass" +fi + +set +e +set +x diff --git a/etc/release/pip-install.sh b/etc/release/pip-install.sh index 1564dc54805..9d3a86e27f2 100755 --- a/etc/release/pip-install.sh +++ b/etc/release/pip-install.sh @@ -15,8 +15,15 @@ echo "### Installing ScanCode release with pip ###" mkdir -p tmp/pip python -m venv tmp/pip -source tmp/pip/bin/activate -pip install dist/scancode_toolkit*.whl +tmp/pip/bin/pip install dist/scancode_toolkit*.whl + +# perform a minimal check of the results for https://github.com/nexB/scancode-toolkit/issues/2201 +if [ `tmp/pip/bin/scancode -i --json-pp - NOTICE | grep -c "scan_timings"` == 1 ]; then + echo "Failed scan that includes timings" + exit 1 +else + echo "pass" +fi set +e set +x diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index a6a0f794275..0031e08c015 100644 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -1,3 +1,23 @@ +*Before generating requirement tools and uploading asset make sure you've installed the prerequisites properly. This mainly :* + - pip-tools + - github-release-retry + +Instruction for generating requirement tools: +============================================= +- For this you must run on your terminal: + + ``pip pip install -r etc/scripts/req_tools.txt`` + +- Then run ``python etc/scripts/freeze_and_update_reqs.py --help`` on terminal + +From this you get the guidelines how to generate requirement tools. + +Instruction for uploading assets: +================================= + +- Just run ``python etc/scripts/github_release.py --help`` +- From this you get the guidelines how to upload asset to github repositotory as an asset. + This directory contains miscellaneous scripts of some use with ScanCode. - - json2csv: convert a scan JSON to a CSV. \ No newline at end of file +- json2csv: convert a scan JSON to a CSV. diff --git a/etc/scripts/freeze_and_update_reqs.py b/etc/scripts/freeze_and_update_reqs.py index c2ef1169b90..8062932e5dd 100644 --- a/etc/scripts/freeze_and_update_reqs.py +++ b/etc/scripts/freeze_and_update_reqs.py @@ -124,8 +124,8 @@ def main_with_args(args: str) -> None: upgrade = args.upgrade or False generate_req_text( find_links=find_links, - requirement=requirement, - upgrade_package=upgrade_package, + req_file=requirement, + package_name=upgrade_package, upgrade=upgrade, ) diff --git a/etc/scripts/req_tools.txt b/etc/scripts/req_tools.txt new file mode 100644 index 00000000000..5cac8a7fb57 --- /dev/null +++ b/etc/scripts/req_tools.txt @@ -0,0 +1,2 @@ +pip-tools +github-release-retry diff --git a/setup.py b/setup.py index 3c5a65ec65d..7f8e17d2d28 100644 --- a/setup.py +++ b/setup.py @@ -137,7 +137,7 @@ def read(*names, **kwargs): # cluecode # Some nltk version ranges are buggy 'nltk >= 3.2, < 4.0', - 'py2_ipaddress >= 2.0, <3.5; python_version<"3"', + 'py2_ipaddress >= 2.0, <3.5; python_version < "3"', 'urlpy', 'publicsuffix2', 'fingerprints >= 0.6.0, < 1.0.0', @@ -146,12 +146,12 @@ def read(*names, **kwargs): 'patch >= 1.15, < 1.20 ', # to work around bug http://bugs.python.org/issue19839 # on multistream bzip2 files: this can removed in Python 3. - 'bz2file >= 0.98; python_version<"3"', + 'bz2file >= 0.98; python_version < "3"', 'extractcode_libarchive', 'extractcode_7z', # commoncode - 'backports.os == 0.1.1; python_version<"3"', + 'backports.os == 0.1.1; python_version < "3"', 'future >= 0.16.0', 'text_unidecode >= 1.0, < 2.0', 'saneyaml', @@ -189,18 +189,19 @@ def read(*names, **kwargs): 'xmltodict >= 0.11.0', 'javaproperties >= 0.5', 'toml >= 0.10.0', + 'gemfileparser >= 0.7.0', 'pkginfo >= 1.5.0.1', 'dparse2', # used to fix mojibake in Windows PE - 'ftfy < 5.0.0; python_version == "2.7"', - 'ftfy>= 5.0.0; python_version > "3"', + # for now we use the evrsion that works on both Python 2 and 3 + 'ftfy < 5.0.0', # scancode 'click >= 6.0.0', 'colorama >= 0.3.9', 'pluggy >= 0.4.0, < 1.0', - 'attrs >= 18.1', + 'attrs >= 18.1, !=20.1.0', 'typing >=3.6, < 3.7', # scancode outputs @@ -212,8 +213,8 @@ def read(*names, **kwargs): # ScanCode caching and locking 'yg.lockfile >= 2.3, < 3.0.0', - # used by yg.lockfile - 'contextlib2', 'pytz', 'tempora', 'jaraco.functools', + # used by yg.lockfile + 'contextlib2', 'pytz', 'tempora', 'jaraco.functools', 'zc.lockfile >= 2.0.0, < 3.0.0', ], diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index 6fd0561b57b..1b69ddb33cf 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -25,20 +25,22 @@ from __future__ import absolute_import from __future__ import unicode_literals +from packagedcode import about +from packagedcode import bower from packagedcode import build +from packagedcode import cargo from packagedcode import chef from packagedcode import debian -from packagedcode import models -from packagedcode import about -from packagedcode import bower from packagedcode import conda -from packagedcode import cargo +from packagedcode import cocoapods from packagedcode import freebsd from packagedcode import golang from packagedcode import haxe from packagedcode import maven +from packagedcode import models from packagedcode import npm from packagedcode import nuget +from packagedcode import opam from packagedcode import phpcomposer from packagedcode import pypi from packagedcode import rpm @@ -64,6 +66,8 @@ phpcomposer.PHPComposerPackage, haxe.HaxePackage, cargo.RustCargoCrate, + cocoapods.CocoapodsPackage, + opam.OpamPackage, models.MeteorPackage, bower.BowerPackage, freebsd.FreeBSDPackage, diff --git a/src/packagedcode/cargo.py b/src/packagedcode/cargo.py index 8d260f4d714..f4ffba81f00 100644 --- a/src/packagedcode/cargo.py +++ b/src/packagedcode/cargo.py @@ -1,5 +1,5 @@ -# Copyright (c) 2019 nexB Inc. and others. All rights reserved. +# Copyright (c) nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/scancode-toolkit/ # The ScanCode software is licensed under the Apache License version 2.0. # Data generated with ScanCode require an acknowledgment. @@ -31,6 +31,7 @@ import re import attr +from packageurl import PackageURL import toml from commoncode import filetype @@ -54,7 +55,7 @@ @attr.s() class RustCargoCrate(models.Package): - metafiles = ('Cargo.toml',) + metafiles = ('Cargo.toml', 'Cargo.lock') default_type = 'cargo' default_primary_language = 'Rust' default_web_baseurl = 'https://crates.io' @@ -70,33 +71,32 @@ def get_package_root(cls, manifest_resource, codebase): return manifest_resource.parent(codebase) def repository_homepage_url(self, baseurl=default_web_baseurl): - return '{}/crates/{}'.format(baseurl, self.name) + if self.name: + return '{}/crates/{}'.format(baseurl, self.name) def repository_download_url(self, baseurl=default_download_baseurl): - return '{}/crates/{}/{}/download'.format(baseurl, self.name, self.version) + if self.name and self.version: + return '{}/crates/{}/{}/download'.format(baseurl, self.name, self.version) def api_data_url(self, baseurl=default_api_baseurl): - return '{}/crates/{}'.format(baseurl, self.name) - - -def is_cargo_toml(location): - return (filetype.is_file(location) and fileutils.file_name(location).lower() == 'cargo.toml') + if self.name: + return '{}/crates/{}'.format(baseurl, self.name) def parse(location): """ - Return a Package object from a Cargo.toml file or None. + Return a Package object from a Cargo.toml/Cargo.lock file. """ - if not is_cargo_toml(location): - return + handlers = {'cargo.toml': build_cargo_toml_package, 'cargo.lock': build_cargo_lock_package} + filename = filetype.is_file(location) and fileutils.file_name(location).lower() + handler = handlers.get(filename) + if handler: + return handler and handler(toml.load(location, _dict=OrderedDict)) - package_data = toml.load(location, _dict=OrderedDict) - return build_package(package_data) - -def build_package(package_data): +def build_cargo_toml_package(package_data): """ - Return a Pacakge object from a package data mapping or None. + Return a Package object from a Cargo.toml package data mapping or None. """ core_package_data = package_data.get('package', {}) @@ -179,3 +179,29 @@ def parse_person(person): person_parser_no_name = re.compile( r'(?P<([^>]+)>)?' ).match + + +def build_cargo_lock_package(package_data): + """ + Return a Package object from a Cargo.lock package data mapping or None. + """ + + package_dependencies = [] + core_package_data = package_data.get('package', []) + for dep in core_package_data: + package_dependencies.append( + models.DependentPackage( + purl=PackageURL( + type='crates', + name=dep.get('name'), + version=dep.get('version') + ).to_string(), + requirement=dep.get('version'), + scope='dependency', + is_runtime=True, + is_optional=False, + is_resolved=True, + ) + ) + + return RustCargoCrate(dependencies=package_dependencies) diff --git a/src/packagedcode/cocoapods.py b/src/packagedcode/cocoapods.py new file mode 100644 index 00000000000..587f0b831b1 --- /dev/null +++ b/src/packagedcode/cocoapods.py @@ -0,0 +1,190 @@ +# All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import logging +import re + +import attr +from packageurl import PackageURL + +from commoncode.fileutils import py2 +from commoncode import filetype +from commoncode import fileutils +from packagedcode import models +from packagedcode.spec import Spec + + +""" +Handle cocoapods packages manifests for macOS and iOS +including .podspec, Podfile and Podfile.lock files. +See https://cocoapods.org +""" + +# TODO: override the license detection to detect declared_license correctly. + + +TRACE = False + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + +@attr.s() +class CocoapodsPackage(models.Package): + metafiles = ('*.podspec',) + extensions = ('.podspec',) + default_type = 'pods' + default_primary_language = 'Objective-C' + default_web_baseurl = 'https://cocoapods.org' + default_download_baseurl = None + default_api_baseurl = None + + @classmethod + def recognize(cls, location): + yield parse(location) + + def repository_homepage_url(self, baseurl=default_web_baseurl): + return '{}/pods/{}'.format(baseurl, self.name) + + def repository_download_url(self): + return '{}/archive/{}.zip'.format(self.homepage_url, self.version) + + +def is_podspec(location): + """ + Checks if the file is actually a podspec file + """ + return (filetype.is_file(location) and location.endswith('.podspec')) + + +def parse(location): + """ + Return a Package object from a .podspec file or None. + """ + if not is_podspec(location): + return + + podspec_object = Spec() + podspec_data = podspec_object.parse_spec(location) + return build_package(podspec_data) + + +def build_package(podspec_data): + """ + Return a Package object from a package data mapping or None. + """ + name = podspec_data.get('name') + version = podspec_data.get('version') + declared_license = podspec_data.get('license') + summary = podspec_data.get('summary') + description = podspec_data.get('description') + homepage_url = podspec_data.get('homepage_url') + source = podspec_data.get('source') + authors = podspec_data.get('author') or [] + + author_names = [] + author_email = [] + if authors: + for split_author in authors: + split_author = split_author.strip() + author, email = parse_person(split_author) + author_names.append(author) + author_email.append(email) + + parties = list(party_mapper(author_names, author_email)) + + package = CocoapodsPackage( + name=name, + version=version, + vcs_url=source, + source_packages=list(source.split('\n')), + description=description, + declared_license=declared_license, + homepage_url=homepage_url, + parties=parties + ) + + return package + + +def party_mapper(author, email): + """ + Yields a Party object with author and email. + """ + for person in author: + yield models.Party( + type=models.party_person, + name=person, + role='author') + + for person in email: + yield models.Party( + type=models.party_person, + email=person, + role='email') + + +person_parser = re.compile( + r'^(?P[\w\s(),-_.,]+)' + r'=>' + r'(?P[\S+]+$)' +).match + +person_parser_only_name = re.compile( + r'^(?P[\w\s(),-_.,]+)' +).match + + +def parse_person(person): + """ + Return name and email from person string. + + https://guides.cocoapods.org/syntax/podspec.html#authors + Author can be in the form: + s.author = 'Rohit Potter' + or + s.author = 'Rohit Potter=>rohit@gmail.com' + Author check: + >>> p = parse_person('Rohit Potter=>rohit@gmail.com') + >>> assert p == ('Rohit Potter', 'rohit@gmail.com') + >>> p = parse_person('Rohit Potter') + >>> assert p == ('Rohit Potter', None) + """ + parsed = person_parser(person) + if not parsed: + parsed = person_parser_only_name(person) + name = parsed.group('name') + email = None + else: + name = parsed.group('name') + email = parsed.group('email') + + return name, email \ No newline at end of file diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index 2d5820a3dbd..f1da47da56a 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -1,5 +1,4 @@ - -# Copyright (c) 2019 nexB Inc. and others. All rights reserved. +# Copyright (c) nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/scancode-toolkit/ # The ScanCode software is licensed under the Apache License version 2.0. # Data generated with ScanCode require an acknowledgment. @@ -31,10 +30,44 @@ import re import attr +from packageurl import PackageURL + + +TRACE = False + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) +@attr.s() +class GoModule(object): + namespace = attr.ib(default=None) + name = attr.ib(default=None) + version = attr.ib(default=None) + module = attr.ib(default=None) + require = attr.ib(default=None) + exclude = attr.ib(default=None) + + def purl(self, include_version=True): + version = None + if include_version: + version = self.version + return PackageURL( + type='golang', + namespace=self.namespace, + name=self.name, + version=version + ).to_string() + + +####################################################################################### """ -https://golang.org/ref/mod#go.mod-files +This part handles go.mod files from Go. +See https://golang.org/ref/mod#go.mod-files for details For example: @@ -61,8 +94,7 @@ module github.com/alecthomas/participle For example: ->>> ob = GoMod() ->>> p = ob.parse_module('module github.com/alecthomas/participle') +>>> p = parse_module('module github.com/alecthomas/participle') >>> assert p.group('module') == ('github.com/alecthomas/participle') require or exclude can be in the form @@ -73,140 +105,183 @@ github.com/davecgh/go-spew v1.1.1 For example: ->>> ob = GoMod() - ->>> p = ob.parse_require('require github.com/davecgh/go-spew v1.1.1') +>>> p = parse_dep_link('require github.com/davecgh/go-spew v1.1.1') >>> assert p.group('namespace') == ('github.com/davecgh') >>> assert p.group('name') == ('go-spew') >>> assert p.group('version') == ('v1.1.1') ->>> p = ob.parse_exclude('exclude github.com/davecgh/go-spew v1.1.1') +>>> p = parse_dep_link('exclude github.com/davecgh/go-spew v1.1.1') >>> assert p.group('namespace') == ('github.com/davecgh') >>> assert p.group('name') == ('go-spew') >>> assert p.group('version') == ('v1.1.1') ->>> p = ob.parse_dep_link('github.com/davecgh/go-spew v1.1.1') +>>> p = parse_dep_link('github.com/davecgh/go-spew v1.1.1') >>> assert p.group('namespace') == ('github.com/davecgh') >>> assert p.group('name') == ('go-spew') >>> assert p.group('version') == ('v1.1.1') """ +# Regex expressions to parse different types of go.mod file dependency +parse_module_name = re.compile( + r'^module\s*' + r'(?P[^\s]*)' + r'\/' + r'(?P[^\s]*)\s*' +).match + +parse_dep_link = re.compile( + r'.*?(\s)*' + r'(?P[^\s]*)' + r'\/' + r'(?P[^\s]*)' + r'\s*' + r'(?P[^\s]*)\s*' +).match + + +def preprocess(line): + """ + Return line string after removing commented portion and excess spaces. + """ + if "//" in line: + line = line[:line.index('//')] + line = line.strip() + return line + + +def parse_gomod(location): + """ + Return a dictionary containing all the important go.mod file data. + """ + with io.open(location, encoding='utf-8', closefd=True) as data: + lines = data.readlines() + + gomods = GoModule() + require = [] + exclude = [] + + for i, line in enumerate(lines): + line = preprocess(line) + + parsed_module_name = parse_module_name(line) + if parsed_module_name: + gomods.name = parsed_module_name.group('name') + gomods.namespace = parsed_module_name.group('namespace') + + if 'require' in line and '(' in line: + for req in lines[i+1:]: + req = preprocess(req) + if ')' in req: + break + parsed_dep_link = parse_dep_link(req) + if parsed_dep_link: + require.append(GoModule( + namespace=parsed_dep_link.group('namespace'), + name=parsed_dep_link.group('name'), + version=parsed_dep_link.group('version') + ) + ) + continue + + if 'exclude' in line and '(' in line: + for exc in lines[i+1:]: + exc = preprocess(exc) + if ')' in exc: + break + parsed_dep_link = parse_dep_link(exc) + if parsed_dep_link: + exclude.append(GoModule( + namespace=parsed_dep_link.group('namespace'), + name=parsed_dep_link.group('name'), + version=parsed_dep_link.group('version') + ) + ) + continue + + parsed_dep = parse_dep_link(line) + if 'require' in line: + require.append(GoModule( + namespace=parsed_dep.group('namespace'), + name=parsed_dep.group('name'), + version=parsed_dep.group('version') + ) + ) + continue + + if 'exclude' in line: + exclude.append(GoModule( + namespace=parsed_dep.group('namespace'), + name=parsed_dep.group('name'), + version=parsed_dep.group('version') + ) + ) + continue + + gomods.require = require + gomods.exclude = exclude + + return gomods + + +####################################################################################### +""" +This part handles go.sum files from Go. +See https://blog.golang.org/using-go-modules for details -TRACE = False +A go.sum file contains pinned Go modules checksums of two styles: -logger = logging.getLogger(__name__) +For example: +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -if TRACE: - import sys - logging.basicConfig(stream=sys.stdout) - logger.setLevel(logging.DEBUG) +... where the line with /go.mod is for a check of that go.mod file +and the other line contains a dirhash for that path as documented as +https://pkg.go.dev/golang.org/x/mod/sumdb/dirhash +Here are some example of usage of this module:: -@attr.s() -class GoMod(object): - # Regex expressions to parse different types of dependency - parse_module = re.compile( - r'^module\s' - r'(?P[a-z].*)' - ).match - - parse_module_name = re.compile( - r'^module(\s)*' - r'(?P(.*))' - r'/' - r'(?P[^\s]*)' - ).match - - parse_require = re.compile( - r'^require(\s)*' - r'(?P(.*))' - r'/' - r'(?P[^\s]*)' - r'\s' - r'(?P(.*))' - ).match - - parse_exclude = re.compile( - r'^exclude(\s)*' - r'(?P(.*))' - r'/' - r'(?P[^\s]*)' - r'\s' - r'(?P(.*))' - ).match - - parse_dep_link = re.compile( - r'(?P(.*))' - r'/' - r'(?P[^\s]*)' - r'\s' - r'(?P(.*))' - ).match - - @classmethod - def preprocess(cls, line): - """ - Return line string after removing commented portion and excess spaces. - """ - if "//" in line: - line = line[:line.index('//')] - line = line.strip() - return line - - @classmethod - def parse_gomod(cls, location): - """ - Return a dictionary containing all the important go.mod file data. - """ - with io.open(location, encoding='utf-8', closefd=True) as data: - lines = data.readlines() - - gomod_data = {} - require = [] - exclude = [] - - for i, line in enumerate(lines): - line = cls.preprocess(line) - parsed_module = cls.parse_module(line) - if parsed_module: - gomod_data['module'] = parsed_module.group('module') - - parsed_module_name = cls.parse_module_name(line) - if parsed_module_name: - gomod_data['name'] = parsed_module_name.group('name') - gomod_data['namespace'] = parsed_module_name.group('namespace') - - parsed_require = cls.parse_require(line) - if parsed_require: - line_req = [parsed_require.group('namespace'), parsed_require.group('name'), parsed_require.group('version')] - require.append(line_req) - - parsed_exclude = cls.parse_exclude(line) - if parsed_exclude: - line_exclude = [parsed_exclude.group('namespace'), parsed_exclude.group('name'), parsed_exclude.group('version')] - exclude.append(line_exclude) - - if 'require' in line and '(' in line: - for req in lines[i+1:]: - req = cls.preprocess(req) - if ')' in req: - break - parsed_dep_link = cls.parse_dep_link(req) - if parsed_dep_link: - line_req = [parsed_dep_link.group('namespace'), parsed_dep_link.group('name'), parsed_dep_link.group('version')] - require.append(line_req) - - if 'exclude' in line and '(' in line: - for exc in lines[i+1:]: - exc = cls.preprocess(exc) - if ')' in exc: - break - parsed_dep_link = cls.parse_dep_link(exc) - if parsed_dep_link: - line_exclude = [parsed_dep_link.group('namespace'), parsed_dep_link.group('name'), parsed_dep_link.group('version')] - exclude.append(line_exclude) - - gomod_data['require'] = require - gomod_data['exclude'] = exclude - - return gomod_data +>>> p = get_dependency('github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=') +>>> assert p.group('namespace') == ('github.com/BurntSushi') +>>> assert p.group('name') == ('toml') +>>> assert p.group('version') == ('v0.3.1') +>>> assert p.group('checksum') == ('WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=') +""" + +# Regex expressions to parse go.sum file dependency +# dep example: github.com/BurntSushi/toml v0.3.1 h1:WXkYY.... +get_dependency = re.compile( + r'(?P[^\s]*)' + r'\/' + r'(?P[^\s]*)' + r'\s*' + r'(?P[^\s]*)' + r'\s*' + r'h1:(?P[^\s]*)\s*' +).match + + +def parse_gosum(location): + """ + Return a list of GoSum from parsing the go.sum file at `location`. + """ + with io.open(location, encoding='utf-8', closefd=True) as data: + lines = data.readlines() + + gosums = [] + + for line in lines: + line = line.replace('/go.mod', '') + parsed_dep = get_dependency(line) + + dep = GoModule( + namespace=parsed_dep.group('namespace'), + name=parsed_dep.group('name'), + version=parsed_dep.group('version') + ) + + if dep in gosums: + continue + + gosums.append(dep) + + return gosums diff --git a/src/packagedcode/go_sum.py b/src/packagedcode/go_sum.py deleted file mode 100644 index 5999f044929..00000000000 --- a/src/packagedcode/go_sum.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. -# ScanCode is a trademark of nexB Inc. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. - -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -import io -import itertools -import logging -import re - -import attr -from packageurl import PackageURL - -""" -This module handles go.sum files from Go. -See https://blog.golang.org/using-go-modules for details - -A go.sum file contains pinned Go modules checksums of two styles: - -For example: -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= - -... where the line with /go.mod is for a check of that go.mod file -and the other line contains a dirhash for that path as documented as -https://pkg.go.dev/golang.org/x/mod/sumdb/dirhash - -Here are some example of usage of this module:: - ->>> p = parse_dep_type2('github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=') ->>> assert p.group('namespace') == ('github.com/BurntSushi') ->>> assert p.group('name') == ('toml') ->>> assert p.group('version') == ('v0.3.1') ->>> assert p.group('checksum') == ('WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=') - ->>> p = parse_dep_type1('github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=') ->>> assert p.group('namespace') == ('github.com/BurntSushi') ->>> assert p.group('name') == ('toml') ->>> assert p.group('version') == ('v0.3.1') ->>> assert p.group('checksum') == ('xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=') -""" - - -TRACE = False - -logger = logging.getLogger(__name__) - -if TRACE: - import sys - logging.basicConfig(stream=sys.stdout) - logger.setLevel(logging.DEBUG) - - -@attr.s() -class GoSum(object): - namespace = attr.ib(default=None) - name = attr.ib(default=None) - version = attr.ib(default=None) - - @property - def purl(self): - return PackageURL( - type='golang', - namespace=self.namespace, - name=self.name, - version=self.version - ).to_string() - -# Regex expressions to parse different types of dependency -# dep_type1 example: github.com/BurntSushi/toml v0.3.1 h1:WXkYY.... -parse_dep_type1 = re.compile( - r'(?P(.*))' - r'\/' - r'(?P[^\s]*)' - r'(\s)*' - r'(?P(.*))' - r'/go.mod(\s)*h1:' - r'(?P(.*))' -).match - -# dep_type2 example: github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCN.... -parse_dep_type2 = re.compile( - r'(?P(.*))' - r'/' - r'(?P[^\s]*)' - r'(\s)*' - r'(?P(.*))' - r'(\s)*h1:' - r'(?P(.*))' -).match - - -def parse_gosum(location): - """ - Return a list of GoSum from parsing the go.sum file at `location`. - """ - with io.open(location, encoding='utf-8', closefd=True) as data: - lines = data.readlines() - - gosums = [] - - for line in lines: - parsed_dep = parse_dep_type1(line) - if not parsed_dep: - parsed_dep = parse_dep_type2(line) - - dep_obj = GoSum( - namespace=parsed_dep.group('namespace').strip(), - name=parsed_dep.group('name').strip(), - version=parsed_dep.group('version').strip() - ) - - if dep_obj not in gosums: - gosums.append(dep_obj) - - return gosums diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 328efb564ef..b74361ce537 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -1,5 +1,5 @@ -# Copyright (c) 2019 nexB Inc. and others. All rights reserved. +# Copyright (c) nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/scancode-toolkit/ # The ScanCode software is licensed under the Apache License version 2.0. # Data generated with ScanCode require an acknowledgment. @@ -36,8 +36,7 @@ from commoncode import filetype from commoncode import fileutils -from packagedcode.go_mod import GoMod -from packagedcode import go_sum +from packagedcode import go_mod from packagedcode import models @@ -73,10 +72,10 @@ class GolangPackage(models.Package): def recognize(cls, location): filename = fileutils.file_name(location).lower() if filename == 'go.mod': - gomod = GoMod.parse_gomod(location) - yield build_gomod_package(gomod) + gomods = go_mod.parse_gomod(location) + yield build_gomod_package(gomods) elif filename == 'go.sum': - gosums = go_sum.parse_gosum(location) + gosums = go_mod.parse_gosum(location) yield build_gosum_package(gosums) @classmethod @@ -88,21 +87,17 @@ def repository_homepage_url(self, baseurl=default_web_baseurl): return '{}/{}/{}'.format(baseurl, self.namespace, self.name) -def build_gomod_package(gomod): +def build_gomod_package(gomods): """ Return a Package object from a go.mod file or None. """ package_dependencies = [] - require = gomod.get('require') or [] - for namespace, name, version in require: + require = gomods.require or [] + for gomod in require: package_dependencies.append( models.DependentPackage( - purl=PackageURL( - type='golang', - namespace=namespace, - name=name - ).to_string(), - requirement=version, + purl=gomod.purl(include_version=False), + requirement=gomod.version, scope='require', is_runtime=True, is_optional=False, @@ -110,16 +105,12 @@ def build_gomod_package(gomod): ) ) - exclude = gomod.get('exclude') or [] - for namespace, name, version in exclude: + exclude = gomods.exclude or [] + for gomod in exclude: package_dependencies.append( models.DependentPackage( - purl=PackageURL( - type='golang', - namespace=namespace, - name=name - ).to_string(), - requirement=version, + purl=gomod.purl(include_version=False), + requirement=gomod.version, scope='exclude', is_runtime=True, is_optional=False, @@ -127,10 +118,10 @@ def build_gomod_package(gomod): ) ) - name = gomod.get('name') - namespace = gomod.get('namespace') - homepage_url = 'https://pkg.go.dev/{}'.format(gomod.get('module')) - vcs_url = 'https://{}.git'.format(gomod.get('module')) + name = gomods.name + namespace = gomods.namespace + homepage_url = 'https://pkg.go.dev/{}/{}'.format(gomods.namespace, gomods.name) + vcs_url = 'https://{}/{}.git'.format(gomods.namespace, gomods.name) return GolangPackage( name=name, @@ -149,7 +140,7 @@ def build_gosum_package(gosums): for gosum in gosums: package_dependencies.append( models.DependentPackage( - purl=gosum.purl, + purl=gosum.purl(), requirement=gosum.version, scope='dependency', is_runtime=True, diff --git a/src/packagedcode/opam.py b/src/packagedcode/opam.py new file mode 100644 index 00000000000..698070a83d3 --- /dev/null +++ b/src/packagedcode/opam.py @@ -0,0 +1,354 @@ + +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from collections import OrderedDict +import io +import logging +import re + +import attr +from packageurl import PackageURL + +from commoncode import filetype +from commoncode import fileutils +from packagedcode import models + + +""" +Handle opam package. +""" + +TRACE = False + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + +@attr.s() +class OpamPackage(models.Package): + metafiles = ('*opam',) + extensions = ('.opam',) + default_type = 'opam' + default_primary_language = 'Ocaml' + default_web_baseurl = 'https://opam.ocaml.org/packages' + default_download_baseurl = None + default_api_baseurl = 'https://github.com/ocaml/opam-repository/blob/master/packages' + + @classmethod + def recognize(cls, location): + yield parse(location) + + @classmethod + def get_package_root(cls, manifest_resource, codebase): + return manifest_resource.parent(codebase) + + def repository_homepage_url(self, baseurl=default_web_baseurl): + if self.name: + return '{}/{}'.format(baseurl, self.name) + + def api_data_url(self, baseurl=default_api_baseurl): + if self.name and self.version: + return '{}/{}/{}.{}/opam'.format(baseurl, self.name, self.name, self.version) + + +def is_opam(location): + return location.endswith('opam') + + +def parse(location): + """ + Return a Package object from a opam file or None. + """ + if not is_opam(location): + return + + opams = parse_opam(location) + return build_opam_package(opams) + + +def build_opam_package(opams): + """ + Return a Package from a opam file or None. + """ + package_dependencies = [] + deps = opams.get('depends') or [] + for dep in deps: + package_dependencies.append( + models.DependentPackage( + purl=dep.purl, + requirement=dep.version, + scope='dependency', + is_runtime=True, + is_optional=False, + is_resolved=False, + ) + ) + + name = opams.get('name') + version = opams.get('version') + homepage_url = opams.get('homepage') + download_url = opams.get('src') + vcs_url = opams.get('dev-repo') + bug_tracking_url = opams.get('bug-reports') + declared_license = opams.get('license') + sha1 = opams.get('sha1') + md5 = opams.get('md5') + sha256 = opams.get('sha256') + sha512 = opams.get('sha512') + + short_desc = opams.get('synopsis') or '' + long_desc = opams.get('description') or '' + if long_desc == short_desc: + long_desc = None + descriptions = [d for d in (short_desc, long_desc) if d and d.strip()] + description = '\n'.join(descriptions) + + parties = [] + authors = opams.get('authors') or [] + for author in authors: + parties.append( + models.Party( + type=models.party_person, + name=author, + role='author' + ) + ) + maintainers = opams.get('maintainer') or [] + for maintainer in maintainers: + parties.append( + models.Party( + type=models.party_person, + email=maintainer, + role='maintainer' + ) + ) + + package = OpamPackage( + name=name, + version=version, + vcs_url=vcs_url, + homepage_url=homepage_url, + download_url=download_url, + sha1=sha1, + md5=md5, + sha256=sha256, + sha512=sha512, + bug_tracking_url=bug_tracking_url, + declared_license=declared_license, + description=description, + parties=parties, + dependencies=package_dependencies + ) + + return package + +""" +Example:- + +Sample opam file(sample3.opam): +opam-version: "2.0" +version: "4.11.0+trunk" +synopsis: "OCaml development version" +depends: [ + "ocaml" {= "4.11.0" & post} + "base-unix" {post} +] +conflict-class: "ocaml-core-compiler" +flags: compiler +setenv: CAML_LD_LIBRARY_PATH = "%{lib}%/stublibs" +build: [ + ["./configure" "--prefix=%{prefix}%"] + [make "-j%{jobs}%"] +] +install: [make "install"] +maintainer: "caml-list@inria.fr" +homepage: "https://github.com/ocaml/ocaml/" +bug-reports: "https://github.com/ocaml/ocaml/issues" +authors: [ + "Xavier Leroy" + "Damien Doligez" + "Alain Frisch" + "Jacques Garrigue" +] + +>>> p = parse_opam('sample3.opam') +>>> for k, v in p.items(): +>>> print(k, v) + +Output: +opam-version 2.0 +version 4.11.0+trunk +synopsis OCaml development version +depends [Opam(name='ocaml', version='= 4.11.0 & post'), Opam(name='base-unix', version='post')] +conflict-class ocaml-core-compiler +flags compiler +setenv CAML_LD_LIBRARY_PATH = %{lib}%/stublibs +build +install make install +maintainer ['caml-list@inria.fr'] +homepage https://github.com/ocaml/ocaml/ +bug-reports https://github.com/ocaml/ocaml/issues +authors ['Xavier Leroy', 'Damien Doligez', 'Alain Frisch', 'Jacques Garrigue'] +""" + +@attr.s() +class Opam(object): + name = attr.ib(default=None) + version = attr.ib(default=None) + + @property + def purl(self): + return PackageURL( + type='opam', + name=self.name + ).to_string() + + +# Regex expressions to parse file lines +parse_file_line = re.compile( + r'(?P^(.+?))' + r'\:\s*' + r'(?P(.*))' +).match + +parse_checksum = re.compile( + r'(?P^(.+?))' + r'\=' + r'(?P(.*))' +).match + +parse_dep = re.compile( + r'^\s*\"' + r'(?P[A-z0-9\-]*)' + r'\"\s*' + r'(?P(.*))' +).match + +""" +Example: +>>> p = parse_file_line('authors: "BAP Team"') +>>> assert p.group('key') == ('authors') +>>> assert p.group('value') == ('"BAP Team"') + +>>> p = parse_file_line('md5=b7a7b7cce64eabf224d05ed9f2b9d471') +>>> assert p.group('key') == ('md5') +>>> assert p.group('value') == ('b7a7b7cce64eabf224d05ed9f2b9d471') + +>>> p = parse_dep('"bap-std" {= "1.0.0"}') +>>> assert p.group('name') == ('bap-std') +>>> assert p.group('version') == ('{= "1.0.0"}') +""" + +def parse_opam(location): + """ + Return a mapping of package data collected from the opam OCaml package manifest file at `location`. + """ + with io.open(location, encoding='utf-8') as data: + lines = data.readlines() + + opam_data = {} + + for i, line in enumerate(lines): + parsed_line = parse_file_line(line) + if parsed_line: + key = parsed_line.group('key').strip() + value = parsed_line.group('value').strip() + if key == 'description': # Get multiline description + value = '' + for cont in lines[i+1:]: + value += ' ' + cont.strip() + if '"""' in cont: + break + + opam_data[key] = clean_data(value) + + if key == 'maintainer': + stripped_val = value.strip('["] ') + stripped_val = stripped_val.split('" "') + opam_data[key] = stripped_val + elif key == 'authors': + if '[' in line: # If authors are present in multiple lines + for authors in lines[i+1:]: + value += ' ' + authors.strip() + if ']' in authors: + break + value = value.strip('["] ') + else: + value = clean_data(value) + value = value.split('" "') + opam_data[key] = value + elif key == 'depends': # Get multiline dependencies + value = [] + for dep in lines[i+1:]: + if ']' in dep: + break + parsed_dep = parse_dep(dep) + if parsed_dep: + value.append(Opam( + name=parsed_dep.group('name').strip(), + version=parsed_dep.group('version').strip('{ } ').replace('"', '') + ) + ) + opam_data[key] = value + elif key == 'src': # Get multiline src + if not value: + value = lines[i+1].strip() + opam_data[key] = clean_data(value) + elif key == 'checksum': # Get checksums + if '[' in line: + for checksum in lines[i+1:]: + checksum = checksum.strip('" ') + if ']' in checksum: + break + parsed_checksum = parse_checksum(checksum) + key = clean_data(parsed_checksum.group('key').strip()) + value = clean_data(parsed_checksum.group('value').strip()) + opam_data[key] = value + else: + value = value.strip('" ') + parsed_checksum = parse_checksum(value) + if parsed_checksum: + key = clean_data(parsed_checksum.group('key').strip()) + value = clean_data(parsed_checksum.group('value').strip()) + opam_data[key] = value + + return opam_data + + +def clean_data(data): + """ + Return data after removing unnecessary special character. + """ + for strippable in ("'", '"', '[', ']',): + data = data.replace(strippable, '') + + return data.strip() diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index d5bd4f70255..e339a2782cf 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -140,7 +140,7 @@ def parse(location): 'requirements.in': parse_requirements_txt, 'Pipfile.lock': parse_pipfile_lock, 'metadata.json': parse_metadata, - 'PKG-INFO': parse_pkg_info, + 'PKG-INFO': parse_unpackaged_source, '.whl': parse_wheel, '.egg': parse_egg_binary, '.tar.gz': parse_source_distribution, @@ -173,10 +173,13 @@ def parse_unpackaged_source(location): def parse_with_pkginfo(pkginfo): if pkginfo and pkginfo.name: + description = pkginfo.description + if not description: + description = pkginfo.summary common_data = dict( name=pkginfo.name, version=pkginfo.version, - description=pkginfo.description, + description=description, download_url=pkginfo.download_url, homepage_url=pkginfo.home_page, ) @@ -197,10 +200,11 @@ def parse_with_pkginfo(pkginfo): package.keywords = other_classifiers if declared_license: package.declared_license = declared_license - if pkginfo.maintainer: - common_data['parties'] = [] - common_data['parties'].append(models.Party( - type=models.party_person, name=pkginfo.maintainer, role='author', email=pkginfo.maintainer_email)) + if pkginfo.author_email: + parties = [] + parties.append(models.Party( + type=models.party_person, name=pkginfo.author, role='author', email=pkginfo.author_email)) + package.parties = parties return package diff --git a/src/packagedcode/rubygems.py b/src/packagedcode/rubygems.py index e513978f1e5..e6ba68a1afc 100644 --- a/src/packagedcode/rubygems.py +++ b/src/packagedcode/rubygems.py @@ -42,6 +42,7 @@ from extractcode.uncompress import get_gz_compressed_file_content from packagedcode import models from packagedcode.gemfile_lock import GemfileLockParser +from packagedcode.spec import Spec from packagedcode.utils import combine_expressions @@ -119,8 +120,7 @@ def recognize(cls, location): yield build_rubygem_package(metadata) if location.endswith('.gemspec'): - # TODO: implement me - pass + yield build_packages_from_gemspec(location) if location.endswith('Gemfile'): # TODO: implement me @@ -559,27 +559,6 @@ def get_dependencies(dependencies): ################################################################################ -def parse_gemspec(location): - raise NotImplementedError - - -def get_gemspec_data(location): - """ - Return a mapping of Gem data from parsing a .gemspec file. - """ - if not location.endswith('.gemspec'): - return - - spec = spec_defaults() - raw_spec = parse_gemspec(location) - if TRACE: - keys = raw_spec.keys() - logger.debug('\nRubygems spec keys for %(gemfile)r:\n%(keys)r' % locals()) - spec.update(raw_spec) - spec = normalize(spec) - return spec - - def spec_defaults(): """ Return a mapping with spec attribute defaults to ensure that the @@ -648,93 +627,74 @@ def normalize(gem_data, known_fields=known_fields): ) -def parse_spec(location): - pass +def build_packages_from_gemspec(location): + """ + Return RubyGem Package from gemspec file. + """ + gemspec_object = Spec() + gemspec_data = gemspec_object.parse_spec(location) + + name = gemspec_data.get('name') + version = gemspec_data.get('version') + homepage_url = gemspec_data.get('homepage_url') + summary = gemspec_data.get('summary') + description = gemspec_data.get('description') + if len(summary) > len(description): + description = summary + + declared_license = gemspec_data.get('license') + if declared_license: + declared_license = declared_license.split(',') + + author = gemspec_data.get('author') or [] + email = gemspec_data.get('email') or [] + parties = list(party_mapper(author, email)) + + package = RubyGem( + name=name, + version=version, + parties=parties, + homepage_url=homepage_url, + description=description, + declared_license=declared_license + ) + + dependencies = gemspec_data.get('dependencies', {}) or {} + package_dependencies = [] + for name, version in dependencies.items(): + package_dependencies.append( + models.DependentPackage( + purl=PackageURL( + type='gem', + name=name + ).to_string(), + requirement=', '.join(version), + scope='dependencies', + is_runtime=True, + is_optional=False, + is_resolved=False, + ) + ) + package.dependencies = package_dependencies + return package -class GemSpec(object): + +def party_mapper(author, email): """ - Represent a Gem specification. + Yields a Party object with author and email. """ - - # TODO: Check if we should use 'summary' instead of description - def __init__(self, location): - """ - Initialize from the gem spec or gem file at location. - """ - spec = parse_spec(location) - self.location = location - self.description = spec.get('description') - self.summary = spec.get('summary') - self.author = spec.get('author') - self.authors = spec.get('authors') - # can be a list - self.email = spec.get('email') - - - self.spec['licenses'] = self.map_licenses() - self.make_unique() - - def __str__(self): - return '<{}: {}>'.format(self.__class__.__name__, self.location) - - def make_unique(self): - """ - Ensure that lists in the spec only contain unique values. - """ - new_spec = {} - for key, value in self.spec.items: - if isinstance(value, list): - newlist = [] - for item in value: - if item not in newlist: - newlist.append(item) - new_spec[key] = newlist - else: - new_spec[key] = value - return new_spec - - def get_description(self): - """ - Using 'description' over 'summary' unless summary contains - more data. - See http://guides.rubygems.org/specification-reference/ - Note that it is common to see this is spec files: s.description = s.summary - """ - description = self.spec.get('description', '') - summary = self.spec.get('summary', '') - - content = description - # FIXME: we should join these. - if len(summary) > len(description): - content = summary - - content = ' '.join(content.split()) - return content.strip() - - def get_email(self): - """ - Join the list of emails as a comma-separated string. - """ - email = self.spec.get('email', u'') - if isinstance(email, list): - email = u', '.join(email) - return email - - def map_licenses(self): - licenses = self.spec.get('licenses', []) - if not isinstance(licenses, list): - licenses = [licenses] - - mapped_licenses = [] - for lic in licenses: - mapped_license = LICENSES_MAPPING.get(lic, None) - if mapped_license: - mapped_licenses.append(mapped_license) - else: - if TRACE: - logger.warning('WARNING: {}: no license mapping for: "{}"'.format(self.filename, lic)) - return mapped_licenses + for person in author: + yield models.Party( + type=models.party_person, + name=person, + role='author') + + for person in email: + yield models.Party( + type=models.party_person, + email=person, + role='email') def build_packages_from_gemfile_lock(gemfile_lock): diff --git a/src/packagedcode/spec.py b/src/packagedcode/spec.py new file mode 100644 index 00000000000..b437d2eb291 --- /dev/null +++ b/src/packagedcode/spec.py @@ -0,0 +1,178 @@ +# All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from collections import OrderedDict +import csv +import glob +import io +import logging +import os +import re + +from gemfileparser import GemfileParser + +""" +Handle Cocoapods(.podspec) and Ruby(.gemspec) files. +""" + + +TRACE = False + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + +class Spec(): + parse_name = re.compile(r'.*\.name(\s*)=(?P.*)') + parse_version = re.compile(r'.*\.version(\s*)=(?P.*)') + parse_license = re.compile(r'.*\.license(\s*)=(?P.*)') + parse_summary = re.compile(r'.*\.summary(\s*)=(?P.*)') + parse_description = re.compile(r'.*\.description(\s*)=(?P.*)') + parse_homepage = re.compile(r'.*\.homepage(\s*)=(?P.*)') + parse_source = re.compile(r'.*\.source(\s*)=(?P.*)') + + def parse_spec(self, location): + """ + Return dictionary contains podspec or gemspec file data. + """ + with io.open(location, encoding='utf-8', closefd=True) as data: + lines = data.readlines() + + spec_data = {} + + for line in lines: + line = pre_process(line) + match = self.parse_name.match(line) + if match: + name = match.group('name') + spec_data['name'] = get_stripped_data(name) + match = self.parse_version.match(line) + if match: + version = match.group('version') + spec_data['version'] = get_stripped_data(version) + match = self.parse_license.match(line) + if match: + license_value = match.group('license') + spec_data['license'] = get_stripped_data(license_value) + match = self.parse_summary.match(line) + if match: + summary = match.group('summary') + spec_data['summary'] = get_stripped_data(summary) + match = self.parse_homepage.match(line) + if match: + homepage = match.group('homepage') + spec_data['homepage_url'] = get_stripped_data(homepage) + match = self.parse_source.match(line) + if match: + source = re.sub(r'/*.*source.*?>', '', line) + stripped_source = re.sub(r',.*', '', source) + spec_data['source'] = get_stripped_data(stripped_source) + match = self.parse_description.match(line) + if match: + if location.endswith('.gemspec'): + # FIXME: description can be in single or multi-lines + # There are many different ways to write description. + description = match.group('description') + spec_data['description'] = get_stripped_data(description) + else: + spec_data['description'] = get_description(location) + if '.email' in line: + _key, _sep, value = line.rpartition('=') + stripped_emails = get_stripped_data(value) + stripped_emails = stripped_emails.strip() + stripped_emails = stripped_emails.split(',') + spec_data['email'] = stripped_emails + elif '.author' in line: + authors = re.sub(r'/*.*author.*?=', '', line) + stripped_authors = get_stripped_data(authors) + stripped_authors = re.sub(r'(\s*=>\s*)', '=>', stripped_authors) + stripped_authors = stripped_authors.strip() + stripped_authors = stripped_authors.split(',') + spec_data['author'] = stripped_authors + + parser = GemfileParser(location) + deps = parser.parse() + dependencies = OrderedDict() + for key in deps: + depends = deps.get(key, []) or [] + for dep in depends: + dependencies[dep.name] = dep.requirement + spec_data['dependencies'] = dependencies + + return spec_data + + +def pre_process(line): + """ + Return line after comments and space. + """ + if '#' in line: + line = line[:line.index('#')] + stripped_data = line.strip() + + return stripped_data + +def get_stripped_data(data): + """ + Return data after removing unnecessary special character + """ + for strippable in ("'",'"', '{', '}', '[', ']', '%q',): + data = data.replace(strippable, '') + + return data.strip() + + +def get_description(location): + """ + Return description from podspec. + + https://guides.cocoapods.org/syntax/podspec.html#description + description is in the form: + spec.description = <<-DESC + Computes the meaning of life. + Features: + 1. Is self aware + ... + 42. Likes candies. + DESC + """ + with io.open(location, encoding='utf-8', closefd=True) as data: + lines = data.readlines() + description = '' + for i, content in enumerate(lines): + if '.description' in content: + for cont in lines[i+1:]: + if 'DESC' in cont: + break + description += ' '.join([description, cont.strip()]) + break + description.strip() + return description \ No newline at end of file diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample1/Cargo.lock b/tests/packagedcode/data/cargo/cargo_lock/sample1/Cargo.lock new file mode 100644 index 00000000000..d05cf9e08a1 --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample1/Cargo.lock @@ -0,0 +1,28 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "cargo-lock" +version = "4.0.1" +dependencies = [ + "gumdrop", + "petgraph", + "semver", + "serde", + "toml", + "url", +] + +[[package]] +name = "gumdrop" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee50908bc1beeac1f2902e0b4e0cd0d844e716f5ebdc6f0cfc1163fe5e10bcde" +dependencies = [ + "gumdrop_derive", +] diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample1/output.expected.json b/tests/packagedcode/data/cargo/cargo_lock/sample1/output.expected.json new file mode 100644 index 00000000000..e5e22684ee3 --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample1/output.expected.json @@ -0,0 +1,60 @@ +{ + "type": "cargo", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Rust", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:crates/autocfg@1.0.0", + "requirement": "1.0.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/cargo-lock@4.0.1", + "requirement": "4.0.1", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/gumdrop@0.7.0", + "requirement": "0.7.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample2/Cargo.lock b/tests/packagedcode/data/cargo/cargo_lock/sample2/Cargo.lock new file mode 100644 index 00000000000..0fa2c98287d --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample2/Cargo.lock @@ -0,0 +1,13 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample2/output.expected.json b/tests/packagedcode/data/cargo/cargo_lock/sample2/output.expected.json new file mode 100644 index 00000000000..35aff5636ca --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample2/output.expected.json @@ -0,0 +1,52 @@ +{ + "type": "cargo", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Rust", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:crates/autocfg@1.0.0", + "requirement": "1.0.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/fixedbitset@0.2.0", + "requirement": "0.2.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample3/Cargo.lock b/tests/packagedcode/data/cargo/cargo_lock/sample3/Cargo.lock new file mode 100644 index 00000000000..252eea4dada --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample3/Cargo.lock @@ -0,0 +1,21 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "gumdrop" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee50908bc1beeac1f2902e0b4e0cd0d844e716f5ebdc6f0cfc1163fe5e10bcde" +dependencies = [ + "gumdrop_derive", +] + +[[package]] +name = "gumdrop_derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90454ce4de40b7ca6a8968b5ef367bdab48413962588d0d2b1638d60090c35d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample3/output.expected.json b/tests/packagedcode/data/cargo/cargo_lock/sample3/output.expected.json new file mode 100644 index 00000000000..6b55be29b65 --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample3/output.expected.json @@ -0,0 +1,52 @@ +{ + "type": "cargo", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Rust", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:crates/gumdrop@0.7.0", + "requirement": "0.7.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/gumdrop_derive@0.7.0", + "requirement": "0.7.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample4/Cargo.lock b/tests/packagedcode/data/cargo/cargo_lock/sample4/Cargo.lock new file mode 100644 index 00000000000..3cce9389652 --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample4/Cargo.lock @@ -0,0 +1,31 @@ +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" +dependencies = [ + "autocfg", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample4/output.expected.json b/tests/packagedcode/data/cargo/cargo_lock/sample4/output.expected.json new file mode 100644 index 00000000000..4f747a42334 --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample4/output.expected.json @@ -0,0 +1,68 @@ +{ + "type": "cargo", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Rust", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:crates/idna@0.2.0", + "requirement": "0.2.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/indexmap@1.3.2", + "requirement": "1.3.2", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/matches@0.1.8", + "requirement": "0.1.8", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/percent-encoding@2.1.0", + "requirement": "2.1.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample5/Cargo.lock b/tests/packagedcode/data/cargo/cargo_lock/sample5/Cargo.lock new file mode 100644 index 00000000000..07d10d30c5c --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample5/Cargo.lock @@ -0,0 +1,30 @@ +[[package]] +name = "indexmap" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" +dependencies = [ + "autocfg", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "semver" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "394cec28fa623e00903caf7ba4fa6fb9a0e260280bb8cdbbba029611108a0190" +dependencies = [ + "semver-parser", + "serde", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" diff --git a/tests/packagedcode/data/cargo/cargo_lock/sample5/output.expected.json b/tests/packagedcode/data/cargo/cargo_lock/sample5/output.expected.json new file mode 100644 index 00000000000..64f1f3af76f --- /dev/null +++ b/tests/packagedcode/data/cargo/cargo_lock/sample5/output.expected.json @@ -0,0 +1,68 @@ +{ + "type": "cargo", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Rust", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:crates/indexmap@1.3.2", + "requirement": "1.3.2", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/matches@0.1.8", + "requirement": "0.1.8", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/semver@0.10.0", + "requirement": "0.10.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:crates/semver-parser@0.7.0", + "requirement": "0.7.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/cargo/clap/Cargo.toml b/tests/packagedcode/data/cargo/cargo_toml/clap/Cargo.toml similarity index 100% rename from tests/packagedcode/data/cargo/clap/Cargo.toml rename to tests/packagedcode/data/cargo/cargo_toml/clap/Cargo.toml diff --git a/tests/packagedcode/data/cargo/clap/Cargo.toml.expected b/tests/packagedcode/data/cargo/cargo_toml/clap/Cargo.toml.expected similarity index 100% rename from tests/packagedcode/data/cargo/clap/Cargo.toml.expected rename to tests/packagedcode/data/cargo/cargo_toml/clap/Cargo.toml.expected diff --git a/tests/packagedcode/data/cargo/clippy/Cargo.toml b/tests/packagedcode/data/cargo/cargo_toml/clippy/Cargo.toml similarity index 100% rename from tests/packagedcode/data/cargo/clippy/Cargo.toml rename to tests/packagedcode/data/cargo/cargo_toml/clippy/Cargo.toml diff --git a/tests/packagedcode/data/cargo/clippy/Cargo.toml.expected b/tests/packagedcode/data/cargo/cargo_toml/clippy/Cargo.toml.expected similarity index 100% rename from tests/packagedcode/data/cargo/clippy/Cargo.toml.expected rename to tests/packagedcode/data/cargo/cargo_toml/clippy/Cargo.toml.expected diff --git a/tests/packagedcode/data/cargo/mdbook/Cargo.toml b/tests/packagedcode/data/cargo/cargo_toml/mdbook/Cargo.toml similarity index 100% rename from tests/packagedcode/data/cargo/mdbook/Cargo.toml rename to tests/packagedcode/data/cargo/cargo_toml/mdbook/Cargo.toml diff --git a/tests/packagedcode/data/cargo/mdbook/Cargo.toml.expected b/tests/packagedcode/data/cargo/cargo_toml/mdbook/Cargo.toml.expected similarity index 100% rename from tests/packagedcode/data/cargo/mdbook/Cargo.toml.expected rename to tests/packagedcode/data/cargo/cargo_toml/mdbook/Cargo.toml.expected diff --git a/tests/packagedcode/data/cargo/rustfmt/Cargo.toml b/tests/packagedcode/data/cargo/cargo_toml/rustfmt/Cargo.toml similarity index 100% rename from tests/packagedcode/data/cargo/rustfmt/Cargo.toml rename to tests/packagedcode/data/cargo/cargo_toml/rustfmt/Cargo.toml diff --git a/tests/packagedcode/data/cargo/rustfmt/Cargo.toml.expected b/tests/packagedcode/data/cargo/cargo_toml/rustfmt/Cargo.toml.expected similarity index 100% rename from tests/packagedcode/data/cargo/rustfmt/Cargo.toml.expected rename to tests/packagedcode/data/cargo/cargo_toml/rustfmt/Cargo.toml.expected diff --git a/tests/packagedcode/data/cargo/rustup/Cargo.toml b/tests/packagedcode/data/cargo/cargo_toml/rustup/Cargo.toml similarity index 100% rename from tests/packagedcode/data/cargo/rustup/Cargo.toml rename to tests/packagedcode/data/cargo/cargo_toml/rustup/Cargo.toml diff --git a/tests/packagedcode/data/cargo/rustup/Cargo.toml.1 b/tests/packagedcode/data/cargo/cargo_toml/rustup/Cargo.toml.1 similarity index 100% rename from tests/packagedcode/data/cargo/rustup/Cargo.toml.1 rename to tests/packagedcode/data/cargo/cargo_toml/rustup/Cargo.toml.1 diff --git a/tests/packagedcode/data/cargo/rustup/Cargo.toml.expected b/tests/packagedcode/data/cargo/cargo_toml/rustup/Cargo.toml.expected similarity index 100% rename from tests/packagedcode/data/cargo/rustup/Cargo.toml.expected rename to tests/packagedcode/data/cargo/cargo_toml/rustup/Cargo.toml.expected diff --git a/tests/packagedcode/data/cocoapods/podspec/BadgeHub.podspec b/tests/packagedcode/data/cocoapods/podspec/BadgeHub.podspec new file mode 100644 index 00000000000..8cc0b829fc7 --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/BadgeHub.podspec @@ -0,0 +1,43 @@ +# +# Be sure to run `pod lib lint BadgeHub.podspec' to ensure this is a +# valid spec before submitting. +# +# Any lines starting with a # are optional, but their use is encouraged +# To learn more about a Podspec see https://guides.cocoapods.org/syntax/podspec.html +# + +Pod::Spec.new do |s| + s.name = 'BadgeHub' + s.version = '0.1.1' + s.summary = 'A way to quickly add a notification bedge icon to any view.' + +# This description is used to generate tags and improve search results. +# * Think: What does it do? Why did you write it? What is the focus? +# * Try to keep it short, snappy and to the point. +# * Write the description between the DESC delimiters below. +# * Finally, don't worry about the indent, CocoaPods strips it! + + s.description = <<-DESC +Make any UIView a full fledged animated notification center. It is a way to quickly add a notification badge icon to a UIView. It make very easy to add badge to any view. + DESC + + s.homepage = 'https://github.com/jogendra/BadgeHub' + # s.screenshots = 'www.example.com/screenshots_1', 'www.example.com/screenshots_2' + s.license = { :type => 'MIT', :file => 'LICENSE' } + s.author = { 'jogendra' => 'imjog24@gmail.com' } + s.source = { :git => 'https://github.com/jogendra/BadgeHub.git', :tag => s.version.to_s } + s.social_media_url = 'https://twitter.com/jogendrafx' + + s.ios.deployment_target = '10.0' + s.swift_version = '5.0' + + s.source_files = 'BadgeHub/Classes/**/*' + + # s.resource_bundles = { + # 'BadgeHub' => ['BadgeHub/Assets/*.png'] + # } + + # s.public_header_files = 'Pod/Classes/**/*.h' + s.frameworks = 'UIKit', 'QuartzCore' + # s.dependency 'AFNetworking', '~> 2.3' +end diff --git a/tests/packagedcode/data/cocoapods/podspec/BadgeHub.podspec.expected.json b/tests/packagedcode/data/cocoapods/podspec/BadgeHub.podspec.expected.json new file mode 100644 index 00000000000..4554a8b2312 --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/BadgeHub.podspec.expected.json @@ -0,0 +1,52 @@ +{ + "type": "pods", + "namespace": null, + "name": "BadgeHub", + "version": "0.1.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "Objective-C", + "description": " Make any UIView a full fledged animated notification center. It is a way to quickly add a notification badge icon to a UIView. It make very easy to add badge to any view.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "jogendra", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "imjog24@gmail.com", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/jogendra/BadgeHub", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/jogendra/BadgeHub.git", + "copyright": null, + "license_expression": "mit AND unknown", + "declared_license": ":type => MIT, :file => LICENSE", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [ + "https://github.com/jogendra/BadgeHub.git" + ], + "purl": "pkg:pods/BadgeHub@0.1.1", + "repository_homepage_url": "https://cocoapods.org/pods/BadgeHub", + "repository_download_url": "https://github.com/jogendra/BadgeHub/archive/0.1.1.zip", + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/cocoapods/podspec/LoadingShimmer.podspec b/tests/packagedcode/data/cocoapods/podspec/LoadingShimmer.podspec new file mode 100644 index 00000000000..47c6d0c4eee --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/LoadingShimmer.podspec @@ -0,0 +1,37 @@ +# +# Be sure to run `pod lib lint LoadingShimmer.podspec' to ensure this is a +# valid spec before submitting. +# +# Any lines starting with a # are optional, but their use is encouraged +# To learn more about a Podspec see https://guides.cocoapods.org/syntax/podspec.html +# + +Pod::Spec.new do |s| + s.name = 'LoadingShimmer' + s.version = '1.0.3' + s.summary = 'An easy way to add a shimmering effect to any view with just one line of code. It is useful as an unobtrusive loading indicator.' + + s.description = <<-DESC + An easy way to add a shimmering effect to any view with just single line of code. It is useful as an unobtrusive loading indicator. This is a network request waiting for the framework, the framework to increase the dynamic effect, convenient and fast, a line of code can be used. + DESC + + s.homepage = 'https://github.com/jogendra/LoadingShimmer' + # s.screenshots = 'https://github.com/jogendra/LoadingShimmer/blob/master/Screenshots/demo.png', 'https://github.com/jogendra/LoadingShimmer/blob/master/Screenshots/shimmer.png' + s.license = { :type => 'MIT', :file => 'LICENSE' } + s.author = { 'jogendra' => 'jogendrafx@gmail.com' } + s.source = { :git => 'https://github.com/jogendra/LoadingShimmer.git', :tag => s.version.to_s } + s.social_media_url = 'https://twitter.com/jogendrafx' + + s.ios.deployment_target = '10.0' + s.swift_version = '5.0' + + s.source_files = 'LoadingShimmer/Classes/**/*' + + # s.resource_bundles = { + # 'LoadingShimmer' => ['LoadingShimmer/Assets/*.png'] + # } + + # s.public_header_files = 'Pod/Classes/**/*.h' + # s.frameworks = 'UIKit', 'MapKit' + # s.dependency 'AFNetworking', '~> 2.3' +end diff --git a/tests/packagedcode/data/cocoapods/podspec/LoadingShimmer.podspec.expected.json b/tests/packagedcode/data/cocoapods/podspec/LoadingShimmer.podspec.expected.json new file mode 100644 index 00000000000..c797a4e40e6 --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/LoadingShimmer.podspec.expected.json @@ -0,0 +1,52 @@ +{ + "type": "pods", + "namespace": null, + "name": "LoadingShimmer", + "version": "1.0.3", + "qualifiers": {}, + "subpath": null, + "primary_language": "Objective-C", + "description": " An easy way to add a shimmering effect to any view with just single line of code. It is useful as an unobtrusive loading indicator. This is a network request waiting for the framework, the framework to increase the dynamic effect, convenient and fast, a line of code can be used.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "jogendra", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "jogendrafx@gmail.com", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/jogendra/LoadingShimmer", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/jogendra/LoadingShimmer.git", + "copyright": null, + "license_expression": "mit AND unknown", + "declared_license": ":type => MIT, :file => LICENSE", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [ + "https://github.com/jogendra/LoadingShimmer.git" + ], + "purl": "pkg:pods/LoadingShimmer@1.0.3", + "repository_homepage_url": "https://cocoapods.org/pods/LoadingShimmer", + "repository_download_url": "https://github.com/jogendra/LoadingShimmer/archive/1.0.3.zip", + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/cocoapods/podspec/Starscream.podspec b/tests/packagedcode/data/cocoapods/podspec/Starscream.podspec new file mode 100644 index 00000000000..c657d2617a1 --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/Starscream.podspec @@ -0,0 +1,16 @@ +Pod::Spec.new do |s| + s.name = "Starscream" + s.version = "4.0.3" + s.summary = "A conforming WebSocket RFC 6455 client library in Swift." + s.homepage = "https://github.com/daltoniam/Starscream" + s.license = 'Apache License, Version 2.0' + s.author = {'Dalton Cherry' => 'http://daltoniam.com', 'Austin Cherry' => 'http://austincherry.me'} + s.source = { :git => 'https://github.com/daltoniam/Starscream.git', :tag => "#{s.version}"} + s.social_media_url = 'http://twitter.com/daltoniam' + s.ios.deployment_target = '8.0' + s.osx.deployment_target = '10.10' + s.tvos.deployment_target = '9.0' + s.watchos.deployment_target = '2.0' + s.source_files = 'Sources/**/*.swift' + s.swift_version = '5.0' +end diff --git a/tests/packagedcode/data/cocoapods/podspec/Starscream.podspec.expected.json b/tests/packagedcode/data/cocoapods/podspec/Starscream.podspec.expected.json new file mode 100644 index 00000000000..45368686efb --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/Starscream.podspec.expected.json @@ -0,0 +1,66 @@ +{ + "type": "pods", + "namespace": null, + "name": "Starscream", + "version": "4.0.3", + "qualifiers": {}, + "subpath": null, + "primary_language": "Objective-C", + "description": null, + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Dalton Cherry", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Austin Cherry", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "http://daltoniam.com", + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "http://austincherry.me", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/daltoniam/Starscream", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/daltoniam/Starscream.git", + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": "Apache License, Version 2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [ + "https://github.com/daltoniam/Starscream.git" + ], + "purl": "pkg:pods/Starscream@4.0.3", + "repository_homepage_url": "https://cocoapods.org/pods/Starscream", + "repository_download_url": "https://github.com/daltoniam/Starscream/archive/4.0.3.zip", + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/cocoapods/podspec/SwiftLib.podspec b/tests/packagedcode/data/cocoapods/podspec/SwiftLib.podspec new file mode 100644 index 00000000000..e18f147facc --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/SwiftLib.podspec @@ -0,0 +1,140 @@ +# +# Be sure to run `pod spec lint SwiftLib.podspec' to ensure this is a +# valid spec and to remove all comments including this before submitting the spec. +# +# To learn more about Podspec attributes see https://docs.cocoapods.org/specification.html +# To see working Podspecs in the CocoaPods repo see https://github.com/CocoaPods/Specs/ +# + +Pod::Spec.new do |spec| + + # ――― Spec Metadata ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # These will help people to find your library, and whilst it + # can feel like a chore to fill in it's definitely to your advantage. The + # summary should be tweet-length, and the description more in depth. + # + + spec.name = "SwiftLib" + spec.version = "0.0.1" + spec.summary = "A CocoaPods library written in Swift" + + # This description is used to generate tags and improve search results. + # * Think: What does it do? Why did you write it? What is the focus? + # * Try to keep it short, snappy and to the point. + # * Write the description between the DESC delimiters below. + # * Finally, don't worry about the indent, CocoaPods strips it! + spec.description = <<-DESC +This CocoaPods library helps you perform calculation. + DESC + + spec.homepage = "https://github.com/alizainprasla/swiftlib" + # spec.screenshots = "www.example.com/screenshots_1.gif", "www.example.com/screenshots_2.gif" + + + # ――― Spec License ――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # Licensing your code is important. See https://choosealicense.com for more info. + # CocoaPods will detect a license file if there is a named LICENSE* + # Popular ones are 'MIT', 'BSD' and 'Apache License, Version 2.0'. + # + + # spec.license = "MIT (example)" + spec.license = { :type => "MIT", :file => "LICENSE" } + + + # ――― Author Metadata ――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # Specify the authors of the library, with email addresses. Email addresses + # of the authors are extracted from the SCM log. E.g. $ git log. CocoaPods also + # accepts just a name if you'd rather not provide an email address. + # + # Specify a social_media_url where others can refer to, for example a twitter + # profile URL. + # + + spec.author = { "alizainprasla" => "alizainprasla@gmail.com" } + # Or just: spec.author = "jeantimex" + # spec.authors = { "jeantimex" => "jean.timex@gmail.com" } + # spec.social_media_url = "https://twitter.com/jeantimex" + + # ――― Platform Specifics ――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # If this Pod runs only on iOS or OS X, then specify the platform and + # the deployment target. You can optionally include the target after the platform. + # + + # spec.platform = :ios + # spec.platform = :ios, "5.0" + + # When using multiple platforms + spec.ios.deployment_target = "12.1" + # spec.osx.deployment_target = "10.7" + # spec.watchos.deployment_target = "2.0" + # spec.tvos.deployment_target = "9.0" + + spec.swift_version = "4.2" + + + # ――― Source Location ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # Specify the location from where the source should be retrieved. + # Supports git, hg, bzr, svn and HTTP. + # + + spec.source = { :git => "https://github.com/alizainprasla/swiftlib.git", :tag => "#{spec.version}" } + + + # ――― Source Code ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # CocoaPods is smart about how it includes source code. For source files + # giving a folder will include any swift, h, m, mm, c & cpp files. + # For header files it will include any header in the folder. + # Not including the public_header_files will make all headers public. + # + + spec.source_files = "SwiftLib/**/*.{h,m,swift}" + # spec.exclude_files = "Classes/Exclude" + + # spec.public_header_files = "Classes/**/*.h" + + + # ――― Resources ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # A list of resources included with the Pod. These are copied into the + # target bundle with a build phase script. Anything else will be cleaned. + # You can preserve files from being cleaned, please don't preserve + # non-essential files like tests, examples and documentation. + # + + # spec.resource = "icon.png" + # spec.resources = "Resources/*.png" + + # spec.preserve_paths = "FilesToSave", "MoreFilesToSave" + + + # ――― Project Linking ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # Link your library with frameworks, or libraries. Libraries do not include + # the lib prefix of their name. + # + + # spec.framework = "SomeFramework" + # spec.frameworks = "SomeFramework", "AnotherFramework" + + # spec.library = "iconv" + # spec.libraries = "iconv", "xml2" + + + # ――― Project Settings ――――――――――――――――――――――――――――――――――――――――――――――――――――――――― # + # + # If your library depends on compiler flags you can set them in the xcconfig hash + # where they will only apply to your library. If you depend on other Podspecs + # you can include multiple dependencies to ensure it works. + + # spec.requires_arc = true + + # spec.xcconfig = { "HEADER_SEARCH_PATHS" => "$(SDKROOT)/usr/include/libxml2" } + # spec.dependency "JSONKit", "~> 1.4" + +end \ No newline at end of file diff --git a/tests/packagedcode/data/cocoapods/podspec/SwiftLib.podspec.expected.json b/tests/packagedcode/data/cocoapods/podspec/SwiftLib.podspec.expected.json new file mode 100644 index 00000000000..2985d494aaa --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/SwiftLib.podspec.expected.json @@ -0,0 +1,52 @@ +{ + "type": "pods", + "namespace": null, + "name": "SwiftLib", + "version": "0.0.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "Objective-C", + "description": " This CocoaPods library helps you perform calculation.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "alizainprasla", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "alizainprasla@gmail.com", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/alizainprasla/swiftlib", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/alizainprasla/swiftlib.git", + "copyright": null, + "license_expression": "mit AND unknown", + "declared_license": ":type => MIT, :file => LICENSE", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [ + "https://github.com/alizainprasla/swiftlib.git" + ], + "purl": "pkg:pods/SwiftLib@0.0.1", + "repository_homepage_url": "https://cocoapods.org/pods/SwiftLib", + "repository_download_url": "https://github.com/alizainprasla/swiftlib/archive/0.0.1.zip", + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/cocoapods/podspec/nanopb.podspec b/tests/packagedcode/data/cocoapods/podspec/nanopb.podspec new file mode 100644 index 00000000000..40264b2d353 --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/nanopb.podspec @@ -0,0 +1,33 @@ +Pod::Spec.new do |s| + s.name = "nanopb" + # CocoaPods minor version is minor * 10,000 + patch * 100 + fourth + s.version = "1.30905.0" + s.summary = "Protocol buffers with small code size." + + s.description = <<-DESC + Nanopb is a small code-size Protocol Buffers implementation + in ansi C. It is especially suitable for use in + microcontrollers, but fits any memory restricted system. + DESC + + s.homepage = "https://github.com/nanopb/nanopb" + s.license = { :type => 'zlib', :file => 'LICENSE.txt' } + s.author = { "Petteri Aimonen" => "jpa@nanopb.mail.kapsi.fi" } + s.source = { :git => "https://github.com/nanopb/nanopb.git", :tag => "0.3.9.5" } + + s.requires_arc = false + s.xcconfig = { 'GCC_PREPROCESSOR_DEFINITIONS' => '$(inherited) PB_FIELD_32BIT=1 PB_NO_PACKED_STRUCTS=1 PB_ENABLE_MALLOC=1' } + + s.source_files = '*.{h,c}' + s.public_header_files = '*.h' + + s.subspec 'encode' do |e| + e.public_header_files = ['pb.h', 'pb_encode.h', 'pb_common.h'] + e.source_files = ['pb.h', 'pb_common.h', 'pb_common.c', 'pb_encode.h', 'pb_encode.c'] + end + + s.subspec 'decode' do |d| + d.public_header_files = ['pb.h', 'pb_decode.h', 'pb_common.h'] + d.source_files = ['pb.h', 'pb_common.h', 'pb_common.c', 'pb_decode.h', 'pb_decode.c'] + end +end diff --git a/tests/packagedcode/data/cocoapods/podspec/nanopb.podspec.expected.json b/tests/packagedcode/data/cocoapods/podspec/nanopb.podspec.expected.json new file mode 100644 index 00000000000..f301d43621a --- /dev/null +++ b/tests/packagedcode/data/cocoapods/podspec/nanopb.podspec.expected.json @@ -0,0 +1,52 @@ +{ + "type": "pods", + "namespace": null, + "name": "nanopb", + "version": "1.30905.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Objective-C", + "description": " Nanopb is a small code-size Protocol Buffers implementation Nanopb is a small code-size Protocol Buffers implementation in ansi C. It is especially suitable for use in Nanopb is a small code-size Protocol Buffers implementation Nanopb is a small code-size Protocol Buffers implementation in ansi C. It is especially suitable for use in microcontrollers, but fits any memory restricted system.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Petteri Aimonen", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "jpa@nanopb.mail.kapsi.fi", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/nanopb/nanopb", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/nanopb/nanopb.git", + "copyright": null, + "license_expression": "unknown", + "declared_license": ":type => zlib, :file => LICENSE.txt", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [ + "https://github.com/nanopb/nanopb.git" + ], + "purl": "pkg:pods/nanopb@1.30905.0", + "repository_homepage_url": "https://cocoapods.org/pods/nanopb", + "repository_download_url": "https://github.com/nanopb/nanopb/archive/1.30905.0.zip", + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/opam/package/ocaml-variants.opam b/tests/packagedcode/data/opam/package/ocaml-variants.opam new file mode 100644 index 00000000000..b53e68c8f6f --- /dev/null +++ b/tests/packagedcode/data/opam/package/ocaml-variants.opam @@ -0,0 +1,28 @@ +opam-version: "2.0" +version: "4.11.0+trunk" +synopsis: "OCaml development version" +depends: [ + "ocaml" {= "4.11.0" & post} + "base-unix" {post} + "base-bigarray" {post} + "base-threads" {post} +] +conflict-class: "ocaml-core-compiler" +flags: compiler +setenv: CAML_LD_LIBRARY_PATH = "%{lib}%/stublibs" +build: [ + ["./configure" "--prefix=%{prefix}%"] + [make "-j%{jobs}%"] +] +install: [make "install"] +maintainer: "caml-list@inria.fr" +homepage: "https://github.com/ocaml/ocaml/" +bug-reports: "https://github.com/ocaml/ocaml/issues" +authors: [ + "Xavier Leroy" + "Damien Doligez" + "Alain Frisch" + "Jacques Garrigue" + "Didier Rémy" + "Jérôme Vouillon" +] \ No newline at end of file diff --git a/tests/packagedcode/data/opam/sample1/output.opam.expected b/tests/packagedcode/data/opam/sample1/output.opam.expected new file mode 100644 index 00000000000..a2334468c02 --- /dev/null +++ b/tests/packagedcode/data/opam/sample1/output.opam.expected @@ -0,0 +1,201 @@ +{ + "type": "opam", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Ocaml", + "description": "The MirageOS library operating system\nMirageOS is a library operating system that constructs unikernels for secure, high-performance network applications across a variety of cloud computing and mobile platforms. Code can be developed on a normal OS such as Linux or MacOS X, and then compiled into a fully-standalone, specialised unikernel that runs under the Xen hypervisor. Since Xen powers most public cloud computing infrastructure such as Amazon EC2 or Rackspace, this lets your servers run more cheaply, securely and with finer control than with a full software stack.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Thomas Gazagnaire", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Anil Madhavapeddy", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Gabriel Radanne", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Mindy Preston", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Thomas Leonard", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Nicolas Ojeda Bar", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Dave Scott", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "David Kaloper", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Hannes Mehnert", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Richard Mortier", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "anil@recoil.org", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "thomas@gazagnaire.org", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/mirage/mirage", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/mirage/mirage/issues/", + "code_view_url": null, + "vcs_url": "git+https://github.com/mirage/mirage.git", + "copyright": null, + "license_expression": "isc", + "declared_license": "ISC", + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:opam/ocaml", + "requirement": ">= 4.06.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/dune", + "requirement": ">= 2.0.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/ipaddr", + "requirement": ">= 3.0.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/functoria", + "requirement": ">= 3.0.2", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/bos", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/astring", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/logs", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/stdlib-shims", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/mirage-runtime", + "requirement": ">= 3.7.0 & <= version", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/alcotest", + "requirement": "with-test", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} diff --git a/tests/packagedcode/data/opam/sample1/sample1.opam b/tests/packagedcode/data/opam/sample1/sample1.opam new file mode 100644 index 00000000000..8f61662522f --- /dev/null +++ b/tests/packagedcode/data/opam/sample1/sample1.opam @@ -0,0 +1,43 @@ +opam-version: "2.0" +maintainer: ["anil@recoil.org" "thomas@gazagnaire.org"] +authors: ["Thomas Gazagnaire" "Anil Madhavapeddy" "Gabriel Radanne" + "Mindy Preston" "Thomas Leonard" "Nicolas Ojeda Bar" + "Dave Scott" "David Kaloper" "Hannes Mehnert" "Richard Mortier"] +homepage: "https://github.com/mirage/mirage" +bug-reports: "https://github.com/mirage/mirage/issues/" +dev-repo: "git+https://github.com/mirage/mirage.git" +license: "ISC" +tags: ["org:mirage" "org:xapi-project"] +doc: "https://mirage.github.io/mirage/" + +build: [ + ["dune" "subst"] {pinned} + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] + +depends: [ + "ocaml" {>= "4.06.0"} + "dune" {>= "2.0.0"} + "ipaddr" {>= "3.0.0"} + "functoria" {>= "3.0.2"} + "bos" + "astring" + "logs" + "stdlib-shims" + "mirage-runtime" {>= "3.7.0" & <= version} + "alcotest" {with-test} +] +synopsis: "The MirageOS library operating system" +description: """ +MirageOS is a library operating system that constructs unikernels for +secure, high-performance network applications across a variety of +cloud computing and mobile platforms. Code can be developed on a +normal OS such as Linux or MacOS X, and then compiled into a +fully-standalone, specialised unikernel that runs under the Xen +hypervisor. + +Since Xen powers most public cloud computing infrastructure such as +Amazon EC2 or Rackspace, this lets your servers run more cheaply, +securely and with finer control than with a full software stack. +""" \ No newline at end of file diff --git a/tests/packagedcode/data/opam/sample2/output.opam.expected b/tests/packagedcode/data/opam/sample2/output.opam.expected new file mode 100644 index 00000000000..87a1f8eaeab --- /dev/null +++ b/tests/packagedcode/data/opam/sample2/output.opam.expected @@ -0,0 +1,99 @@ +{ + "type": "opam", + "namespace": null, + "name": "js_of_ocaml", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Ocaml", + "description": "Compiler from OCaml bytecode to Javascript\nJs_of_ocaml is a compiler from OCaml bytecode to JavaScript. It makes it possible to run pure OCaml programs in JavaScript environment like browsers and Node.js", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Ocsigen team", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "dev@ocsigen.org", + "url": null + } + ], + "keywords": [], + "homepage_url": "http://ocsigen.github.io/js_of_ocaml", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/ocsigen/js_of_ocaml/issues", + "code_view_url": null, + "vcs_url": "git+https://github.com/ocsigen/js_of_ocaml.git", + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:opam/ocaml", + "requirement": ">= 4.02.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/dune", + "requirement": ">= 1.11.1", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/ocaml-migrate-parsetree", + "requirement": ">= 1.4", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/ppx_tools_versioned", + "requirement": ">= 5.2.3", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/uchar", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/js_of_ocaml-compiler", + "requirement": "= version", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:opam/js_of_ocaml", + "repository_homepage_url": "https://opam.ocaml.org/packages/js_of_ocaml", + "repository_download_url": null, + "api_data_url": null +} diff --git a/tests/packagedcode/data/opam/sample2/sample2.opam b/tests/packagedcode/data/opam/sample2/sample2.opam new file mode 100644 index 00000000000..d6bb91fdbb8 --- /dev/null +++ b/tests/packagedcode/data/opam/sample2/sample2.opam @@ -0,0 +1,25 @@ +pam-version: "2.0" +maintainer: "dev@ocsigen.org" +authors: "Ocsigen team" +bug-reports: "https://github.com/ocsigen/js_of_ocaml/issues" +homepage: "http://ocsigen.github.io/js_of_ocaml" +dev-repo: "git+https://github.com/ocsigen/js_of_ocaml.git" +synopsis: "Compiler from OCaml bytecode to Javascript" +description: """ +Js_of_ocaml is a compiler from OCaml bytecode to JavaScript. +It makes it possible to run pure OCaml programs in JavaScript +environment like browsers and Node.js +""" + +name: "js_of_ocaml" + +build: [["dune" "build" "-p" name "-j" jobs]] + +depends: [ + "ocaml" {>= "4.02.0"} + "dune" {>= "1.11.1"} + "ocaml-migrate-parsetree" {>= "1.4"} + "ppx_tools_versioned" {>= "5.2.3"} + "uchar" + "js_of_ocaml-compiler" {= version} +] \ No newline at end of file diff --git a/tests/packagedcode/data/opam/sample3/output.opam.expected b/tests/packagedcode/data/opam/sample3/output.opam.expected new file mode 100644 index 00000000000..c221bbef12e --- /dev/null +++ b/tests/packagedcode/data/opam/sample3/output.opam.expected @@ -0,0 +1,89 @@ +{ + "type": "opam", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Ocaml", + "description": "Interactive theorem prover based on lambda-tree syntax", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Andrew Gacek", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Yuting Wang", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Kaustuv Chaudhuri", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "kaustuv@chaudhuri.info", + "url": null + } + ], + "keywords": [], + "homepage_url": "http://abella-prover.org", + "download_url": "https://codeload.github.com/abella-prover/abella/tar.gz/v2.0.2", + "size": null, + "sha1": null, + "md5": "d7d94e62e9c0a0dbeeded9fa0104e18f", + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "gpl-3.0", + "declared_license": "GPL-3.0-only", + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:opam/ocaml", + "requirement": ">= 4.01.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/ocamlfind", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/ocamlbuild", + "requirement": "build", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} diff --git a/tests/packagedcode/data/opam/sample3/sample3.opam b/tests/packagedcode/data/opam/sample3/sample3.opam new file mode 100644 index 00000000000..b7eead158e3 --- /dev/null +++ b/tests/packagedcode/data/opam/sample3/sample3.opam @@ -0,0 +1,21 @@ +opam-version: "2.0" +maintainer: "kaustuv@chaudhuri.info" +authors: [ + "Andrew Gacek" + "Yuting Wang" + "Kaustuv Chaudhuri" +] +homepage: "http://abella-prover.org" +license: "GPL-3.0-only" +build: [[make]] +depends: [ + "ocaml" {>= "4.01.0"} + "ocamlfind" + "ocamlbuild" {build} +] +synopsis: "Interactive theorem prover based on lambda-tree syntax" +extra-files: ["abella.install" "md5=73ccb44a028691726373a70edb080f16"] +url { + src: "https://codeload.github.com/abella-prover/abella/tar.gz/v2.0.2" + checksum: "md5=d7d94e62e9c0a0dbeeded9fa0104e18f" +} \ No newline at end of file diff --git a/tests/packagedcode/data/opam/sample4/opam b/tests/packagedcode/data/opam/sample4/opam new file mode 100644 index 00000000000..be191055ea1 --- /dev/null +++ b/tests/packagedcode/data/opam/sample4/opam @@ -0,0 +1,36 @@ +opam-version: "2.0" +maintainer: "Christophe.Troestler@umons.ac.be" +authors: [ "Christophe Troestler" + "Vincent Hugot" ] +license: "LGPL-3.0-only with OCaml-LGPL-linking-exception" +homepage: "https://github.com/Chris00/ANSITerminal" +dev-repo: "git+https://github.com/Chris00/ANSITerminal.git" +bug-reports: "https://github.com/Chris00/ANSITerminal/issues" +tags: [ "terminal" ] +build: [ + ["ocaml" "setup.ml" "-configure" "--prefix" prefix] + ["ocaml" "setup.ml" "-build"] + ["ocaml" "setup.ml" "-configure" "--enable-tests"] {with-test} + ["ocaml" "setup.ml" "-build"] {with-test} + ["ocaml" "setup.ml" "-test"] {with-test} + ["ocaml" "setup.ml" "-doc"] {with-doc} +] +install: ["ocaml" "setup.ml" "-install"] +remove: [["ocamlfind" "remove" "ANSITerminal"]] +depends: [ + "ocaml" {< "4.05.0"} + "base-unix" + "ocamlbuild" {build} + "ocamlfind" {build} +] +synopsis: "Basic control of ANSI compliant terminals and the windows shell." +description: """ +ANSITerminal is a module allowing to use the colors and cursor +movements on ANSI terminals. It also works on the windows shell (but +this part is currently work in progress).""" +flags: light-uninstall +url { + src: + "https://download.ocamlcore.org/ansiterminal/ansiterminal/0.6.2/ANSITerminal-0.6.2.tar.gz" + checksum: "md5=b7a7b7cce64eabf224d05ed9f2b9d471" +} diff --git a/tests/packagedcode/data/opam/sample4/output.opam.expected b/tests/packagedcode/data/opam/sample4/output.opam.expected new file mode 100644 index 00000000000..28d9249c4bd --- /dev/null +++ b/tests/packagedcode/data/opam/sample4/output.opam.expected @@ -0,0 +1,90 @@ +{ + "type": "opam", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Ocaml", + "description": "Basic control of ANSI compliant terminals and the windows shell.\nANSITerminal is a module allowing to use the colors and cursor movements on ANSI terminals. It also works on the windows shell (but this part is currently work in progress).", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Christophe Troestler", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Vincent Hugot", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "Christophe.Troestler@umons.ac.be", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/Chris00/ANSITerminal", + "download_url": "https://download.ocamlcore.org/ansiterminal/ansiterminal/0.6.2/ANSITerminal-0.6.2.tar.gz", + "size": null, + "sha1": null, + "md5": "b7a7b7cce64eabf224d05ed9f2b9d471", + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/Chris00/ANSITerminal/issues", + "code_view_url": null, + "vcs_url": "git+https://github.com/Chris00/ANSITerminal.git", + "copyright": null, + "license_expression": "lgpl-3.0 WITH ocaml-lgpl-linking-exception", + "declared_license": "LGPL-3.0-only with OCaml-LGPL-linking-exception", + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:opam/ocaml", + "requirement": "< 4.05.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/base-unix", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/ocamlbuild", + "requirement": "build", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/ocamlfind", + "requirement": "build", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} diff --git a/tests/packagedcode/data/opam/sample5/opam b/tests/packagedcode/data/opam/sample5/opam new file mode 100644 index 00000000000..9bb3a34a14a --- /dev/null +++ b/tests/packagedcode/data/opam/sample5/opam @@ -0,0 +1,34 @@ +opam-version: "2.0" +name: "bap-elf" +version: "1.0.0" +maintainer: "Ivan Gotovchits " +authors: "BAP Team" +homepage: "https://github.com/BinaryAnalysisPlatform/bap/" +bug-reports: "https://github.com/BinaryAnalysisPlatform/bap/issues" +dev-repo: "git://github.com/BinaryAnalysisPlatform/bap/" +license: "MIT" +build: [ + ["./configure" "--prefix=%{prefix}%" "--enable-elf"] + [make] +] + +install: [[make "install"]] + +remove: [ + ["ocamlfind" "remove" "bap-elf"] + ["ocamlfind" "remove" "bap-plugin-elf_loader"] + ["bapbundle" "remove" "elf_loader.plugin"] +] + +depends: [ + "ocaml" + "bap-std" {= "1.0.0"} + "bap-dwarf" {= "1.0.0"} + "camlp4" + "bitstring" {< "3.0.0"} +] +synopsis: "BAP ELF parser and loader written in native OCaml" +url { + src: "https://github.com/BinaryAnalysisPlatform/bap/archive/v1.0.0.tar.gz" + checksum: "md5=07dce66dd871e448652d8885283c6631" +} \ No newline at end of file diff --git a/tests/packagedcode/data/opam/sample5/output.opam.expected b/tests/packagedcode/data/opam/sample5/output.opam.expected new file mode 100644 index 00000000000..64a234049e4 --- /dev/null +++ b/tests/packagedcode/data/opam/sample5/output.opam.expected @@ -0,0 +1,91 @@ +{ + "type": "opam", + "namespace": null, + "name": "bap-elf", + "version": "1.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ocaml", + "description": "BAP ELF parser and loader written in native OCaml", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "BAP Team", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "Ivan Gotovchits ", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/BinaryAnalysisPlatform/bap/", + "download_url": "https://github.com/BinaryAnalysisPlatform/bap/archive/v1.0.0.tar.gz", + "size": null, + "sha1": null, + "md5": "07dce66dd871e448652d8885283c6631", + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/BinaryAnalysisPlatform/bap/issues", + "code_view_url": null, + "vcs_url": "git://github.com/BinaryAnalysisPlatform/bap/", + "copyright": null, + "license_expression": "mit", + "declared_license": "MIT", + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:opam/ocaml", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/bap-std", + "requirement": "= 1.0.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/bap-dwarf", + "requirement": "= 1.0.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/camlp4", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/bitstring", + "requirement": "< 3.0.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:opam/bap-elf@1.0.0", + "repository_homepage_url": "https://opam.ocaml.org/packages/bap-elf", + "repository_download_url": null, + "api_data_url": "https://github.com/ocaml/opam-repository/blob/master/packages/bap-elf/bap-elf.1.0.0/opam" +} diff --git a/tests/packagedcode/data/opam/sample6/output.opam.expected b/tests/packagedcode/data/opam/sample6/output.opam.expected new file mode 100644 index 00000000000..eebac0435ac --- /dev/null +++ b/tests/packagedcode/data/opam/sample6/output.opam.expected @@ -0,0 +1,59 @@ +{ + "type": "opam", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Ocaml", + "description": "Library to make OCaml program act as a Windows service", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Romain Beauxis \"] license: \"GPL-2.0\" homepage: \"https://github.com/savonet/ocaml-winsvc\" bug-reports: \"https://github.com/savonet/ocaml-winsvc/issues\" depends: [ \"dune\" {> \"2.0\"}", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "The Savonet Team ", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/savonet/ocaml-winsvc", + "download_url": "https://github.com/savonet/ocaml-winsvc/archive/v1.0.0.tar.gz", + "size": null, + "sha1": null, + "md5": "86d48dc11dd66adac6daadbecb5f6888", + "sha256": null, + "sha512": "30e208d35ed7eb30e90d5fd4f0dde3ff4f527155df90e2d9cffadec15513b65b72503fc223bd784203f2b9081f68bedd5a2b157ffb0b2d9b765546dac1094875", + "bug_tracking_url": "https://github.com/savonet/ocaml-winsvc/issues", + "code_view_url": null, + "vcs_url": "git+https://github.com/savonet/ocaml-winsvc.git", + "copyright": null, + "license_expression": "gpl-2.0", + "declared_license": "GPL-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:opam/dune", + "requirement": "> 2.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/opam/sample6/sample6.opam b/tests/packagedcode/data/opam/sample6/sample6.opam new file mode 100644 index 00000000000..e03646c1f7f --- /dev/null +++ b/tests/packagedcode/data/opam/sample6/sample6.opam @@ -0,0 +1,33 @@ +opam-version: "2.0" +synopsis: "Library to make OCaml program act as a Windows service" +maintainer: ["The Savonet Team "] +authors: ["Romain Beauxis "] +license: "GPL-2.0" +homepage: "https://github.com/savonet/ocaml-winsvc" +bug-reports: "https://github.com/savonet/ocaml-winsvc/issues" +depends: [ + "dune" {> "2.0"} +] +build: [ + ["dune" "subst"] {pinned} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/savonet/ocaml-winsvc.git" +available: [ os = "win32" ] +url { + src: "https://github.com/savonet/ocaml-winsvc/archive/v1.0.0.tar.gz" + checksum: [ + "md5=86d48dc11dd66adac6daadbecb5f6888" + "sha512=30e208d35ed7eb30e90d5fd4f0dde3ff4f527155df90e2d9cffadec15513b65b72503fc223bd784203f2b9081f68bedd5a2b157ffb0b2d9b765546dac1094875" + ] +} diff --git a/tests/packagedcode/data/opam/sample7/output.opam.expected b/tests/packagedcode/data/opam/sample7/output.opam.expected new file mode 100644 index 00000000000..c8bdcc5d058 --- /dev/null +++ b/tests/packagedcode/data/opam/sample7/output.opam.expected @@ -0,0 +1,90 @@ +{ + "type": "opam", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Ocaml", + "description": "Basic control of ANSI compliant terminals and the windows shell\nANSITerminal is a module allowing to use the colors and cursor movements on ANSI terminals. It also works on the windows shell (but this part is currently work in progress).", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Christophe Troestler", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Vincent Hugot", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "Christophe Troestler ", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/Chris00/ANSITerminal", + "download_url": "https://github.com/Chris00/ANSITerminal/releases/download/0.8/ANSITerminal-0.8.tbz", + "size": null, + "sha1": null, + "md5": "b3fc33f17823e85c86a4d9cf4498c40e", + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/Chris00/ANSITerminal/issues", + "code_view_url": null, + "vcs_url": "git+https://github.com/Chris00/ANSITerminal.git", + "copyright": null, + "license_expression": "lgpl-3.0 WITH ocaml-lgpl-linking-exception", + "declared_license": "LGPL-3.0-only with OCaml-LGPL-linking-exception", + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:opam/ocaml", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/jbuilder", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/base-bytes", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/base-unix", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/opam/sample7/sample7.opam b/tests/packagedcode/data/opam/sample7/sample7.opam new file mode 100644 index 00000000000..1e976f1b0f1 --- /dev/null +++ b/tests/packagedcode/data/opam/sample7/sample7.opam @@ -0,0 +1,31 @@ +opam-version: "2.0" +maintainer: "Christophe Troestler " +authors: [ "Christophe Troestler" + "Vincent Hugot" ] +license: "LGPL-3.0-only with OCaml-LGPL-linking-exception" +homepage: "https://github.com/Chris00/ANSITerminal" +dev-repo: "git+https://github.com/Chris00/ANSITerminal.git" +bug-reports: "https://github.com/Chris00/ANSITerminal/issues" +doc: "https://Chris00.github.io/ANSITerminal/doc" +tags: [ "terminal" ] +build: [ + ["jbuilder" "subst" "-p" name] {pinned} + ["jbuilder" "build" "-p" name "-j" jobs] + ["jbuilder" "runtest" "-p" name "-j" jobs] {with-test} +] +depends: [ + "ocaml" + "jbuilder" + "base-bytes" + "base-unix" +] +synopsis: "Basic control of ANSI compliant terminals and the windows shell" +description: """ +ANSITerminal is a module allowing to use the colors and cursor +movements on ANSI terminals. It also works on the windows shell (but +this part is currently work in progress).""" +url { + src: + "https://github.com/Chris00/ANSITerminal/releases/download/0.8/ANSITerminal-0.8.tbz" + checksum: "md5=b3fc33f17823e85c86a4d9cf4498c40e" +} diff --git a/tests/packagedcode/data/opam/sample8/opam b/tests/packagedcode/data/opam/sample8/opam new file mode 100644 index 00000000000..be191055ea1 --- /dev/null +++ b/tests/packagedcode/data/opam/sample8/opam @@ -0,0 +1,36 @@ +opam-version: "2.0" +maintainer: "Christophe.Troestler@umons.ac.be" +authors: [ "Christophe Troestler" + "Vincent Hugot" ] +license: "LGPL-3.0-only with OCaml-LGPL-linking-exception" +homepage: "https://github.com/Chris00/ANSITerminal" +dev-repo: "git+https://github.com/Chris00/ANSITerminal.git" +bug-reports: "https://github.com/Chris00/ANSITerminal/issues" +tags: [ "terminal" ] +build: [ + ["ocaml" "setup.ml" "-configure" "--prefix" prefix] + ["ocaml" "setup.ml" "-build"] + ["ocaml" "setup.ml" "-configure" "--enable-tests"] {with-test} + ["ocaml" "setup.ml" "-build"] {with-test} + ["ocaml" "setup.ml" "-test"] {with-test} + ["ocaml" "setup.ml" "-doc"] {with-doc} +] +install: ["ocaml" "setup.ml" "-install"] +remove: [["ocamlfind" "remove" "ANSITerminal"]] +depends: [ + "ocaml" {< "4.05.0"} + "base-unix" + "ocamlbuild" {build} + "ocamlfind" {build} +] +synopsis: "Basic control of ANSI compliant terminals and the windows shell." +description: """ +ANSITerminal is a module allowing to use the colors and cursor +movements on ANSI terminals. It also works on the windows shell (but +this part is currently work in progress).""" +flags: light-uninstall +url { + src: + "https://download.ocamlcore.org/ansiterminal/ansiterminal/0.6.2/ANSITerminal-0.6.2.tar.gz" + checksum: "md5=b7a7b7cce64eabf224d05ed9f2b9d471" +} diff --git a/tests/packagedcode/data/opam/sample8/output.opam.expected b/tests/packagedcode/data/opam/sample8/output.opam.expected new file mode 100644 index 00000000000..16ebc4603ac --- /dev/null +++ b/tests/packagedcode/data/opam/sample8/output.opam.expected @@ -0,0 +1,90 @@ +{ + "type": "opam", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Ocaml", + "description": "Basic control of ANSI compliant terminals and the windows shell.\nANSITerminal is a module allowing to use the colors and cursor movements on ANSI terminals. It also works on the windows shell (but this part is currently work in progress).", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Christophe Troestler", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Vincent Hugot", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "Christophe.Troestler@umons.ac.be", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/Chris00/ANSITerminal", + "download_url": "https://download.ocamlcore.org/ansiterminal/ansiterminal/0.6.2/ANSITerminal-0.6.2.tar.gz", + "size": null, + "sha1": null, + "md5": "b7a7b7cce64eabf224d05ed9f2b9d471", + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/Chris00/ANSITerminal/issues", + "code_view_url": null, + "vcs_url": "git+https://github.com/Chris00/ANSITerminal.git", + "copyright": null, + "license_expression": "lgpl-3.0 WITH ocaml-lgpl-linking-exception", + "declared_license": "LGPL-3.0-only with OCaml-LGPL-linking-exception", + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:opam/ocaml", + "requirement": "< 4.05.0", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/base-unix", + "requirement": "", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/ocamlbuild", + "requirement": "build", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/ocamlfind", + "requirement": "build", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/plugin/help.txt b/tests/packagedcode/data/plugin/help.txt index 545bd87d155..ca1913b66f6 100644 --- a/tests/packagedcode/data/plugin/help.txt +++ b/tests/packagedcode/data/plugin/help.txt @@ -51,7 +51,7 @@ Package: cab -------------------------------------------- Package: cargo class: packagedcode.cargo:RustCargoCrate - metafiles: Cargo.toml + metafiles: Cargo.toml, Cargo.lock -------------------------------------------- Package: chef @@ -205,6 +205,18 @@ Package: nuget extensions: .nupkg filetypes: zip archive, microsoft ooxml +-------------------------------------------- +Package: opam + class: packagedcode.opam:OpamPackage + metafiles: *opam + extensions: .opam + +-------------------------------------------- +Package: pods + class: packagedcode.cocoapods:CocoapodsPackage + metafiles: *.podspec + extensions: .podspec + -------------------------------------------- Package: pypi class: packagedcode.pypi:PythonPackage diff --git a/tests/packagedcode/data/plugin/opam-package-expected.json b/tests/packagedcode/data/plugin/opam-package-expected.json new file mode 100644 index 00000000000..df90404aa20 --- /dev/null +++ b/tests/packagedcode/data/plugin/opam-package-expected.json @@ -0,0 +1,147 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--json": "", + "--package": true, + "--processes": "-1", + "--strip-root": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 1 + } + } + ], + "files": [ + { + "path": "ocaml-variants.opam", + "type": "file", + "packages": [ + { + "type": "opam", + "namespace": null, + "name": null, + "version": "4.11.0+trunk", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ocaml", + "description": "OCaml development version", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Xavier Leroy", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Damien Doligez", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Alain Frisch", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Jacques Garrigue", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "Didier R\u00e9my", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": "J\u00e9r\u00f4me Vouillon", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": null, + "email": "caml-list@inria.fr", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/ocaml/ocaml/", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/ocaml/ocaml/issues", + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:opam/ocaml", + "requirement": "= 4.11.0 & post", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/base-unix", + "requirement": "post", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/base-bigarray", + "requirement": "post", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:opam/base-threads", + "requirement": "post", + "scope": "dependency", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + } + ], + "scan_errors": [] + } + ] +} diff --git a/tests/packagedcode/data/plugin/python-package-expected.json b/tests/packagedcode/data/plugin/python-package-expected.json index 9268972eff7..1de0242401a 100644 --- a/tests/packagedcode/data/plugin/python-package-expected.json +++ b/tests/packagedcode/data/plugin/python-package-expected.json @@ -47,9 +47,9 @@ "parties": [ { "type": "person", - "role": "", + "role": "author", "name": "Francois Granade", - "email": null, + "email": "fg@nexb.com", "url": null } ], @@ -66,7 +66,9 @@ "vcs_url": null, "copyright": null, "license_expression": "unknown", - "declared_license": "BSD", + "declared_license": { + "license": "BSD" + }, "notice_text": null, "root_path": "PKG-INFO", "dependencies": [], @@ -94,7 +96,15 @@ "primary_language": "Python", "description": "===================\npython-atomicwrites\n===================\n\n.. image:: https://travis-ci.org/untitaker/python-atomicwrites.svg?branch=master\n :target: https://travis-ci.org/untitaker/python-atomicwrites\n\n.. image:: https://ci.appveyor.com/api/projects/status/vadc4le3c27to59x/branch/master?svg=true\n :target: https://ci.appveyor.com/project/untitaker/python-atomicwrites/branch/master\n\nAtomic file writes.\n\n.. code-block:: python\n\n from atomicwrites import atomic_write\n\n with atomic_write('foo.txt', overwrite=True) as f:\n f.write('Hello world.')\n # \"foo.txt\" doesn't exist yet.\n\n # Now it does.\n\n\nFeatures that distinguish it from other similar libraries (see `Alternatives and Credit`_):\n\n- Race-free assertion that the target file doesn't yet exist. This can be\n controlled with the ``overwrite`` parameter.\n\n- Windows support, although not well-tested. The MSDN resources are not very\n explicit about which operations are atomic. I'm basing my assumptions off `a\n comment\n `_\n by `Doug Crook\n `_, who appears\n to be a Microsoft employee:\n\n FAQ: Is MoveFileEx atomic\n Frequently asked question: Is MoveFileEx atomic if the existing and new\n files are both on the same drive?\n\n The simple answer is \"usually, but in some cases it will silently fall-back\n to a non-atomic method, so don't count on it\".\n\n The implementation of MoveFileEx looks something like this: [...]\n\n The problem is if the rename fails, you might end up with a CopyFile, which\n is definitely not atomic.\n\n If you really need atomic-or-nothing, you can try calling\n NtSetInformationFile, which is unsupported but is much more likely to be\n atomic. \n\n- Simple high-level API that wraps a very flexible class-based API.\n\n- Consistent error handling across platforms.\n\n\nHow it works\n============\n\nIt uses a temporary file in the same directory as the given path. This ensures\nthat the temporary file resides on the same filesystem.\n\nThe temporary file will then be atomically moved to the target location: On\nPOSIX, it will use ``rename`` if files should be overwritten, otherwise a\ncombination of ``link`` and ``unlink``. On Windows, it uses MoveFileEx_ through\nstdlib's ``ctypes`` with the appropriate flags.\n\nNote that with ``link`` and ``unlink``, there's a timewindow where the file\nmight be available under two entries in the filesystem: The name of the\ntemporary file, and the name of the target file.\n\nAlso note that the permissions of the target file may change this way. In some\nsituations a ``chmod`` can be issued without any concurrency problems, but\nsince that is not always the case, this library doesn't do it by itself.\n\n.. _MoveFileEx: https://msdn.microsoft.com/en-us/library/windows/desktop/aa365240%28v=vs.85%29.aspx\n\nfsync\n-----\n\nOn POSIX, ``fsync`` is invoked on the temporary file after it is written (to\nflush file content and metadata), and on the parent directory after the file is\nmoved (to flush filename).\n\n``fsync`` does not take care of disks' internal buffers, but there don't seem\nto be any standard POSIX APIs for that. On OS X, ``fcntl`` is used with\n``F_FULLFSYNC`` instead of ``fsync`` for that reason.\n\nOn Windows, `_commit `_\nis used, but there are no guarantees about disk internal buffers.\n\nAlternatives and Credit\n=======================\n\nAtomicwrites is directly inspired by the following libraries (and shares a\nminimal amount of code):\n\n- The Trac project's `utility functions\n `_,\n also used in `Werkzeug `_ and\n `mitsuhiko/python-atomicfile\n `_. The idea to use\n ``ctypes`` instead of ``PyWin32`` originated there.\n\n- `abarnert/fatomic `_. Windows support\n (based on ``PyWin32``) was originally taken from there.\n\nOther alternatives to atomicwrites include:\n\n- `sashka/atomicfile `_. Originally I\n considered using that, but at the time it was lacking a lot of features I\n needed (Windows support, overwrite-parameter, overriding behavior through\n subclassing).\n\n- The `Boltons library collection `_\n features a class for atomic file writes, which seems to have a very similar\n ``overwrite`` parameter. It is lacking Windows support though.\n\nLicense\n=======\n\nLicensed under the MIT, see ``LICENSE``.\n\n\n", "release_date": null, - "parties": [], + "parties": [ + { + "type": "person", + "role": "author", + "name": "Markus Unterwaditzer", + "email": "markus@unterwaditzer.net", + "url": null + } + ], "keywords": [ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", diff --git a/tests/packagedcode/data/pypi/unpackage_source_parser-expected.json b/tests/packagedcode/data/pypi/unpackage_source_parser-expected.json index 86c07578bf8..2c75d4ff7d7 100644 --- a/tests/packagedcode/data/pypi/unpackage_source_parser-expected.json +++ b/tests/packagedcode/data/pypi/unpackage_source_parser-expected.json @@ -6,9 +6,17 @@ "qualifiers": {}, "subpath": null, "primary_language": "Python", - "description": null, + "description": "Import CSV and Excel files", "release_date": null, - "parties": [], + "parties": [ + { + "type": "person", + "role": "author", + "name": "Francois Granade", + "email": "fg@nexb.com", + "url": null + } + ], "keywords": [], "homepage_url": "http://nexb.com", "download_url": null, diff --git a/tests/packagedcode/data/pypi/wheel/parse-wheel-expected.json b/tests/packagedcode/data/pypi/wheel/parse-wheel-expected.json index a1d90514171..89bf6b7dcb6 100644 --- a/tests/packagedcode/data/pypi/wheel/parse-wheel-expected.json +++ b/tests/packagedcode/data/pypi/wheel/parse-wheel-expected.json @@ -8,7 +8,15 @@ "primary_language": "Python", "description": "===================\npython-atomicwrites\n===================\n\n.. image:: https://travis-ci.org/untitaker/python-atomicwrites.svg?branch=master\n :target: https://travis-ci.org/untitaker/python-atomicwrites\n\n.. image:: https://ci.appveyor.com/api/projects/status/vadc4le3c27to59x/branch/master?svg=true\n :target: https://ci.appveyor.com/project/untitaker/python-atomicwrites/branch/master\n\nAtomic file writes.\n\n.. code-block:: python\n\n from atomicwrites import atomic_write\n\n with atomic_write('foo.txt', overwrite=True) as f:\n f.write('Hello world.')\n # \"foo.txt\" doesn't exist yet.\n\n # Now it does.\n\n\nFeatures that distinguish it from other similar libraries (see `Alternatives and Credit`_):\n\n- Race-free assertion that the target file doesn't yet exist. This can be\n controlled with the ``overwrite`` parameter.\n\n- Windows support, although not well-tested. The MSDN resources are not very\n explicit about which operations are atomic. I'm basing my assumptions off `a\n comment\n `_\n by `Doug Crook\n `_, who appears\n to be a Microsoft employee:\n\n FAQ: Is MoveFileEx atomic\n Frequently asked question: Is MoveFileEx atomic if the existing and new\n files are both on the same drive?\n\n The simple answer is \"usually, but in some cases it will silently fall-back\n to a non-atomic method, so don't count on it\".\n\n The implementation of MoveFileEx looks something like this: [...]\n\n The problem is if the rename fails, you might end up with a CopyFile, which\n is definitely not atomic.\n\n If you really need atomic-or-nothing, you can try calling\n NtSetInformationFile, which is unsupported but is much more likely to be\n atomic. \n\n- Simple high-level API that wraps a very flexible class-based API.\n\n- Consistent error handling across platforms.\n\n\nHow it works\n============\n\nIt uses a temporary file in the same directory as the given path. This ensures\nthat the temporary file resides on the same filesystem.\n\nThe temporary file will then be atomically moved to the target location: On\nPOSIX, it will use ``rename`` if files should be overwritten, otherwise a\ncombination of ``link`` and ``unlink``. On Windows, it uses MoveFileEx_ through\nstdlib's ``ctypes`` with the appropriate flags.\n\nNote that with ``link`` and ``unlink``, there's a timewindow where the file\nmight be available under two entries in the filesystem: The name of the\ntemporary file, and the name of the target file.\n\nAlso note that the permissions of the target file may change this way. In some\nsituations a ``chmod`` can be issued without any concurrency problems, but\nsince that is not always the case, this library doesn't do it by itself.\n\n.. _MoveFileEx: https://msdn.microsoft.com/en-us/library/windows/desktop/aa365240%28v=vs.85%29.aspx\n\nfsync\n-----\n\nOn POSIX, ``fsync`` is invoked on the temporary file after it is written (to\nflush file content and metadata), and on the parent directory after the file is\nmoved (to flush filename).\n\n``fsync`` does not take care of disks' internal buffers, but there don't seem\nto be any standard POSIX APIs for that. On OS X, ``fcntl`` is used with\n``F_FULLFSYNC`` instead of ``fsync`` for that reason.\n\nOn Windows, `_commit `_\nis used, but there are no guarantees about disk internal buffers.\n\nAlternatives and Credit\n=======================\n\nAtomicwrites is directly inspired by the following libraries (and shares a\nminimal amount of code):\n\n- The Trac project's `utility functions\n `_,\n also used in `Werkzeug `_ and\n `mitsuhiko/python-atomicfile\n `_. The idea to use\n ``ctypes`` instead of ``PyWin32`` originated there.\n\n- `abarnert/fatomic `_. Windows support\n (based on ``PyWin32``) was originally taken from there.\n\nOther alternatives to atomicwrites include:\n\n- `sashka/atomicfile `_. Originally I\n considered using that, but at the time it was lacking a lot of features I\n needed (Windows support, overwrite-parameter, overriding behavior through\n subclassing).\n\n- The `Boltons library collection `_\n features a class for atomic file writes, which seems to have a very similar\n ``overwrite`` parameter. It is lacking Windows support though.\n\nLicense\n=======\n\nLicensed under the MIT, see ``LICENSE``.\n\n\n", "release_date": null, - "parties": [], + "parties": [ + { + "type": "person", + "role": "author", + "name": "Markus Unterwaditzer", + "email": "markus@unterwaditzer.net", + "url": null + } + ], "keywords": [ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", diff --git a/tests/packagedcode/data/recon/base.opam b/tests/packagedcode/data/recon/base.opam new file mode 100644 index 00000000000..3c7134d0cec --- /dev/null +++ b/tests/packagedcode/data/recon/base.opam @@ -0,0 +1,37 @@ +opam-version: "2.0" +maintainer: "opensource@janestreet.com" +authors: ["Jane Street Group, LLC "] +homepage: "https://github.com/janestreet/base" +bug-reports: "https://github.com/janestreet/base/issues" +dev-repo: "git+https://github.com/janestreet/base.git" +license: "Apache-2.0" +build: [ + ["jbuilder" "build" "-p" name "-j" jobs] +] +depends: [ + "ocaml" {>= "4.04.1" & < "4.07.0"} + "jbuilder" {>= "1.0+beta12"} + "sexplib" {>= "v0.10" & < "v0.11"} +] +depopts: [ + "base-native-int63" +] +synopsis: "Full standard library replacement for OCaml" +description: """ +Full standard library replacement for OCaml + +Base is a complete and portable alternative to the OCaml standard +library. It provides all standard functionalities one would expect +from a language standard library. It uses consistent conventions +across all of its module. + +Base aims to be usable in any context. As a result system dependent +features such as I/O are not offered by Base. They are instead +provided by companion libraries such as stdio: + + https://github.com/janestreet/stdio""" +url { + src: + "https://ocaml.janestreet.com/ocaml-core/v0.10/files/base-v0.10.0.tar.gz" + checksum: "md5=60a9db475c689720cc7fc4304e00b00e" +} diff --git a/tests/packagedcode/data/recon/opam b/tests/packagedcode/data/recon/opam new file mode 100644 index 00000000000..1ff52ef32b2 --- /dev/null +++ b/tests/packagedcode/data/recon/opam @@ -0,0 +1,47 @@ +opam-version: "2.0" +name: "bap" +version: "0.9.1" +maintainer: "Ivan Gotovchits " +authors: "BAP Team" +homepage: "https://github.com/BinaryAnalysisPlatform/bap/" +bug-reports: "https://github.com/BinaryAnalysisPlatform/bap/issues" +dev-repo: "git://github.com/BinaryAnalysisPlatform/bap/" +license: "MIT" +build: [ + ["./configure" "--prefix=%{prefix}%" "--with-cxx=`which clang++`"] + [make] +] +install: [make "install"] +remove: [ + ["ocamlfind" "remove" "bap"] + ["ocamlfind" "remove" "core_lwt"] +] + +depends: [ + "ocaml" {= "4.01.0" } + "base-unix" + "bitstring" + "cmdliner" + "cohttp" {= "0.15.0"} + "core_kernel" {= "111.28.0"} + "ezjsonm" {= "0.4.0"} + "faillib" + "lwt" + "oasis" {build & >= "0.4.0" & < "0.4.7"} + "re" + "uri" {= "1.7.2"} + "zarith" + "ocamlbuild" {build} +] +synopsis: "Binary Analysis Platform" +description: """ +BAP library provides basic facilities for performing binary +analysis using OCaml, Python or other languages. + +BAP uses LLVM library as a disassembling backend, and provide facilities to +parse files in ELF and DWARF formats.""" +flags: light-uninstall +url { + src: "https://github.com/BinaryAnalysisPlatform/bap/archive/v0.9.1.tar.gz" + checksum: "md5=f0e9d067bceb7a9d01eced6668de214c" +} diff --git a/tests/packagedcode/data/rubygems/gemspec/address_standardization.gemspec.expected.json b/tests/packagedcode/data/rubygems/gemspec/address_standardization.gemspec.expected.json index 097986ebb46..d3320b4ae69 100644 --- a/tests/packagedcode/data/rubygems/gemspec/address_standardization.gemspec.expected.json +++ b/tests/packagedcode/data/rubygems/gemspec/address_standardization.gemspec.expected.json @@ -1,54 +1,77 @@ -{ - "platform": "ruby", - "name": "mysmallidea-address_standardization", - "version": "0.4.1", - "homepage": "http://github.com/mcmire/address_standardization", - "summary": "A tiny Ruby library to quickly standardize a postal address", - "description": "A tiny Ruby library to quickly standardize a postal address", - "licenses": null, - "email": "elliot.winkler@gmail.com", - "authors": [ - "Elliot Winkler" - ], - "date": "2010-02-01 00:00:00 UTC", - "requirements": null, - "dependencies": [ - "mechanize (>= 0)", - "mcmire-context (>= 0, development)", - "mcmire-matchy (>= 0, development)" - ], - "files": [ - ".gitignore", - "README.md", - "Rakefile", - "TODO", - "address_standardization.gemspec", - "lib/address_standardization.rb", - "lib/address_standardization/abstract_service.rb", - "lib/address_standardization/address.rb", - "lib/address_standardization/class_level_inheritable_attributes.rb", - "lib/address_standardization/google_maps.rb", - "lib/address_standardization/melissa_data.rb", - "lib/address_standardization/ruby_ext.rb", - "lib/address_standardization/version.rb", - "test/google_maps_test.rb", - "test/melissa_data_test.rb", - "test/test_helper.rb" - ], - "test_files": [ - "test/google_maps_test.rb", - "test/melissa_data_test.rb", - "test/test_helper.rb" - ], - "extra_rdoc_files": [ - "README.md", - "TODO" - ], - "rubygems_version": "1.3.5", - "required_ruby_version": ">= 0", - "rubyforge_project": null, - "loaded_from": "rubygems/address_standardization.gemspec", - "original_platform": null, - "new_platform": "ruby", - "specification_version": 3 -} \ No newline at end of file +[ + { + "type": "gem", + "namespace": null, + "name": "mysmallidea-address_standardization", + "version": "0.4.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ruby", + "description": "A tiny Ruby library to quickly standardize a postal address", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Elliot Winkler", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "elliot.winkler@gmail.com", + "url": null + } + ], + "keywords": [], + "homepage_url": "http://github.com/mcmire/address_standardization", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gem/mcmire-context", + "requirement": ">= 0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/mcmire-matchy", + "requirement": ">= 0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/mechanize", + "requirement": ">= 0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gem/mysmallidea-address_standardization@0.4.1", + "repository_homepage_url": "https://rubygems.org/gems/mysmallidea-address_standardization/versions/0.4.1", + "repository_download_url": "https://rubygems.org/downloads/mysmallidea-address_standardization-0.4.1.gem", + "api_data_url": "https://rubygems.org/api/v2/rubygems/mysmallidea-address_standardization/versions/0.4.1.json" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/rubygems/gemspec/arel.gemspec.expected.json b/tests/packagedcode/data/rubygems/gemspec/arel.gemspec.expected.json index 32bcc7da9ce..adec2fe27c7 100644 --- a/tests/packagedcode/data/rubygems/gemspec/arel.gemspec.expected.json +++ b/tests/packagedcode/data/rubygems/gemspec/arel.gemspec.expected.json @@ -1,187 +1,111 @@ -{ - "platform": "ruby", - "name": "arel", - "version": "2.0.7.beta.20110429111451", - "homepage": "http://github.com/rails/arel", - "summary": "Arel is a SQL AST manager for Ruby", - "description": "Arel is a SQL AST manager for Ruby. It\n\n1. Simplifies the generation complex of SQL queries\n2. Adapts to various RDBMS systems\n\nIt is intended to be a framework framework; that is, you can build your own ORM\nwith it, focusing on innovative object and collection modeling as opposed to\ndatabase compatibility and query generation.", - "licenses": null, - "email": [ - "aaron@tenderlovemaking.com", - "bryan@brynary.com", - "miloops@gmail.com", - "nick@example.org" - ], - "authors": [ - "Aaron Patterson", - "Bryan Halmkamp", - "Emilio Tagua", - "Nick Kallen" - ], - "date": "2011-04-29 00:00:00 UTC", - "requirements": null, - "dependencies": [ - "minitest (>= 2.0.2, development)", - "hoe (>= 2.9.1, development)" - ], - "files": [ - ".autotest", - ".gemtest", - "History.txt", - "MIT-LICENSE.txt", - "Manifest.txt", - "README.markdown", - "Rakefile", - "arel.gemspec", - "lib/arel.rb", - "lib/arel/alias_predication.rb", - "lib/arel/attributes.rb", - "lib/arel/attributes/attribute.rb", - "lib/arel/compatibility/wheres.rb", - "lib/arel/crud.rb", - "lib/arel/delete_manager.rb", - "lib/arel/deprecated.rb", - "lib/arel/expression.rb", - "lib/arel/expressions.rb", - "lib/arel/factory_methods.rb", - "lib/arel/insert_manager.rb", - "lib/arel/math.rb", - "lib/arel/nodes.rb", - "lib/arel/nodes/and.rb", - "lib/arel/nodes/binary.rb", - "lib/arel/nodes/count.rb", - "lib/arel/nodes/delete_statement.rb", - "lib/arel/nodes/equality.rb", - "lib/arel/nodes/function.rb", - "lib/arel/nodes/in.rb", - "lib/arel/nodes/infix_operation.rb", - "lib/arel/nodes/inner_join.rb", - "lib/arel/nodes/insert_statement.rb", - "lib/arel/nodes/join_source.rb", - "lib/arel/nodes/named_function.rb", - "lib/arel/nodes/node.rb", - "lib/arel/nodes/ordering.rb", - "lib/arel/nodes/outer_join.rb", - "lib/arel/nodes/select_core.rb", - "lib/arel/nodes/select_statement.rb", - "lib/arel/nodes/sql_literal.rb", - "lib/arel/nodes/string_join.rb", - "lib/arel/nodes/table_alias.rb", - "lib/arel/nodes/terminal.rb", - "lib/arel/nodes/unary.rb", - "lib/arel/nodes/unqualified_column.rb", - "lib/arel/nodes/update_statement.rb", - "lib/arel/nodes/values.rb", - "lib/arel/nodes/with.rb", - "lib/arel/order_predications.rb", - "lib/arel/predications.rb", - "lib/arel/relation.rb", - "lib/arel/select_manager.rb", - "lib/arel/sql/engine.rb", - "lib/arel/sql_literal.rb", - "lib/arel/table.rb", - "lib/arel/tree_manager.rb", - "lib/arel/update_manager.rb", - "lib/arel/visitors.rb", - "lib/arel/visitors/depth_first.rb", - "lib/arel/visitors/dot.rb", - "lib/arel/visitors/ibm_db.rb", - "lib/arel/visitors/join_sql.rb", - "lib/arel/visitors/mssql.rb", - "lib/arel/visitors/mysql.rb", - "lib/arel/visitors/oracle.rb", - "lib/arel/visitors/order_clauses.rb", - "lib/arel/visitors/postgresql.rb", - "lib/arel/visitors/sqlite.rb", - "lib/arel/visitors/to_sql.rb", - "lib/arel/visitors/visitor.rb", - "lib/arel/visitors/where_sql.rb", - "test/attributes/test_attribute.rb", - "test/helper.rb", - "test/nodes/test_as.rb", - "test/nodes/test_bin.rb", - "test/nodes/test_count.rb", - "test/nodes/test_delete_statement.rb", - "test/nodes/test_equality.rb", - "test/nodes/test_insert_statement.rb", - "test/nodes/test_named_function.rb", - "test/nodes/test_node.rb", - "test/nodes/test_not.rb", - "test/nodes/test_or.rb", - "test/nodes/test_select_core.rb", - "test/nodes/test_select_statement.rb", - "test/nodes/test_sql_literal.rb", - "test/nodes/test_sum.rb", - "test/nodes/test_update_statement.rb", - "test/support/fake_record.rb", - "test/test_activerecord_compat.rb", - "test/test_attributes.rb", - "test/test_crud.rb", - "test/test_delete_manager.rb", - "test/test_factory_methods.rb", - "test/test_insert_manager.rb", - "test/test_select_manager.rb", - "test/test_table.rb", - "test/test_update_manager.rb", - "test/visitors/test_depth_first.rb", - "test/visitors/test_dot.rb", - "test/visitors/test_ibm_db.rb", - "test/visitors/test_join_sql.rb", - "test/visitors/test_mssql.rb", - "test/visitors/test_mysql.rb", - "test/visitors/test_oracle.rb", - "test/visitors/test_postgres.rb", - "test/visitors/test_sqlite.rb", - "test/visitors/test_to_sql.rb" - ], - "test_files": [ - "test/attributes/test_attribute.rb", - "test/nodes/test_as.rb", - "test/nodes/test_bin.rb", - "test/nodes/test_count.rb", - "test/nodes/test_delete_statement.rb", - "test/nodes/test_equality.rb", - "test/nodes/test_insert_statement.rb", - "test/nodes/test_named_function.rb", - "test/nodes/test_node.rb", - "test/nodes/test_not.rb", - "test/nodes/test_or.rb", - "test/nodes/test_select_core.rb", - "test/nodes/test_select_statement.rb", - "test/nodes/test_sql_literal.rb", - "test/nodes/test_sum.rb", - "test/nodes/test_update_statement.rb", - "test/test_activerecord_compat.rb", - "test/test_attributes.rb", - "test/test_crud.rb", - "test/test_delete_manager.rb", - "test/test_factory_methods.rb", - "test/test_insert_manager.rb", - "test/test_select_manager.rb", - "test/test_table.rb", - "test/test_update_manager.rb", - "test/visitors/test_depth_first.rb", - "test/visitors/test_dot.rb", - "test/visitors/test_ibm_db.rb", - "test/visitors/test_join_sql.rb", - "test/visitors/test_mssql.rb", - "test/visitors/test_mysql.rb", - "test/visitors/test_oracle.rb", - "test/visitors/test_postgres.rb", - "test/visitors/test_sqlite.rb", - "test/visitors/test_to_sql.rb" - ], - "extra_rdoc_files": [ - "History.txt", - "MIT-LICENSE.txt", - "Manifest.txt", - "README.markdown" - ], - "rubygems_version": "1.6.1", - "required_ruby_version": ">= 0", - "rubyforge_project": "arel", - "loaded_from": "rubygems/arel.gemspec", - "original_platform": null, - "new_platform": "ruby", - "specification_version": 3 -} \ No newline at end of file +[ + { + "type": "gem", + "namespace": null, + "name": "arel", + "version": "2.0.7.beta.20110429111451", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ruby", + "description": "Arel is a SQL AST manager for Ruby. It", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Aaron Patterson", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": " Bryan Halmkamp", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": " Emilio Tagua", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": " Nick Kallen", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "aaron@tenderlovemaking.com", + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": " bryan@brynary.com", + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": " miloops@gmail.com", + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": " nick@example.org", + "url": null + } + ], + "keywords": [], + "homepage_url": "http://github.com/rails/arel", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gem/minitest", + "requirement": ">= 2.0.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/hoe", + "requirement": ">= 2.9.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gem/arel@2.0.7.beta.20110429111451", + "repository_homepage_url": "https://rubygems.org/gems/arel/versions/2.0.7.beta.20110429111451", + "repository_download_url": "https://rubygems.org/downloads/arel-2.0.7.beta.20110429111451.gem", + "api_data_url": "https://rubygems.org/api/v2/rubygems/arel/versions/2.0.7.beta.20110429111451.json" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/rubygems/gemspec/cat.gemspec.expected.json b/tests/packagedcode/data/rubygems/gemspec/cat.gemspec.expected.json index 8c284842871..4e4d6f54d43 100644 --- a/tests/packagedcode/data/rubygems/gemspec/cat.gemspec.expected.json +++ b/tests/packagedcode/data/rubygems/gemspec/cat.gemspec.expected.json @@ -1,35 +1,101 @@ -{ - "platform": "ruby", - "name": "cat_", - "version": "12", - "homepage": "https://github.com/elct9620/.cat", - "summary": "The loading cat generator.", - "description": "The loading cat generator.", - "licenses": null, - "email": [ - "elct9620@frost.tw" - ], - "authors": [ - "\u84bc\u6642\u5f26\u4e5f" - ], - "date": "2019-01-07 00:00:00 UTC", - "requirements": null, - "dependencies": [ - "rack (>= 0)", - "sassc (>= 0)", - "slim (>= 0)", - "bundler (~> 1.13, development)", - "rake (~> 10.0, development)", - "rspec (~> 3.0, development)" - ], - "files": [], - "test_files": null, - "extra_rdoc_files": null, - "rubygems_version": "2.5.2.1", - "required_ruby_version": ">= 0", - "rubyforge_project": null, - "loaded_from": "rubygems/cat.gemspec", - "original_platform": null, - "new_platform": "ruby", - "specification_version": 4 -} +[ + { + "type": "gem", + "namespace": null, + "name": "cat_", + "version": "12", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ruby", + "description": "The loading cat generator.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "\u84bc\u6642\u5f26\u4e5f", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "elct9620@frost.tw", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/elct9620/.cat", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gem/bundler", + "requirement": "~> 1.13", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/rake", + "requirement": "~> 10.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/rspec", + "requirement": "~> 3.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/rack", + "requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/sassc", + "requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/slim", + "requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gem/cat_@12", + "repository_homepage_url": "https://rubygems.org/gems/cat_/versions/12", + "repository_download_url": "https://rubygems.org/downloads/cat_-12.gem", + "api_data_url": "https://rubygems.org/api/v2/rubygems/cat_/versions/12.json" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/rubygems/gemspec/github.gemspec b/tests/packagedcode/data/rubygems/gemspec/github.gemspec new file mode 100644 index 00000000000..38a12dd4088 --- /dev/null +++ b/tests/packagedcode/data/rubygems/gemspec/github.gemspec @@ -0,0 +1,31 @@ +# -*- encoding: utf-8 -*- +$:.push File.expand_path("../lib", __FILE__) +require "github/version" + +Gem::Specification.new do |s| + s.name = "github" + s.version = GitHub::VERSION + s.platform = Gem::Platform::RUBY + s.authors = ['Chris Wanstrath', 'Kevin Ballard', 'Scott Chacon', 'Dr Nic Williams'] + s.email = ["drnicwilliams@gmail.com"] + s.homepage = "https://github.com/defunkt/github-gem" + s.summary = "The official `github` command line helper for simplifying your GitHub experience." + s.description = "The official `github` command line helper for simplifying your GitHub experience." + + s.rubyforge_project = "github" + + s.files = `git ls-files`.split("\n") + s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") + s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } + s.require_paths = ["lib"] + + s.add_dependency "text-hyphen", "1.0.0" + s.add_dependency "text-format", "1.0.0" + s.add_dependency "highline", "~> 1.6" + s.add_dependency "json_pure", "~> 1.5.1" + s.add_dependency "launchy", "~> 2.0.2" + + s.add_development_dependency "rake" + s.add_development_dependency "rspec", "~>1.3.1" + s.add_development_dependency "activerecord", "~>3.0.0" +end diff --git a/tests/packagedcode/data/rubygems/gemspec/github.gemspec.expected.json b/tests/packagedcode/data/rubygems/gemspec/github.gemspec.expected.json new file mode 100644 index 00000000000..b2ec1e617b2 --- /dev/null +++ b/tests/packagedcode/data/rubygems/gemspec/github.gemspec.expected.json @@ -0,0 +1,138 @@ +[ + { + "type": "gem", + "namespace": null, + "name": "github", + "version": "GitHub::VERSION", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ruby", + "description": "The official `github` command line helper for simplifying your GitHub experience.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Chris Wanstrath", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": " Kevin Ballard", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": " Scott Chacon", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": " Dr Nic Williams", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "drnicwilliams@gmail.com", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/defunkt/github-gem", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gem/rake", + "requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/rspec", + "requirement": "~>1.3.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/activerecord", + "requirement": "~>3.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/text-hyphen", + "requirement": "1.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/text-format", + "requirement": "1.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/highline", + "requirement": "~> 1.6", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/json_pure", + "requirement": "~> 1.5.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/launchy", + "requirement": "~> 2.0.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gem/github@GitHub::VERSION", + "repository_homepage_url": "https://rubygems.org/gems/github/versions/GitHub::VERSION", + "repository_download_url": "https://rubygems.org/downloads/github-GitHub::VERSION.gem", + "api_data_url": "https://rubygems.org/api/v2/rubygems/github/versions/GitHub::VERSION.json" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/rubygems/gemspec/mecab-ruby.gemspec b/tests/packagedcode/data/rubygems/gemspec/mecab-ruby.gemspec new file mode 100644 index 00000000000..f49f356b937 --- /dev/null +++ b/tests/packagedcode/data/rubygems/gemspec/mecab-ruby.gemspec @@ -0,0 +1,16 @@ +Gem::Specification.new do |s| + s.name = %q{mecab-ruby} + s.version = '0.99' + s.author = 'Taku Kudo' + s.date = '2011-12-24' + s.description = "Ruby bindings for MeCab, a morphological analyzer." + s.email = 'taku@chasen.org' + s.extensions = [ 'extconf.rb' ] + s.files = [ 'AUTHORS', 'BSD', 'COPYING', 'GPL', 'LGPL', + 'MeCab_wrap.cpp', 'README', 'bindings.html', + 'extconf.rb', 'mecab-ruby.gemspec', 'test.rb' ] + s.has_rdoc = false + s.homepage = 'http://mecab.sourceforge.net/' + s.summary = 'Ruby bindings for MeCab.' +end + diff --git a/tests/packagedcode/data/rubygems/gemspec/mecab-ruby.gemspec.expected.json b/tests/packagedcode/data/rubygems/gemspec/mecab-ruby.gemspec.expected.json new file mode 100644 index 00000000000..9efe0a01382 --- /dev/null +++ b/tests/packagedcode/data/rubygems/gemspec/mecab-ruby.gemspec.expected.json @@ -0,0 +1,52 @@ +[ + { + "type": "gem", + "namespace": null, + "name": "mecab-ruby", + "version": "0.99", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ruby", + "description": "Ruby bindings for MeCab, a morphological analyzer.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Taku Kudo", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "taku@chasen.org", + "url": null + } + ], + "keywords": [], + "homepage_url": "http://mecab.sourceforge.net/", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gem/mecab-ruby@0.99", + "repository_homepage_url": "https://rubygems.org/gems/mecab-ruby/versions/0.99", + "repository_download_url": "https://rubygems.org/downloads/mecab-ruby-0.99.gem", + "api_data_url": "https://rubygems.org/api/v2/rubygems/mecab-ruby/versions/0.99.json" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/rubygems/gemspec/oj.gemspec.expected.json b/tests/packagedcode/data/rubygems/gemspec/oj.gemspec.expected.json index e69de29bb2d..138456eba1d 100644 --- a/tests/packagedcode/data/rubygems/gemspec/oj.gemspec.expected.json +++ b/tests/packagedcode/data/rubygems/gemspec/oj.gemspec.expected.json @@ -0,0 +1,85 @@ +[ + { + "type": "gem", + "namespace": null, + "name": "oj", + "version": "::Oj::VERSION", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ruby", + "description": "The fastest JSON parser and object serializer.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Peter Ohler", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "peter@ohler.com", + "url": null + } + ], + "keywords": [], + "homepage_url": "http://www.ohler.com/oj", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gem/rake-compiler", + "requirement": ">= 0.9, < 2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/minitest", + "requirement": "~> 5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/test-unit", + "requirement": "~> 3.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/wwtd", + "requirement": "~> 0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gem/oj@::Oj::VERSION", + "repository_homepage_url": "https://rubygems.org/gems/oj/versions/::Oj::VERSION", + "repository_download_url": "https://rubygems.org/downloads/oj-::Oj::VERSION.gem", + "api_data_url": "https://rubygems.org/api/v2/rubygems/oj/versions/::Oj::VERSION.json" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/rubygems/gemspec/rubocop.gemspec b/tests/packagedcode/data/rubygems/gemspec/rubocop.gemspec index 6c9fbeeb54a..8087e2de78d 100644 --- a/tests/packagedcode/data/rubygems/gemspec/rubocop.gemspec +++ b/tests/packagedcode/data/rubygems/gemspec/rubocop.gemspec @@ -11,10 +11,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 2.2.0' s.authors = ['Bozhidar Batsov', 'Jonas Arvidsson', 'Yuji Nakayama'] - s.description = <<-DESCRIPTION - Automatic Ruby code style checking tool. - Aims to enforce the community-driven Ruby Style Guide. - DESCRIPTION + s.description = "Automatic Ruby code style checking tool. Aims to enforce the community-driven Ruby Style Guide." s.email = 'rubocop@googlegroups.com' s.files = `git ls-files assets bin config lib LICENSE.txt README.md` diff --git a/tests/packagedcode/data/rubygems/gemspec/rubocop.gemspec.expected.json b/tests/packagedcode/data/rubygems/gemspec/rubocop.gemspec.expected.json index e69de29bb2d..6af1b30844e 100644 --- a/tests/packagedcode/data/rubygems/gemspec/rubocop.gemspec.expected.json +++ b/tests/packagedcode/data/rubygems/gemspec/rubocop.gemspec.expected.json @@ -0,0 +1,139 @@ +[ + { + "type": "gem", + "namespace": null, + "name": "rubocop", + "version": "RuboCop::Version::STRING", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ruby", + "description": "Automatic Ruby code style checking tool. Aims to enforce the community-driven Ruby Style Guide.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Bozhidar Batsov", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": " Jonas Arvidsson", + "email": null, + "url": null + }, + { + "type": "person", + "role": "author", + "name": " Yuji Nakayama", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "rubocop@googlegroups.com", + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/rubocop-hq/rubocop", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gem/bundler", + "requirement": ">= 1.3.0, < 3.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/rack", + "requirement": ">= 2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/jaro_winkler", + "requirement": "~> 1.5.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/parallel", + "requirement": "~> 1.10", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/parser", + "requirement": ">= 2.5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/powerpack", + "requirement": "~> 0.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/rainbow", + "requirement": ">= 2.2.2, < 4.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/ruby-progressbar", + "requirement": "~> 1.7", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/unicode-display_width", + "requirement": "~> 1.4.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gem/rubocop@RuboCop::Version::STRING", + "repository_homepage_url": "https://rubygems.org/gems/rubocop/versions/RuboCop::Version::STRING", + "repository_download_url": "https://rubygems.org/downloads/rubocop-RuboCop::Version::STRING.gem", + "api_data_url": "https://rubygems.org/api/v2/rubygems/rubocop/versions/RuboCop::Version::STRING.json" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/rubygems/gemspec/with_variables.gemspec.expected.json b/tests/packagedcode/data/rubygems/gemspec/with_variables.gemspec.expected.json index e69de29bb2d..036d15f6daa 100644 --- a/tests/packagedcode/data/rubygems/gemspec/with_variables.gemspec.expected.json +++ b/tests/packagedcode/data/rubygems/gemspec/with_variables.gemspec.expected.json @@ -0,0 +1,125 @@ +[ + { + "type": "gem", + "namespace": null, + "name": "ProviderDSL::GemDescription::NAME", + "version": "ProviderDSL::GemDescription::VERSION", + "qualifiers": {}, + "subpath": null, + "primary_language": "Ruby", + "description": "See the project home page for more information", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "ProviderDSL::GemDescription::AUTHORS", + "email": null, + "url": null + }, + { + "type": "person", + "role": "email", + "name": null, + "email": "ProviderDSL::GemDescription::EMAIL", + "url": null + } + ], + "keywords": [], + "homepage_url": "ProviderDSL::GemDescription::PAGE", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gem/rake", + "requirement": "~> 11.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/rubocop", + "requirement": "~> 0.44.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/rspec", + "requirement": "~> 3.5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/rspec-mocks", + "requirement": "~> 3.5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/ipaddress", + "requirement": "~> 0.8.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/gandi", + "requirement": "~> 3.3, >= 3.3.27", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/gcloud", + "requirement": "~> 0.21.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/google-cloud-error_reporting", + "requirement": "~> 0.21.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gem/map", + "requirement": "~> 6.6", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gem/ProviderDSL::GemDescription::NAME@ProviderDSL::GemDescription::VERSION", + "repository_homepage_url": "https://rubygems.org/gems/ProviderDSL::GemDescription::NAME/versions/ProviderDSL::GemDescription::VERSION", + "repository_download_url": "https://rubygems.org/downloads/ProviderDSL::GemDescription::NAME-ProviderDSL::GemDescription::VERSION.gem", + "api_data_url": "https://rubygems.org/api/v2/rubygems/ProviderDSL::GemDescription::NAME/versions/ProviderDSL::GemDescription::VERSION.json" + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_cargo.py b/tests/packagedcode/test_cargo.py index b9b0e3ab5ef..b89b200043b 100644 --- a/tests/packagedcode/test_cargo.py +++ b/tests/packagedcode/test_cargo.py @@ -37,33 +37,63 @@ class TestCargo(PackageTester): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') - def test_parse_clap(self): - test_file = self.get_test_loc('cargo/clap/Cargo.toml') - expected_loc = self.get_test_loc('cargo/clap/Cargo.toml.expected') + def test_parse_cargo_toml_clap(self): + test_file = self.get_test_loc('cargo/cargo_toml/clap/Cargo.toml') + expected_loc = self.get_test_loc('cargo/cargo_toml/clap/Cargo.toml.expected') package = cargo.parse(test_file) self.check_package(package, expected_loc, regen=False) - def test_parse_clippy(self): - test_file = self.get_test_loc('cargo/clippy/Cargo.toml') - expected_loc = self.get_test_loc('cargo/clippy/Cargo.toml.expected') + def test_parse_cargo_toml_clippy(self): + test_file = self.get_test_loc('cargo/cargo_toml/clippy/Cargo.toml') + expected_loc = self.get_test_loc('cargo/cargo_toml/clippy/Cargo.toml.expected') package = cargo.parse(test_file) self.check_package(package, expected_loc, regen=False) - def test_parse_mdbook(self): - test_file = self.get_test_loc('cargo/mdbook/Cargo.toml') - expected_loc = self.get_test_loc('cargo/mdbook/Cargo.toml.expected') + def test_parse_cargo_toml_mdbook(self): + test_file = self.get_test_loc('cargo/cargo_toml/mdbook/Cargo.toml') + expected_loc = self.get_test_loc('cargo/cargo_toml/mdbook/Cargo.toml.expected') package = cargo.parse(test_file) self.check_package(package, expected_loc, regen=False) - def test_parse_rustfmt(self): - test_file = self.get_test_loc('cargo/rustfmt/Cargo.toml') - expected_loc = self.get_test_loc('cargo/rustfmt/Cargo.toml.expected') + def test_parse_cargo_toml_rustfmt(self): + test_file = self.get_test_loc('cargo/cargo_toml/rustfmt/Cargo.toml') + expected_loc = self.get_test_loc('cargo/cargo_toml/rustfmt/Cargo.toml.expected') package = cargo.parse(test_file) self.check_package(package, expected_loc, regen=False) - def test_parse_rustup(self): - test_file = self.get_test_loc('cargo/rustup/Cargo.toml') - expected_loc = self.get_test_loc('cargo/rustup/Cargo.toml.expected') + def test_parse_cargo_toml_rustup(self): + test_file = self.get_test_loc('cargo/cargo_toml/rustup/Cargo.toml') + expected_loc = self.get_test_loc('cargo/cargo_toml/rustup/Cargo.toml.expected') + package = cargo.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_cargo_lock_sample1(self): + test_file = self.get_test_loc('cargo/cargo_lock/sample1/Cargo.lock') + expected_loc = self.get_test_loc('cargo/cargo_lock/sample1/output.expected.json') + package = cargo.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_cargo_lock_sample2(self): + test_file = self.get_test_loc('cargo/cargo_lock/sample2/Cargo.lock') + expected_loc = self.get_test_loc('cargo/cargo_lock/sample2/output.expected.json') + package = cargo.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_cargo_lock_sample3(self): + test_file = self.get_test_loc('cargo/cargo_lock/sample3/Cargo.lock') + expected_loc = self.get_test_loc('cargo/cargo_lock/sample3/output.expected.json') + package = cargo.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_cargo_lock_sample4(self): + test_file = self.get_test_loc('cargo/cargo_lock/sample4/Cargo.lock') + expected_loc = self.get_test_loc('cargo/cargo_lock/sample4/output.expected.json') + package = cargo.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_cargo_lock_sample5(self): + test_file = self.get_test_loc('cargo/cargo_lock/sample5/Cargo.lock') + expected_loc = self.get_test_loc('cargo/cargo_lock/sample5/output.expected.json') package = cargo.parse(test_file) self.check_package(package, expected_loc, regen=False) diff --git a/tests/packagedcode/test_cocoapods.py b/tests/packagedcode/test_cocoapods.py new file mode 100644 index 00000000000..a5d061bf08f --- /dev/null +++ b/tests/packagedcode/test_cocoapods.py @@ -0,0 +1,69 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import os +import pytest + +from commoncode.system import py2 +from packagedcode import cocoapods +from packages_test_utils import PackageTester + + +@pytest.mark.skipif(py2, reason='Does not pass on Python2') +class TestRubyGemspec(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_rubygems_can_parse_BadgeHub(self): + test_file = self.get_test_loc('cocoapods/podspec/BadgeHub.podspec') + expected_loc = self.get_test_loc('cocoapods/podspec/BadgeHub.podspec.expected.json') + packages = cocoapods.parse(test_file) + self.check_package(packages, expected_loc, regen=False) + + def test_rubygems_can_parse_LoadingShimmer(self): + test_file = self.get_test_loc('cocoapods/podspec/LoadingShimmer.podspec') + expected_loc = self.get_test_loc('cocoapods/podspec/LoadingShimmer.podspec.expected.json') + packages = cocoapods.parse(test_file) + self.check_package(packages, expected_loc, regen=False) + + def test_rubygems_can_parse_nanopb(self): + test_file = self.get_test_loc('cocoapods/podspec/nanopb.podspec') + expected_loc = self.get_test_loc('cocoapods/podspec/nanopb.podspec.expected.json') + packages = cocoapods.parse(test_file) + self.check_package(packages, expected_loc, regen=False) + + def test_rubygems_can_parse_Starscream(self): + test_file = self.get_test_loc('cocoapods/podspec/Starscream.podspec') + expected_loc = self.get_test_loc('cocoapods/podspec/Starscream.podspec.expected.json') + packages = cocoapods.parse(test_file) + self.check_package(packages, expected_loc, regen=False) + + def test_rubygems_can_parse_SwiftLib(self): + test_file = self.get_test_loc('cocoapods/podspec/SwiftLib.podspec') + expected_loc = self.get_test_loc('cocoapods/podspec/SwiftLib.podspec.expected.json') + packages = cocoapods.parse(test_file) + self.check_package(packages, expected_loc, regen=False) \ No newline at end of file diff --git a/tests/packagedcode/test_opam.py b/tests/packagedcode/test_opam.py new file mode 100644 index 00000000000..393184dc8ea --- /dev/null +++ b/tests/packagedcode/test_opam.py @@ -0,0 +1,140 @@ + +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import os +import pytest + +from packagedcode import opam + +from packages_test_utils import PackageTester + + +class TestOcaml(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_parse_sample1(self): + test_file = self.get_test_loc('opam/sample1/sample1.opam') + expected_loc = self.get_test_loc('opam/sample1/output.opam.expected') + package = opam.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_sample2(self): + test_file = self.get_test_loc('opam/sample2/sample2.opam') + expected_loc = self.get_test_loc('opam/sample2/output.opam.expected') + package = opam.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_sample3(self): + test_file = self.get_test_loc('opam/sample3/sample3.opam') + expected_loc = self.get_test_loc('opam/sample3/output.opam.expected') + package = opam.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_sample4(self): + test_file = self.get_test_loc('opam/sample4/opam') + expected_loc = self.get_test_loc('opam/sample4/output.opam.expected') + package = opam.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_sample5(self): + test_file = self.get_test_loc('opam/sample5/opam') + expected_loc = self.get_test_loc('opam/sample5/output.opam.expected') + package = opam.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_sample6(self): + test_file = self.get_test_loc('opam/sample6/sample6.opam') + expected_loc = self.get_test_loc('opam/sample6/output.opam.expected') + package = opam.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_sample7(self): + test_file = self.get_test_loc('opam/sample7/sample7.opam') + expected_loc = self.get_test_loc('opam/sample7/output.opam.expected') + package = opam.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + def test_parse_sample8(self): + test_file = self.get_test_loc('opam/sample8/opam') + expected_loc = self.get_test_loc('opam/sample8/output.opam.expected') + package = opam.parse(test_file) + self.check_package(package, expected_loc, regen=False) + + +FILE_LINE = [ + ('authors: "BAP Team"', ('authors', '"BAP Team"')), + ('homepage: "https://github.com/BinaryAnalysisPlatform/bap/"', ('homepage', '"https://github.com/BinaryAnalysisPlatform/bap/"')) +] + +CHECKSUM_LINE = [ + ('md5=b7a7b7cce64eabf224d05ed9f2b9d471', ('md5', 'b7a7b7cce64eabf224d05ed9f2b9d471')), + ('md5=86d48dc11dd66adac6daadbecb5f6888', ('md5', '86d48dc11dd66adac6daadbecb5f6888')) +] + +DEP_LINE = [ + ('"ocaml"', ('ocaml', '')), + ('"bap-std" {= "1.0.0"}', ('bap-std', '{= "1.0.0"}')), + ('"camlp4"', ('camlp4', '')), + ('"bitstring" {< "3.0.0"}', ('bitstring', '{< "3.0.0"}')) +] + + +class TestRegex(object): + @pytest.mark.parametrize('line, expected_data', FILE_LINE) + def test_parse_file_line(self, line, expected_data): + parsed_line = opam.parse_file_line(line) + line_information = parsed_line.groupdict() + key, value = line_information.get('key'), line_information.get('value') + assert (key, value) == expected_data + + @pytest.mark.parametrize('checksum, expected_data', CHECKSUM_LINE) + def test_parse_checksum(self, checksum, expected_data): + parsed_checksum = opam.parse_checksum(checksum) + checksum_information = parsed_checksum.groupdict() + key, value = checksum_information.get('key'), checksum_information.get('value') + assert (key, value) == expected_data + + @pytest.mark.parametrize('dep, expected_data', DEP_LINE) + def test_parse_dep(self, dep, expected_data): + parsed_dep = opam.parse_dep(dep) + dep_information = parsed_dep.groupdict() + name, version = dep_information.get('name'), dep_information.get('version') + assert (name, version) == expected_data + + +LINE = [ + ('"caml-list@inria.fr"', 'caml-list@inria.fr'), + ('[make "install"]', 'make install') +] + + +class TestFunction(object): + @pytest.mark.parametrize('line, expected_data', LINE) + def test_parse_file_line(self, line, expected_data): + stripped_line = opam.clean_data(line) + assert stripped_line == expected_data diff --git a/tests/packagedcode/test_plugin.py b/tests/packagedcode/test_plugin.py index dc4db2fa7cf..7032a3a4bad 100644 --- a/tests/packagedcode/test_plugin.py +++ b/tests/packagedcode/test_plugin.py @@ -130,6 +130,13 @@ def test_package_command_scan_nuget(self): run_scan_click(['--package', '--strip-root', '--processes', '-1', test_dir, '--json', result_file]) check_json_scan(expected_file, result_file, regen=False) + def test_package_command_scan_opam(self): + test_dir = self.get_test_loc('opam/package') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin/opam-package-expected.json') + run_scan_click(['--package', '--strip-root', '--processes', '-1', test_dir, '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False) + def test_package_command_scan_phpcomposer(self): test_dir = self.get_test_loc('phpcomposer/package') result_file = self.get_temp_file('json') diff --git a/tests/packagedcode/test_recognize.py b/tests/packagedcode/test_recognize.py index ec417465d58..0171840027b 100644 --- a/tests/packagedcode/test_recognize.py +++ b/tests/packagedcode/test_recognize.py @@ -34,6 +34,7 @@ from packagedcode import maven from packagedcode import npm from packagedcode import cargo +from packagedcode import opam from packagedcode import phpcomposer from packagedcode import rpm from packagedcode.recognize import recognize_packages @@ -126,6 +127,18 @@ def test_recognize_cargo(self): assert packages assert isinstance(packages[0], cargo.RustCargoCrate) + def test_recognize_opam(self): + test_file = self.get_test_loc('recon/opam') + packages = recognize_packages(test_file) + assert packages + assert isinstance(packages[0], opam.OpamPackage) + + def test_recognize_opam1(self): + test_file = self.get_test_loc('recon/base.opam') + packages = recognize_packages(test_file) + assert packages + assert isinstance(packages[0], opam.OpamPackage) + def test_recognize_composer(self): test_file = self.get_test_loc('recon/composer.json') packages = recognize_packages(test_file) diff --git a/tests/packagedcode/test_rubygems.py b/tests/packagedcode/test_rubygems.py index 28337475ee9..e7c6a8689e8 100644 --- a/tests/packagedcode/test_rubygems.py +++ b/tests/packagedcode/test_rubygems.py @@ -30,6 +30,7 @@ import io import json import os +import pytest from unittest.case import expectedFailure import saneyaml @@ -47,66 +48,57 @@ # this is a multiple personality package (Java and Ruby) # see also https://rubygems.org/downloads/jaro_winkler-1.5.1-java.gem -# NOTE: this needs to be implemented first -@expectedFailure -class TestRubyGemspec(FileBasedTesting): +@pytest.mark.skipif(py2, reason='Does not pass on Python2') +class TestRubyGemspec(PackageTester): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') - def check_gemspec(self, test_loc, expected_loc, regen=False): - test_loc = self.get_test_loc(test_loc) - expected_loc = self.get_test_loc(expected_loc) - results = rubygems.get_gemspec_data(test_loc) - - try: - # fix absolute paths for testing - rel_path = results['loaded_from'] - rel_path = [p for p in rel_path.split('/') if p] - rel_path = '/'.join(rel_path[-2:]) - results['loaded_from'] = rel_path - except: - pass - - if regen: - if py2: - mode = 'wb' - if py3: - mode = 'w' - with open(expected_loc, mode) as ex: - json.dump(results, ex, indent=2) - with io.open(expected_loc, encoding='UTF-8') as ex: - expected = json.load(ex) - - assert sorted(expected.items()) == sorted(results.items()) - def test_rubygems_can_parse_gemspec_address_standardization_gemspec(self): - self.check_gemspec( - 'rubygems/gemspec/address_standardization.gemspec', - 'rubygems/gemspec/address_standardization.gemspec.expected.json') + test_file = self.get_test_loc('rubygems/gemspec/address_standardization.gemspec') + expected_loc = self.get_test_loc('rubygems/gemspec/address_standardization.gemspec.expected.json') + packages = rubygems.RubyGem.recognize(test_file) + self.check_packages(packages, expected_loc, regen=False) def test_rubygems_can_parse_gemspec_arel_gemspec(self): - self.check_gemspec( - 'rubygems/gemspec/arel.gemspec', - 'rubygems/gemspec/arel.gemspec.expected.json') + test_file = self.get_test_loc('rubygems/gemspec/arel.gemspec') + expected_loc = self.get_test_loc('rubygems/gemspec/arel.gemspec.expected.json') + packages = rubygems.RubyGem.recognize(test_file) + self.check_packages(packages, expected_loc, regen=False) + + def test_rubygems_cat_gemspec(self): + test_file = self.get_test_loc('rubygems/gemspec/cat.gemspec') + expected_loc = self.get_test_loc('rubygems/gemspec/cat.gemspec.expected.json') + packages = rubygems.RubyGem.recognize(test_file) + self.check_packages(packages, expected_loc, regen=False) - def test_rubygems_modern_gemspec(self): - self.check_gemspec( - 'rubygems/gemspec/cat.gemspec', - 'rubygems/gemspec/cat.gemspec.expected.json') + def test_rubygems_github_gemspec(self): + test_file = self.get_test_loc('rubygems/gemspec/github.gemspec') + expected_loc = self.get_test_loc('rubygems/gemspec/github.gemspec.expected.json') + packages = rubygems.RubyGem.recognize(test_file) + self.check_packages(packages, expected_loc, regen=False) + + def test_rubygems_mecab_ruby_gemspec(self): + test_file = self.get_test_loc('rubygems/gemspec/mecab-ruby.gemspec') + expected_loc = self.get_test_loc('rubygems/gemspec/mecab-ruby.gemspec.expected.json') + packages = rubygems.RubyGem.recognize(test_file) + self.check_packages(packages, expected_loc, regen=False) def test_rubygems_oj_gemspec(self): - self.check_gemspec( - 'rubygems/gemspec/oj.gemspec', - 'rubygems/gemspec/oj.gemspec.expected.json') + test_file = self.get_test_loc('rubygems/gemspec/oj.gemspec') + expected_loc = self.get_test_loc('rubygems/gemspec/oj.gemspec.expected.json') + packages = rubygems.RubyGem.recognize(test_file) + self.check_packages(packages, expected_loc, regen=False) def test_rubygems_rubocop_gemspec(self): - self.check_gemspec( - 'rubygems/gemspec/rubocop.gemspec', - 'rubygems/gemspec/rubocop.gemspec.expected.json') - - def test_rubygems_gemspec_with_variables(self): - self.check_gemspec( - 'rubygems/gemspec/with_variables.gemspec', - 'rubygems/gemspec/with_variables.gemspec.expected.json') + test_file = self.get_test_loc('rubygems/gemspec/rubocop.gemspec') + expected_loc = self.get_test_loc('rubygems/gemspec/rubocop.gemspec.expected.json') + packages = rubygems.RubyGem.recognize(test_file) + self.check_packages(packages, expected_loc, regen=False) + + def test_rubygems_with_variables_gemspec(self): + test_file = self.get_test_loc('rubygems/gemspec/with_variables.gemspec') + expected_loc = self.get_test_loc('rubygems/gemspec/with_variables.gemspec.expected.json') + packages = rubygems.RubyGem.recognize(test_file) + self.check_packages(packages, expected_loc, regen=False) class TestRubyGemMetadata(FileBasedTesting): diff --git a/thirdparty/ftfy-4.4.3-py2-none-any.whl.ABOUT b/thirdparty/ftfy-4.4.3-py2-none-any.whl.ABOUT new file mode 100644 index 00000000000..ed68add3b87 --- /dev/null +++ b/thirdparty/ftfy-4.4.3-py2-none-any.whl.ABOUT @@ -0,0 +1,12 @@ +name: ftfy +version: 4.4.3 +about_resource: ftfy-4.4.3-py2-none-any.whl +download_url: https://files.pythonhosted.org/packages/21/5d/9385540977b00df1f3a0c0f07b7e6c15b5e7a3109d7f6ae78a0a764dab22/ftfy-4.4.3.tar.gz +license_expression: mit +copyright: Copyright (c) Rob Speer (rob@luminoso.com) +attribute: yes +package_url: pkg:pypi/ftfy@4.4.3 +licenses: + - key: mit + name: MIT License + file: mit.LICENSE diff --git a/thirdparty/ftfy-4.4.3-py3-none-any.whl b/thirdparty/ftfy-4.4.3-py3-none-any.whl new file mode 100644 index 00000000000..8d171e65f73 Binary files /dev/null and b/thirdparty/ftfy-4.4.3-py3-none-any.whl differ diff --git a/thirdparty/ftfy-4.4.3-py3-none-any.whl.ABOUT b/thirdparty/ftfy-4.4.3-py3-none-any.whl.ABOUT new file mode 100644 index 00000000000..ae2a1d82109 --- /dev/null +++ b/thirdparty/ftfy-4.4.3-py3-none-any.whl.ABOUT @@ -0,0 +1,12 @@ +name: ftfy +version: 4.4.3 +about_resource: ftfy-4.4.3-py3-none-any.whl +download_url: https://files.pythonhosted.org/packages/21/5d/9385540977b00df1f3a0c0f07b7e6c15b5e7a3109d7f6ae78a0a764dab22/ftfy-4.4.3.tar.gz +license_expression: mit +copyright: Copyright (c) Rob Speer (rob@luminoso.com) +attribute: yes +package_url: pkg:pypi/ftfy@4.4.3 +licenses: + - key: mit + name: MIT License + file: mit.LICENSE diff --git a/thirdparty/ftfy-4.4.3.tar.gz b/thirdparty/ftfy-4.4.3.tar.gz new file mode 100644 index 00000000000..63df7176fc7 Binary files /dev/null and b/thirdparty/ftfy-4.4.3.tar.gz differ diff --git a/thirdparty/ftfy-5.5.1-py3-none-any.whl b/thirdparty/ftfy-5.5.1-py3-none-any.whl deleted file mode 100644 index c15b09cc9e4..00000000000 Binary files a/thirdparty/ftfy-5.5.1-py3-none-any.whl and /dev/null differ diff --git a/thirdparty/ftfy-5.5.1-py3-none-any.whl.ABOUT b/thirdparty/ftfy-5.5.1-py3-none-any.whl.ABOUT deleted file mode 100644 index 8d89b047fce..00000000000 --- a/thirdparty/ftfy-5.5.1-py3-none-any.whl.ABOUT +++ /dev/null @@ -1,14 +0,0 @@ -name: ftfy -version: 5.5.1 -about_resource: ftfy-5.5.1-py3-none-any.whl -download_url: https://files.pythonhosted.org/packages/8f/86/df789c5834f15ae1ca53a8d4c1fc4788676c2e32112f6a786f2625d9c6e6/ftfy-5.5.1-py3-none-any.whl -license_expression: mit -copyright: Copyright (c) Robyn Speer (rspeer@luminoso.com) -attribute: yes -checksum_md5: 9b9716f707a0984f97f61569831da556 -checksum_sha1: 4b7a7c9bb0f5f3f2d9e1dd8564ea1e2681df815e -package_url: pkg:pypi/ftfy@5.5.1 -licenses: - - key: mit - name: MIT License - file: mit.LICENSE diff --git a/thirdparty/gemfileparser-0.8.0-py2.py3-none-any.whl b/thirdparty/gemfileparser-0.8.0-py2.py3-none-any.whl new file mode 100644 index 00000000000..207a08eeed9 Binary files /dev/null and b/thirdparty/gemfileparser-0.8.0-py2.py3-none-any.whl differ diff --git a/thirdparty/gemfileparser-0.8.0.tar.gz b/thirdparty/gemfileparser-0.8.0.tar.gz new file mode 100644 index 00000000000..28bb3fb21fd Binary files /dev/null and b/thirdparty/gemfileparser-0.8.0.tar.gz differ diff --git a/thirdparty/gemfileparser.ABOUT b/thirdparty/gemfileparser.ABOUT new file mode 100644 index 00000000000..bb9a4233831 --- /dev/null +++ b/thirdparty/gemfileparser.ABOUT @@ -0,0 +1,8 @@ +about_resource: gemfileparser-0.8.0-py2.py3-none-any.whl +name: gemfileparser +version: 0.8.0 +download_url: https://files.pythonhosted.org/packages/ed/b5/c3f4d21e121a65172f43b08b2699e469e3853455b5def48097f40164bb44/gemfileparser-0.8.0-py2.py3-none-any.whl +license_expression: mit +license_file: gemfileparser.LICENSE +owner: Balasankar "Balu" C +home_url: https://github.com/gemfileparser/gemfileparser diff --git a/thirdparty/gemfileparser.LICENSE b/thirdparty/gemfileparser.LICENSE new file mode 100644 index 00000000000..1b710007455 --- /dev/null +++ b/thirdparty/gemfileparser.LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2020 Gemfileparser authors (listed in AUTHORS file) + 2015-2018 Balasankar C + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.