From a3495cb55fe835181ae57a1a48b9ff8cbe7b7415 Mon Sep 17 00:00:00 2001 From: Paul Horton Date: Thu, 21 Oct 2021 14:35:17 +0100 Subject: [PATCH 1/3] feat: add support for conda #66 fix: character encoding issues on Windows #67 Signed-off-by: Paul Horton --- jake/command/sbom.py | 69 ++++++++++++++++++++++++++++++++++---------- poetry.lock | 31 ++++++++++---------- pyproject.toml | 2 +- 3 files changed, 71 insertions(+), 31 deletions(-) diff --git a/jake/command/sbom.py b/jake/command/sbom.py index 71187e7..8c1f4aa 100644 --- a/jake/command/sbom.py +++ b/jake/command/sbom.py @@ -15,16 +15,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import argparse +import sys from cyclonedx.model.bom import Bom from cyclonedx.output import BaseOutput, get_instance, OutputFormat, SchemaVersion, DEFAULT_SCHEMA_VERSION from cyclonedx.parser import BaseParser +from cyclonedx.parser.conda import CondaListJsonParser, CondaListExplicitParser from cyclonedx.parser.environment import EnvironmentParser -from cyclonedx.parser.pipenv import PipEnvFileParser -from cyclonedx.parser.poetry import PoetryFileParser -from cyclonedx.parser.requirements import RequirementsFileParser +from cyclonedx.parser.pipenv import PipEnvParser, PipEnvFileParser +from cyclonedx.parser.poetry import PoetryParser, PoetryFileParser +from cyclonedx.parser.requirements import RequirementsParser, RequirementsFileParser + from . import BaseCommand @@ -59,13 +61,26 @@ def setup_argument_parser(self, subparsers: argparse._SubParsersAction): help='generate a CycloneDX software-bill-of-materials (no vulnerabilities)', ) - parser.add_argument('-it', '--input-type', + parser.add_argument('-i', '--input', action='store', metavar='FILE_PATH', + type=argparse.FileType('r'), default=(None if sys.stdin.isatty() else sys.stdin), + help='Where to get input data from. If a path to a file is not specified directly here,' + 'then we will attempt to read data from STDIN. If there is no data on STDIN, we will ' + 'then fall back to looking for standard files in the current directory that relate ' + 'to the type of input indicated by the -t flag.', dest='sbom_input_source', + required=False) + + parser.add_argument('-t', '--type', '-it', '--input-type', help='how jake should find the packages from which to generate your SBOM.' - 'ENV = Read from the current Python Environment; PIP = read from a requirements.txt; ' - 'PIPENV = read from Pipfile.lock; POETRY = read from a poetry.lock. ' + 'ENV = Read from the current Python Environment; ' + 'CONDA = Read output from `conda list --explicit`; ' + 'CONDA_JSON = Read output from `conda list --json`; ' + 'PIP = read from a requirements.txt; ' + 'PIPENV = read from Pipfile.lock; ' + 'POETRY = read from a poetry.lock. ' '(Default = ENV)', - metavar='TYPE', choices={'ENV', 'PIP', 'PIPENV', 'POETRY'}, default='ENV', - dest='sbom_input_type') + metavar='TYPE', choices={'CONDA', 'CONDA_JSON', 'ENV', 'PIP', 'PIPENV', 'POETRY'}, + default='ENV', dest='sbom_input_type') + parser.add_argument('-o', '--output-file', help='Specify a file to output the SBOM to', metavar='PATH/TO/FILE', dest='sbom_output_file') parser.add_argument('--output-format', help='SBOM output format (default = xml)', choices={'json', 'xml'}, @@ -78,13 +93,37 @@ def _get_parser(self) -> BaseParser: if self._arguments.sbom_input_type == 'ENV': return EnvironmentParser() - if self._arguments.sbom_input_type == 'PIP': - return RequirementsFileParser(requirements_file='requirements.txt') + # All other input types require INPUT - let's grab it now if provided via STDIN or supplied FILE + input_data_fh = self._arguments.sbom_input_source + if input_data_fh: + with input_data_fh: + input_data = input_data_fh.read() + input_data_fh.close() + + if self._arguments.sbom_input_type == 'CONDA': + return CondaListExplicitParser(conda_data=input_data) + + if self._arguments.sbom_input_type == 'CONDA_JSON': + return CondaListJsonParser(conda_data=input_data) + + if self._arguments.sbom_input_type == 'PIP': + return RequirementsParser(requirements_content=input_data) + + if self._arguments.sbom_input_type == 'PIPENV': + return PipEnvParser(pipenv_contents=input_data) + + if self._arguments.sbom_input_type == 'POETRY': + return PoetryParser(poetry_lock_contents=input_data) + + else: + # No data available on STDIN or the supplied FILE, so we'll try standard filenames in the current directory + if self._arguments.sbom_input_type == 'PIP': + return RequirementsFileParser(requirements_file='requirements.txt') - if self._arguments.sbom_input_type == 'PIPENV': - return PipEnvFileParser(pipenv_lock_filename='Pipfile.lock') + if self._arguments.sbom_input_type == 'PIPENV': + return PipEnvFileParser(pipenv_lock_filename='Pipfile.lock') - if self._arguments.sbom_input_type == 'POETRY': - return PoetryFileParser(poetry_lock_filename='poetry.lock') + if self._arguments.sbom_input_type == 'POETRY': + return PoetryFileParser(poetry_lock_filename='poetry.lock') raise NotImplementedError diff --git a/poetry.lock b/poetry.lock index 156dea8..f702405 100644 --- a/poetry.lock +++ b/poetry.lock @@ -65,17 +65,18 @@ toml = ["toml"] [[package]] name = "cyclonedx-python-lib" -version = "0.8.0" +version = "0.10.2" description = "A library for producing CycloneDX SBOM (Software Bill of Materials) files." category = "main" optional = false python-versions = ">=3.6,<4.0" [package.dependencies] -importlib-metadata = ">=4.8.1,<5.0.0" +importlib-metadata = {version = ">=4.8.1,<5.0.0", markers = "python_version >= \"3.6\" and python_version < \"3.8\""} packageurl-python = ">=0.9.4,<0.10.0" requirements_parser = ">=0.2.0,<0.3.0" toml = ">=0.10.2,<0.11.0" +typing-extensions = {version = ">=3.10.0,<4.0.0", markers = "python_version >= \"3.6\" and python_version < \"3.8\""} [[package]] name = "dataclasses" @@ -95,7 +96,7 @@ python-versions = "*" [[package]] name = "filelock" -version = "3.3.0" +version = "3.3.1" description = "A platform independent file lock." category = "dev" optional = false @@ -121,7 +122,7 @@ pyflakes = ">=2.3.0,<2.4.0" [[package]] name = "idna" -version = "3.2" +version = "3.3" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false @@ -146,7 +147,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [[package]] name = "importlib-resources" -version = "5.2.2" +version = "5.3.0" description = "Read resources from Python packages" category = "dev" optional = false @@ -157,7 +158,7 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy"] [[package]] name = "mccabe" @@ -446,7 +447,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "5b57767bfd168e8ba37512984eca6dccee67e12d91786c6f058927134321e018" +content-hash = "3ebb223373142b751b59f5eddc59f46694952d410dfaebe8e9287a67124e2651" [metadata.files] "backports.entry-points-selectable" = [ @@ -524,8 +525,8 @@ coverage = [ {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, ] cyclonedx-python-lib = [ - {file = "cyclonedx-python-lib-0.8.0.tar.gz", hash = "sha256:b3b68ea87a452b358f59184f591abfadd34b2326c380cb171925e6d0778da01b"}, - {file = "cyclonedx_python_lib-0.8.0-py3-none-any.whl", hash = "sha256:35eec307a8bb254a2eeaf3fb47b93e85b9016f9ac1e3bcc0a266135b7e65c531"}, + {file = "cyclonedx-python-lib-0.10.2.tar.gz", hash = "sha256:6f79742ca1728b9016ea272bbe58a60441848e6b4f9742918b8eb67ca987df3b"}, + {file = "cyclonedx_python_lib-0.10.2-py3-none-any.whl", hash = "sha256:e0b2cd3e91c3d55746ce7175a05c23ea2f618158a407b7c9d166eef083a4ee04"}, ] dataclasses = [ {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, @@ -536,24 +537,24 @@ distlib = [ {file = "distlib-0.3.3.zip", hash = "sha256:d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05"}, ] filelock = [ - {file = "filelock-3.3.0-py3-none-any.whl", hash = "sha256:bbc6a0382fe8ec4744ecdf6683a2e07f65eb10ff1aff53fc02a202565446cde0"}, - {file = "filelock-3.3.0.tar.gz", hash = "sha256:8c7eab13dc442dc249e95158bcc12dec724465919bdc9831fdbf0660f03d1785"}, + {file = "filelock-3.3.1-py3-none-any.whl", hash = "sha256:2b5eb3589e7fdda14599e7eb1a50e09b4cc14f34ed98b8ba56d33bfaafcbef2f"}, + {file = "filelock-3.3.1.tar.gz", hash = "sha256:34a9f35f95c441e7b38209775d6e0337f9a3759f3565f6c5798f19618527c76f"}, ] flake8 = [ {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, ] idna = [ - {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, - {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] importlib-metadata = [ {file = "importlib_metadata-4.8.1-py3-none-any.whl", hash = "sha256:b618b6d2d5ffa2f16add5697cf57a46c76a56229b0ed1c438322e4e95645bd15"}, {file = "importlib_metadata-4.8.1.tar.gz", hash = "sha256:f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1"}, ] importlib-resources = [ - {file = "importlib_resources-5.2.2-py3-none-any.whl", hash = "sha256:2480d8e07d1890056cb53c96e3de44fead9c62f2ba949b0f2e4c4345f4afa977"}, - {file = "importlib_resources-5.2.2.tar.gz", hash = "sha256:a65882a4d0fe5fbf702273456ba2ce74fe44892c25e42e057aca526b702a6d4b"}, + {file = "importlib_resources-5.3.0-py3-none-any.whl", hash = "sha256:7a65eb0d8ee98eedab76e6deb51195c67f8e575959f6356a6e15fd7e1148f2a3"}, + {file = "importlib_resources-5.3.0.tar.gz", hash = "sha256:f2e58e721b505a79abe67f5868d99f8886aec8594c962c7490d0c22925f518da"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, diff --git a/pyproject.toml b/pyproject.toml index 89958f5..cf699a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ tinydb = "^4.5.1" PyYAML = "^5.4.1" requests = "^2.26.0" terminaltables = "^3.1.0" -cyclonedx-python-lib = "^0.8.0" +cyclonedx-python-lib = "^0.10.2" polling2 = "^0.5.0" ossindex-lib = "^0.2.1" From 7d9274fc16d53b0ae9117be77520d66fcd6361b7 Mon Sep 17 00:00:00 2001 From: Paul Horton Date: Thu, 21 Oct 2021 16:44:12 +0100 Subject: [PATCH 2/3] doc: update README to cover updated paramters and conda support Signed-off-by: Paul Horton --- README.md | 79 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 400edfe..7086798 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,9 @@ # Jake -![GitHub Workflow Status](https://img.shields.io/github/workflow/status/sonatype-nexus-community/jake/Python%20CI) +[![CircleCI](https://circleci.com/gh/sonatype-nexus-community/jake/tree/main.svg?style=svg)](https://circleci.com/gh/sonatype-nexus-community/jake/tree/main) ![Python Version Support](https://img.shields.io/badge/python-3.6+-blue) -![PyPI Version](https://img.shields.io/pypi/v/jake?label=PyPI&logo=pypi) +[![PyPI Version](https://img.shields.io/pypi/v/jake?label=PyPI&logo=pypi)](https://pypi.org/project/jake) [![GitHub license](https://img.shields.io/github/license/sonatype-nexus-community/jake)](https://github.com/sonatype-nexus-community/jake/blob/main/LICENSE) [![GitHub issues](https://img.shields.io/github/issues/sonatype-nexus-community/jake)](https://github.com/sonatype-nexus-community/jake/issues) [![GitHub forks](https://img.shields.io/github/forks/sonatype-nexus-community/jake)](https://github.com/sonatype-nexus-community/jake/network) @@ -63,27 +63,78 @@ _Other Python package managers are available._ ``` > jake --help -usage: jake [-h] [-v] [-X] ... +usage: app.py [-h] [-v] [-w] [-X] ... Put your Python dependencies in a chokehold optional arguments: - -h, --help show this help message and exit - -v, --version show which version of jake you are running - -w, --warn-only prevents exit with non-zero code when issues have been - detected - -X enable debug output + -h, --help show this help message and exit + -v, --version show which version of jake you are running + -w, --warn-only prevents exit with non-zero code when issues have been + detected + -X enable debug output Jake sub-commands: - - iq perform a scan backed by Nexus Lifecycle - ddt perform a scan backed by OSS Index - sbom generate a CycloneDX software-bill-of-materials (no vulnerabilities) + + iq perform a scan backed by Nexus Lifecycle + ddt perform a scan backed by OSS Index + sbom generate a CycloneDX software-bill-of-materials (no + vulnerabilities) ``` `jake` will exit with code `0` under normal operation and `1` if vulnerabilities are found (OssIndex) or Policy Violations are detected (Nexus IQ), unless you pass the `-w` flag in which case `jake` will always exit with code `0`.... +### Generating an SBOM + +`jake` can take data from various inputs (or just look at your current Python environment) and produce a CycloneDX for +you. + +``` +> jake sbom --help + +usage: jake sbom [-h] [-i FILE_PATH] [-t TYPE] [-o PATH/TO/FILE] + [--output-format {json,xml}] + [--schema-version {1.0,1.1,1.2,1.3}] + +optional arguments: + -h, --help show this help message and exit + -i FILE_PATH, --input FILE_PATH + Where to get input data from. If a path to a file is + not specified directly here,then we will attempt to + read data from STDIN. If there is no data on STDIN, we + will then fall back to looking for standard files in + the current directory that relate to the type of input + indicated by the -t flag. + -t TYPE, --type TYPE, -it TYPE, --input-type TYPE + how jake should find the packages from which to + generate your SBOM.ENV = Read from the current Python + Environment; CONDA = Read output from `conda list + --explicit`; CONDA_JSON = Read output from `conda list + --json`; PIP = read from a requirements.txt; PIPENV = + read from Pipfile.lock; POETRY = read from a + poetry.lock. (Default = ENV) + -o PATH/TO/FILE, --output-file PATH/TO/FILE + Specify a file to output the SBOM to + --output-format {json,xml} + SBOM output format (default = xml) + --schema-version {1.0,1.1,1.2,1.3} + CycloneDX schema version to use (default = 1.3) +``` + +Check out these examples using STDIN: +``` +conda list --explicit --md5 | jake sbom -t CONDA +conda list --json | jake sbom -t CONDA_JSON +cat /path/to/Pipfile.lock | python -m jake.app sbom -t PIPENV +``` + +Check out these examples specifying a manifest: +``` +jake sbom -t PIP -i /path/to/requirements.txt +jake sbom -t PIPENV -i /path/to/Pipfile.lock +``` + ### Check for vulnerabilities using OSS Index `jake` will look at the packaged installed in your current Python environment and check these against OSS Index for you. @@ -129,7 +180,7 @@ So you can quickly get a report by running: -Jake Version: 1.0.0 +Jake Version: 1.1.0 Put your Python dependencies in a chokehold. 🐍 Collected 42 packages from your environment (0:00:00.10) @@ -189,7 +240,7 @@ So passing parameters that suit your Nexus Lifecycle environment you can get a r -Jake Version: 1.0.0 +Jake Version: 1.0.1 Put your Python dependencies in a chokehold 🐍 IQ Server at https://my-nexus-lifecyle is up and accessible (0:00:00.14) From 286ea276f61b719343a2eaef7fce00ce8d9c7ab5 Mon Sep 17 00:00:00 2001 From: Paul Horton Date: Fri, 22 Oct 2021 08:52:33 +0100 Subject: [PATCH 3/3] doc: typo resolved Signed-off-by: Paul Horton --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7086798..98ba8fd 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ _Other Python package managers are available._ ``` > jake --help -usage: app.py [-h] [-v] [-w] [-X] ... +usage: jake [-h] [-v] [-w] [-X] ... Put your Python dependencies in a chokehold