feat: Support for all input formats when running jake ddt or jake iq (#…

…125) * feat: Support for all input formats when running jake ddt or jake iq closes #104 BREAKING CHANGE: changed iq -t switch to -st, use common `-f` argument for input file Co-authored-by: Dan Rollo <danrollo@gmail.com>
sonatype-nexus-community · Dec 2, 2022 · 9a597b5 · 9a597b5
1 parent 9f0bd52
commit 9a597b5
Show file tree

Hide file tree

Showing 7 changed files with 227 additions and 104 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -22,7 +22,7 @@ commands:
       - run: |
           python -m ensurepip --default-pip
           pip install --upgrade pip
-          pip install poetry==1.1.11
+          pip install poetry==1.1.15
 
 executors:
   python310:
@@ -75,7 +75,7 @@ jobs:
 #          name: "Run Jake on Jake"
 #          command: |
 #            poetry install --no-dev
-#            poetry run jake ddt
+#            poetry run jake ddt --whitelist jake-whitelist.json
 
   coding_standards:
     executor: python310

diff --git a/README.md b/README.md
@@ -93,13 +93,13 @@ you.
 ```
 > jake sbom --help
 
-usage: jake sbom [-h] [-i FILE_PATH] [-t TYPE] [-o PATH/TO/FILE]
+usage: jake sbom [-h] [-f FILE_PATH] [-t TYPE] [-o PATH/TO/FILE]
                    [--output-format {json,xml}]
                    [--schema-version {1.0,1.1,1.2,1.3}]
 
 optional arguments:
   -h, --help            show this help message and exit
-  -i FILE_PATH, --input FILE_PATH
+  -f FILE_PATH, --input FILE_PATH
                         Where to get input data from. If a path to a file is
                         not specified directly here,then we will attempt to
                         read data from STDIN. If there is no data on STDIN, we
@@ -143,17 +143,37 @@ Optionally, it can create a CycloneDX software bill-of-materials at the same tim
 ```
 > jake ddt --help
 
-usage: jake ddt [-h] [--clear-cache] [-o PATH/TO/FILE] [--output-format {xml,json}] [--schema-version {1.2,1.1,1.0,1.3}]
+usage: jake ddt [-h] [-f FILE_PATH] [-t TYPE] [--clear-cache] [-o PATH/TO/FILE] 
+                   [--output-format {xml,json}]
+                   [--schema-version {1.2,1.1,1.0,1.3}]
+                   [--whitelist OSS_WHITELIST_JSON_FILE]
 
 optional arguments:
   -h, --help            show this help message and exit
+  -f FILE_PATH, --input-file FILE_PATH
+                        Where to get input data from. If a path to a file is
+                        not specified directly here,then we will attempt to
+                        read data from STDIN. If there is no data on STDIN, we
+                        will then fall back to looking for standard files in
+                        the current directory that relate to the type of input
+                        indicated by the -t flag.
+  -t TYPE, --type TYPE, -it TYPE, --input-type TYPE
+                        how jake should find the packages from which to
+                        generate your SBOM.ENV = Read from the current Python
+                        Environment; CONDA = Read output from `conda list
+                        --explicit`; CONDA_JSON = Read output from `conda list
+                        --json`; PIP = read from a requirements.txt; PIPENV =
+                        read from Pipfile.lock; POETRY = read from a
+                        poetry.lock. (Default = ENV)
   --clear-cache         Clears any local cached OSS Index data prior to execution
   -o PATH/TO/FILE, --output-file PATH/TO/FILE
                         Specify a file to output the SBOM to. If not specified the report will be output to the console. STDOUT is not supported.
   --output-format {xml,json}
                         SBOM output format (default = xml)
   --schema-version {1.2,1.1,1.0,1.3}
                         CycloneDX schema version to use (default = 1.3)
+  --whitelist OSS_WHITELIST_JSON_FILE
+                        Set path to whitelist json file
 ```
 
 So you can quickly get a report by running:
@@ -267,6 +287,19 @@ Vulnerability Details for pkg:pypi/cryptography@2.2
 └──────────────────────┴───────────────────────┘
 ```
 
+Check out these examples using STDIN:
+```
+conda list --explicit --md5 | jake ddt -t CONDA
+conda list --json | jake ddt -t CONDA_JSON
+cat /path/to/Pipfile.lock | python -m jake.app ddt -t PIPENV
+```
+
+Check out these examples specifying a manifest:
+```
+jake ddt -t PIP -i /path/to/requirements.txt
+jake ddt -t PIPENV -i /path/to/Pipfile.lock
+```
+
 A pre-commit hook is also available for use
 
 ```Yaml
@@ -303,10 +336,25 @@ Access Sonatype's proprietary vulnerability data using `jake`:
 ```
 > jake iq --help
 
-usage: jake iq [-h] -s https://localhost:8070 -i APP_ID -u USER_ID -p PASSWORD [-t STAGE]
+usage: jake iq [-h] [-f FILE_PATH] [-t TYPE] -s https://localhost:8070 -i APP_ID -u USER_ID -p PASSWORD [-st STAGE]
 
 optional arguments:
   -h, --help            show this help message and exit
+  -f FILE_PATH, --input-file FILE_PATH
+                        Where to get input data from. If a path to a file is
+                        not specified directly here,then we will attempt to
+                        read data from STDIN. If there is no data on STDIN, we
+                        will then fall back to looking for standard files in
+                        the current directory that relate to the type of input
+                        indicated by the -t flag.
+  -t TYPE, --type TYPE, -it TYPE, --input-type TYPE
+                        how jake should find the packages from which to
+                        generate your SBOM.ENV = Read from the current Python
+                        Environment; CONDA = Read output from `conda list
+                        --explicit`; CONDA_JSON = Read output from `conda list
+                        --json`; PIP = read from a requirements.txt; PIPENV =
+                        read from Pipfile.lock; POETRY = read from a
+                        poetry.lock. (Default = ENV)
   -s https://localhost:8070, --server-url https://localhost:8070
                         Full http(s):// URL to your Nexus Lifecycle server
   -i APP_ID, --application-id APP_ID
@@ -315,7 +363,7 @@ optional arguments:
                         Username for authentication to Nexus Lifecycle
   -p PASSWORD, --password PASSWORD
                         Password for authentication to Nexus Lifecycle
-  -t STAGE, --stage STAGE
+  -st STAGE, --stage STAGE
                         The stage for the report
 ```
 
@@ -397,4 +445,3 @@ community (read: you!)
 * DO file issues here on GitHub, so that the community can pitch in
 
 Phew, that was easier than I thought. Last but not least of all - have fun!
-
diff --git a/docs/usage.rst b/docs/usage.rst
@@ -58,13 +58,13 @@ you.
 
     > jake sbom --help
 
-    usage: jake sbom [-h] [-i FILE_PATH] [-t TYPE] [-o PATH/TO/FILE]
+    usage: jake sbom [-h] [-f FILE_PATH] [-t TYPE] [-o PATH/TO/FILE]
                        [--output-format {json,xml}]
                        [--schema-version {1.0,1.1,1.2,1.3}]
 
     optional arguments:
       -h, --help            show this help message and exit
-      -i FILE_PATH, --input FILE_PATH
+      -f FILE_PATH, --input-file FILE_PATH
                             Where to get input data from. If a path to a file is
                             not specified directly here,then we will attempt to
                             read data from STDIN. If there is no data on STDIN, we
@@ -99,8 +99,8 @@ Check out these examples specifying a manifest:
 
 .. code-block::
 
-    jake sbom -t PIP -i /path/to/requirements.txt
-    jake sbom -t PIPENV -i /path/to/Pipfile.lock
+    jake sbom -t PIP -f /path/to/requirements.txt
+    jake sbom -t PIPENV -f /path/to/Pipfile.lock
 
 
 Check for vulnerabilities using OSS Index
@@ -113,17 +113,25 @@ you.
 
     > jake ddt --help
 
-    usage: jake ddt [-h] [--clear-cache] [-o PATH/TO/FILE] [--output-format {xml,json}] [--schema-version {1.2,1.1,1.0,1.3}]
+    usage: jake ddt [-h] [-f FILE_PATH] [-t TYPE] [--clear-cache] [-o PATH/TO/FILE] [--output-format {json,xml}] [--schema-version {1.2,1.3,1.4,1.1,1.0}] [--whitelist OSS_WHITELIST_JSON_FILE]
 
-    optional arguments:
+    options:
       -h, --help            show this help message and exit
+      -f FILE_PATH, --input-file FILE_PATH
+                            Where to get input data from. If a path to a file is not specified directly here,then we will attempt to read data from STDIN. If there is no data on STDIN, we will then fall back to looking for standard
+                            files in the current directory that relate to the type of input indicated by the -t flag.
+      -t TYPE, -it TYPE, --type TYPE, --input-type TYPE
+                            how jake should find the packages from which to generate your SBOM.ENV = Read from the current Python Environment; CONDA = Read output from `conda list --explicit`; CONDA_JSON = Read output from `conda list
+                            --json`; PIP = read from a requirements.txt; PIPENV = read from Pipfile.lock; POETRY = read from a poetry.lock. (Default = ENV)
       --clear-cache         Clears any local cached OSS Index data prior to execution
       -o PATH/TO/FILE, --output-file PATH/TO/FILE
                             Specify a file to output the SBOM to. If not specified the report will be output to the console. STDOUT is not supported.
-      --output-format {xml,json}
+      --output-format {json,xml}
                             SBOM output format (default = xml)
-      --schema-version {1.2,1.1,1.0,1.3}
-                            CycloneDX schema version to use (default = 1.3)
+      --schema-version {1.2,1.3,1.4,1.1,1.0}
+                            CycloneDX schema version to use (default = 1.4)
+      --whitelist OSS_WHITELIST_JSON_FILE
+                            Set path to whitelist json file
 
 So you can quickly get a report by running:
 
@@ -150,7 +158,7 @@ So you can quickly get a report by running:
 
 
 
-    Jake Version: 1.1.0
+    Jake Version: 2.1.1
     Put your Python dependencies in a chokehold.
 
     🐍 Collected 42 packages from your environment (0:00:00.10)
@@ -187,12 +195,13 @@ This is what ``jake`` will output if any bad things are found:
 
 
 
-    Jake Version: 1.1.5
+    Jake Version: 2.1.1
     Put your Python dependencies in a chokehold
 
-    🐍 Collected 69 packages from your environment                       ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% -:--:--
+    🐍 Collected 69 packages from your python environment                ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% -:--:--
     🐍 Successfully queried OSS Index for package and vulnerability info ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% -:--:--
     🐍 Sane number of results from OSS Index                             ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% -:--:--
+    🐍 Munching & crunching data...                                      ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% -:--:--
 
     [59/69] - pkg:pypi/cryptography@2.2 [VULNERABLE]
     Vulnerability Details for pkg:pypi/cryptography@2.2
@@ -238,6 +247,23 @@ This is what ``jake`` will output if any bad things are found:
     └──────────────────────┴───────────────────────┘
 
 
+Check out these examples using STDIN:
+
+.. code-block::
+
+    conda list --explicit --md5 | jake ddt -t CONDA
+    conda list --json | jake ddt -t CONDA_JSON
+    cat /path/to/Pipfile.lock | jake ddt -t PIPENV
+
+
+Check out these examples specifying a manifest:
+
+.. code-block::
+
+    jake ddt -t PIP -f /path/to/requirements.txt
+    jake ddt -t PIPENV -f /path/to/Pipfile.lock
+
+
 Pre-commit Hook
 ~~~~~~~~~~~~~~~
 
@@ -259,10 +285,16 @@ Access Sonatype's proprietary vulnerability data using ``jake``:
 
     > jake iq --help
 
-    usage: jake iq [-h] -s https://localhost:8070 -i APP_ID -u USER_ID -p PASSWORD [-t STAGE]
+    usage: jake iq [-h] [-f FILE_PATH] [-t TYPE] -s https://localhost:8070 -i APP_ID -u USER_ID -p PASSWORD [-st STAGE]
 
-    optional arguments:
+    options:
       -h, --help            show this help message and exit
+      -f FILE_PATH, --input-file FILE_PATH
+                            Where to get input data from. If a path to a file is not specified directly here,then we will attempt to read data from STDIN. If there is no data on STDIN, we will then fall back to looking for standard
+                            files in the current directory that relate to the type of input indicated by the -t flag.
+      -t TYPE, -it TYPE, --type TYPE, --input-type TYPE
+                            how jake should find the packages from which to generate your SBOM.ENV = Read from the current Python Environment; CONDA = Read output from `conda list --explicit`; CONDA_JSON = Read output from `conda list
+                            --json`; PIP = read from a requirements.txt; PIPENV = read from Pipfile.lock; POETRY = read from a poetry.lock. (Default = ENV)
       -s https://localhost:8070, --server-url https://localhost:8070
                             Full http(s):// URL to your Nexus Lifecycle server
       -i APP_ID, --application-id APP_ID
@@ -271,7 +303,7 @@ Access Sonatype's proprietary vulnerability data using ``jake``:
                             Username for authentication to Nexus Lifecycle
       -p PASSWORD, --password PASSWORD
                             Password for authentication to Nexus Lifecycle
-      -t STAGE, --stage STAGE
+      -st STAGE, --stage STAGE
                             The stage for the report
 
 So passing parameters that suit your Nexus Lifecycle environment you can get a report:
@@ -299,7 +331,7 @@ So passing parameters that suit your Nexus Lifecycle environment you can get a r
 
 
 
-    Jake Version: 1.0.1
+    Jake Version: 2.1.1
     Put your Python dependencies in a chokehold
 
     🐍 IQ Server at https://my-nexus-lifecyle is up and accessible (0:00:00.14)

diff --git a/jake/command/iq.py b/jake/command/iq.py
@@ -39,12 +39,12 @@
 import requests
 from cyclonedx.model.bom import Bom
 from cyclonedx.output import get_instance
-from cyclonedx_py.parser.environment import EnvironmentParser
 from polling2 import poll_decorator  # type: ignore
 from requests.auth import HTTPBasicAuth
 from rich.progress import Progress
 
 from . import BaseCommand, jake_version
+from . import parser_selector
 
 
 class IqCommand(BaseCommand):
@@ -185,13 +185,14 @@ def __init__(self) -> None:
 
     def handle_args(self) -> int:
         exit_code: int = 0
+        input_source_msg = "your python environment" if self.arguments.sbom_input_type == "ENV" else "provided specs"
 
         with Progress() as progress:
             task_validate_iq = progress.add_task(
                 description="[yellow]Checking out your Nexus IQ Server", start=True, total=10
             )
             task_parser = progress.add_task(
-                description="[yellow]Collecting packages in your Python Environment", start=True, total=10
+                description=f"[yellow]Collecting packages in {input_source_msg}", start=True, total=10
             )
             task_query_iq = progress.add_task(
                 description="[yellow]Submitting to Nexus Lifecycle for Policy Evaluation", start=True, total=10
@@ -209,11 +210,13 @@ def handle_args(self) -> int:
             )
 
             # task_parser
-            parser = EnvironmentParser()
+            parser = parser_selector.get_parser(
+                self.arguments.sbom_input_type, self.arguments.sbom_input_source
+            )
             total_packages_collected = len(parser.get_components())
             progress.update(
                 task_parser, completed=10,
-                description=f'🐍 [green]Collected {total_packages_collected} packages from your environment'
+                description=f'🐍 [green]Collected {total_packages_collected} packages from {input_source_msg}'
             )
 
             # task_query_iq
@@ -257,6 +260,7 @@ def get_argument_parser_help(self) -> str:
         return 'perform a scan backed by Sonatype Nexus Lifecycle'
 
     def setup_argument_parser(self, arg_parser: ArgumentParser) -> None:
+        parser_selector.add_parser_selector_arguments(arg_parser)
         arg_parser.add_argument('-s', '--server-url', help='Full http(s):// URL to your Nexus Lifecycle server',
                                 metavar='https://localhost:8070', required=True, dest='iq_server_url')
 
@@ -269,5 +273,5 @@ def setup_argument_parser(self, arg_parser: ArgumentParser) -> None:
         arg_parser.add_argument('-p', '--password', help='Password for authentication to Nexus Lifecycle',
                                 metavar='PASSWORD', required=True, dest='iq_password')
 
-        arg_parser.add_argument('-t', '--stage', help='The stage for the report',
+        arg_parser.add_argument('-st', '--stage', help='The stage for the report',
                                 metavar='STAGE', required=False, dest='iq_scan_stage', default='source')
diff --git a/jake/command/oss.py b/jake/command/oss.py
@@ -45,7 +45,6 @@
 from cyclonedx.model.vulnerability import BomTarget, BomTargetVersionRange, Vulnerability, VulnerabilityAdvisory, \
     VulnerabilityRating, VulnerabilityReference, VulnerabilityScoreSource, VulnerabilitySeverity, VulnerabilitySource
 from cyclonedx.output import get_instance, OutputFormat, SchemaVersion, LATEST_SUPPORTED_SCHEMA_VERSION
-from cyclonedx_py.parser.environment import EnvironmentParser
 from ossindex.model import OssIndexComponent
 from ossindex.ossindex import OssIndex
 # See https://github.com/package-url/packageurl-python/issues/65
@@ -57,6 +56,7 @@
 from rich.tree import Tree
 
 from . import BaseCommand
+from . import parser_selector
 
 
 class OssCommand(BaseCommand):
@@ -66,10 +66,11 @@ def handle_args(self) -> int:
         self._console = Console()
 
         exit_code: int = 0
+        input_source_msg = "your python environment" if self.arguments.sbom_input_type == "ENV" else "provided specs"
 
         with Progress() as progress:
             task_parser = progress.add_task(
-                description="[yellow]Collecting packages in your Python Environment", start=True, total=10
+                description=f"[yellow]Collecting packages in {input_source_msg}", start=True, total=10
             )
             task_query_ossi = progress.add_task(
                 description="[yellow]Querying OSS Index for details on your packages", start=True, total=10
@@ -78,11 +79,13 @@ def handle_args(self) -> int:
                 description="[cyan]Sanity checking...", start=True, total=10
             )
 
-            parser = EnvironmentParser()
+            parser = parser_selector.get_parser(
+                self.arguments.sbom_input_type, self.arguments.sbom_input_source
+            )
             total_packages_collected = len(parser.get_components())
             progress.update(
                 task_parser, completed=10,
-                description=f'🐍 [green]Collected {total_packages_collected} packages from your environment'
+                description=f'🐍 [green]Collected {total_packages_collected} packages from {input_source_msg}'
             )
 
             oss_index_results: List[OssIndexComponent]
@@ -233,6 +236,7 @@ def get_argument_parser_help(self) -> str:
         return 'perform a scan backed by OSS Index'
 
     def setup_argument_parser(self, arg_parser: ArgumentParser) -> None:
+        parser_selector.add_parser_selector_arguments(arg_parser)
         arg_parser.add_argument('--clear-cache', help='Clears any local cached OSS Index data prior to execution',
                                 action='store_true', dest='oss_clear_cache', default=False)