From 7e050e4179e3249729479b8aab3af682ccaa37a3 Mon Sep 17 00:00:00 2001 From: Al Crowley Date: Tue, 11 Jan 2022 17:38:29 -0500 Subject: [PATCH 1/2] Added support for an ignore file --- safety/cli.py | 20 +++++++++++--------- safety/safety.py | 44 +++++++++++++++++++++++++++++++++++++++++++- tests/test_safety.py | 39 +++++++++++++++++++++++++++++++++++---- 3 files changed, 89 insertions(+), 14 deletions(-) diff --git a/safety/cli.py b/safety/cli.py index 09737831..ae08a90b 100644 --- a/safety/cli.py +++ b/safety/cli.py @@ -43,6 +43,8 @@ def cli(): help="Read input from one (or multiple) requirement files. Default: empty") @click.option("ignore", "--ignore", "-i", multiple=True, type=str, default=[], help="Ignore one (or multiple) vulnerabilities by ID. Default: empty") +@click.option("ignorefile", "--ignore-file", "-f", multiple=False, type=click.File(), default=None, + help="File with a list of vulnerability IDs to ignore. Default: None") @click.option("--output", "-o", default="", help="Path to where output file will be placed. Default: empty") @click.option("proxyhost", "--proxy-host", "-ph", multiple=False, type=str, default=None, @@ -51,7 +53,7 @@ def cli(): help="Proxy port number --proxy-port") @click.option("proxyprotocol", "--proxy-protocol", "-pr", multiple=False, type=str, default='http', help="Proxy protocol (https or http) --proxy-protocol") -def check(key, db, json, full_report, bare, stdin, files, cache, ignore, output, proxyprotocol, proxyhost, proxyport): +def check(key, db, json, full_report, bare, stdin, files, cache, ignore, ignorefile, output, proxyprotocol, proxyhost, proxyport): if files and stdin: click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red", file=sys.stderr) sys.exit(-1) @@ -65,16 +67,16 @@ def check(key, db, json, full_report, bare, stdin, files, cache, ignore, output, packages = [ d for d in pkg_resources.working_set if d.key not in {"python", "wsgiref", "argparse"} - ] + ] proxy_dictionary = get_proxy_dict(proxyprotocol, proxyhost, proxyport) try: vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore, proxy=proxy_dictionary) - output_report = report(vulns=vulns, - full=full_report, - json_report=json, + output_report = report(vulns=vulns, + full=full_report, + json_report=json, bare_report=bare, checked_packages=len(packages), - db=db, + db=db, key=key) if output: @@ -154,15 +156,15 @@ def license(key, db, json, bare, cache, files, proxyprotocol, proxyhost, proxypo packages = [ d for d in pkg_resources.working_set if d.key not in {"python", "wsgiref", "argparse"} - ] - + ] + proxy_dictionary = get_proxy_dict(proxyprotocol, proxyhost, proxyport) try: licenses_db = safety.get_licenses(key, db, cache, proxy_dictionary) except InvalidKeyError as invalid_key_error: if str(invalid_key_error): message = str(invalid_key_error) - else: + else: message = "Your API Key '{key}' is invalid. See {link}".format( key=key, link='https://goo.gl/O7Y1rS' ) diff --git a/safety/safety.py b/safety/safety.py index 9f3a5bbd..fd6b7053 100644 --- a/safety/safety.py +++ b/safety/safety.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- +from datetime import datetime import errno import json import os +import re import time from collections import namedtuple @@ -133,7 +135,47 @@ def get_vulnerabilities(pkg, spec, db): yield entry -def check(packages, key, db_mirror, cached, ignore_ids, proxy): +def parse_ignore_file_line(line): + """ + Parses the ignore file + + Function returns a tople with the (ID, optional expiration date, comment) for the line. + Comments lines or any line not matching the proper pattern will return None + good example line: + 1234 2022-01-01 # comment + """ + + # The Regex breaks down to: start of line, + # number, + # optional date in YYYY-MM-DD format, + # optional comment starting with # character + + matches = re.match('^\s*([0-9]+)\s*(\d{4}-\d{2}-\d{2})?\s*(\#.*)?$', line) + return matches.groups() if matches else None + +def get_ignore_ids_from_file(ignore_file): + """ + Reads the file provided by ignore_file and returns a tuple of all vulnerability IDs parsed from there + """ + + ignored_ids = [] + today = datetime.today().strftime('%Y-%m-%d') + + for line in ignore_file: + parsed_touple = parse_ignore_file_line(line) + if not parsed_touple: + continue + id, date, comment = parsed_touple + if (not date) or today < date: # append the id as long as the expiration date is in the future (or None) + ignored_ids.append(id) + + return tuple(ignored_ids) + + +def check(packages, key, db_mirror, cached, ignore_ids, proxy, ignore_file=None): + if ignore_file: + ignore_ids = ignore_ids + get_ignore_ids_from_file(ignore_file) + key = key if key else os.environ.get("SAFETY_API_KEY", False) db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy) db_full = None diff --git a/tests/test_safety.py b/tests/test_safety.py index b0f5da27..ef4a191b 100644 --- a/tests/test_safety.py +++ b/tests/test_safety.py @@ -13,6 +13,7 @@ import textwrap from click.testing import CliRunner from unittest.mock import Mock, patch +from datetime import datetime, timedelta from safety import safety from safety import cli @@ -298,7 +299,7 @@ def test_get_packages_licenses(self): def test_get_packages_licenses_without_api_key(self): from safety.errors import InvalidKeyError - # without providing an API-KEY + # without providing an API-KEY with self.assertRaises(InvalidKeyError) as error: safety.get_licenses( db_mirror=False, @@ -341,7 +342,7 @@ def test_get_packages_licenses_db_fetch_error(self, requests): proxy={}, key="MY-VALID-KEY" ) - + def test_get_packages_licenses_with_invalid_db_file(self): from safety.errors import DatabaseFileNotFoundError @@ -401,7 +402,7 @@ def test_get_cached_packages_licenses(self, requests): f.write(json.dumps({})) except Exception: pass - + # In order to cache the db (and get), we must set cached as True response = safety.get_licenses( db_mirror=False, @@ -421,7 +422,7 @@ def test_get_cached_packages_licenses(self, requests): proxy={}, key="MY-VALID-KEY" ) - + self.assertNotEqual(resp, licenses_db) self.assertEqual(resp, original_db) @@ -514,3 +515,33 @@ def test_recursive_requirement(self): with open(test_filename) as fh: result = list(read_requirements(fh, resolve=True)) self.assertEqual(len(result), 2) + +class TestIgnoreFile(unittest.TestCase): + def test_ignore_file_pattern(self): + test_cases = [ + ["123", ("123", None,None)], + [(" 123 #comment"), ("123", None, "#comment")], + ["10001 2022-01-01", ("10001", "2022-01-01", None)], + ["6 2022-11-29", ("6", "2022-11-29", None)], + ["10001 2022-01-01 # spaced comment", ("10001", "2022-01-01", "# spaced comment")], + ["55 22-01-01", None] # invalid date format, ignore the line + ] + self.assertFalse(safety.parse_ignore_file_line("# comment"), "Comments should be skipped") + self.assertFalse(safety.parse_ignore_file_line("fail string 123"), "lines not starting with a number or space should be skipped") + + for input,output in test_cases: + print (output) + self.assertEqual(safety.parse_ignore_file_line(input), output) + + def test_ignore_file_read(self): + tomorrow = (datetime.today() + timedelta(days=+1)).strftime('%Y-%m-%d') + yesterday = (datetime.today() + timedelta(days=-1)).strftime('%Y-%m-%d') + + self.assertEqual(safety.get_ignore_ids_from_file(["123","456"]), ("123","456")) + self.assertEqual(safety.get_ignore_ids_from_file(["123","#comment line", "456"]), ("123","456")) + self.assertEqual(safety.get_ignore_ids_from_file(["123","#comment line", "456"]), ("123","456")) + + self.assertEqual(safety.get_ignore_ids_from_file([f"123 {yesterday}", "456"]), ("456",)) + self.assertEqual(safety.get_ignore_ids_from_file([f"123 {tomorrow}", "456"]), ("123", "456")) + + self.assertEqual(safety.get_ignore_ids_from_file([f"# comment lines", "#only"]), ()) From 2a647ea2783b9e45b269dbc03463f048b8b4199f Mon Sep 17 00:00:00 2001 From: Al Crowley Date: Tue, 11 Jan 2022 17:49:04 -0500 Subject: [PATCH 2/2] Adding documentation for the ignore-file option --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a1043325..f6271743 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,10 @@ [![Travis](https://img.shields.io/travis/pyupio/safety.svg)](https://travis-ci.org/pyupio/safety) [![Updates](https://pyup.io/repos/github/pyupio/safety/shield.svg)](https://pyup.io/repos/github/pyupio/safety/) -Safety checks your installed dependencies for known security vulnerabilities. +Safety checks your installed dependencies for known security vulnerabilities. -By default it uses the open Python vulnerability database [Safety DB](https://github.com/pyupio/safety-db), -but can be upgraded to use pyup.io's [Safety API](https://github.com/pyupio/safety/blob/master/docs/api_key.md) using the `--key` option. +By default it uses the open Python vulnerability database [Safety DB](https://github.com/pyupio/safety-db), +but can be upgraded to use pyup.io's [Safety API](https://github.com/pyupio/safety/blob/master/docs/api_key.md) using the `--key` option. # Installation @@ -140,7 +140,7 @@ of Safety. ## Using Safety with a CI service -Safety works great in your CI pipeline. It returns a non-zero exit status if it finds a vulnerability. +Safety works great in your CI pipeline. It returns a non-zero exit status if it finds a vulnerability. Run it before or after your tests. If Safety finds something, your tests will fail. @@ -177,9 +177,9 @@ commands = **Deep GitHub Integration** -If you are looking for a deep integration with your GitHub repositories: Safety is available as a -part of [pyup.io](https://pyup.io/), called [Safety CI](https://pyup.io/safety/ci/). Safety CI -checks your commits and pull requests for dependencies with known security vulnerabilities +If you are looking for a deep integration with your GitHub repositories: Safety is available as a +part of [pyup.io](https://pyup.io/), called [Safety CI](https://pyup.io/safety/ci/). Safety CI +checks your commits and pull requests for dependencies with known security vulnerabilities and displays a status on GitHub. ![Safety CI](https://github.com/pyupio/safety/raw/master/safety_ci.png) @@ -359,6 +359,45 @@ safety check --ignore=1234 safety check -i 1234 -i 4567 -i 89101 ``` +--- + +### `--ignore-file`, `-f` + +*Read list from file of vulnerability IDs that will be ignored + +**Example** +```bash +safety check -f dependencies.ignore +``` +```bash +safety check --ignore-file dependencies.ignore +``` + +The ignore file should be in the format: +``` +ID YYYY-MM-DD # comment +``` +* The ID is the vulerability ID number +* YYYY-MM-DD is an optional expiration date after which the vulerability will no longer be ignored +* You can end each line with an optional comment + +You may also have lines that consisten only of comments. Here is an example ignore file: +``` +# Any vulnerability ID numbers listed in this file will be ignored when +# running the safety dependency check. Each line should have the ID number +# and a date. The ID will be ignored by the CI pipeline check unitl the date +# in YYYY-MM-DD format listed for that line. +# If no date is listed, the exception will never expire. (NOT RECOMMENDED) +# +# test +# Example: +# 40104 2022-01-15 +# +40105 2022-01-15 # gunicorn vulnerability +40104 +1234 1999-10-15 # this entry is expired and will not be ignored +``` + ### `--output`, `-o` *Save the report to a file* @@ -537,7 +576,7 @@ ___ *Proxy host IP or DNS* -### `--proxy-port`, `-pp` +### `--proxy-port`, `-pp` *Proxy port number*