diff --git a/.gitignore b/.gitignore index 5f82252..e5147b2 100644 --- a/.gitignore +++ b/.gitignore @@ -106,6 +106,7 @@ venv/ ENV/ env.bak/ venv.bak/ +jake.json # Spyder project settings .spyderproject diff --git a/README.md b/README.md index 3928173..21f5bde 100644 --- a/README.md +++ b/README.md @@ -11,27 +11,61 @@ ### Usage ``` - ~ > jake +$ jake --help +usage: jake [-h] [-V] [-VV] [-C] {ddt} + +positional arguments: + {ddt} run jake + +optional arguments: + -h, --help show this help message and exit + -V, --version show program version + -VV, --verbose set verbosity level to debug + -C, --clean wipe out jake cache ``` +Typical usage of `jake` is to run it like so: `conda list | jake ddt`, which will feed your Conda dependencies in your current Conda environment to `jake`, which will then reach out and check OSS Index to see if they are vulnerable! + ### Options +You may also run `jake` with `-VV` for a slew of debug data, in case you are running in to an odd situation, or you want to help out on development! + +You can also run `jake -C` to clean out your local cache if desired. We cache results from OSS Index for 12 hours to prevent you from potentially getting rate limited (as your dependencies likely won't change super often). + ## Why Jake? Jake The Snake was scared of Snakes. The finishing move was DDT. He finishes the Snake with DDT. ## Installation +### Download from PyPI + +TBD + ### Build from source -### Download as Conda package +* Clone the repo +* Install Python 3.7 or higher +* Ensure pip is installed (it should be) +* Run `python3 venv .venv` (or whatever virtual environment you prefer) +* Run `source .venv/bin/activate` +* Run `pip install -r requirements` +* Run `pip install -e .` + +Once you've done this, you should have `jake` available to test with fairly globally, pointed at the local source you've cloned. ## Development -`jake` is written using Python. +`jake` is written using Python 3.7 This project also uses `pip` for dependencies, so you will need to download make sure you have `pip`. +Follow instructions in Build from source. + +Tests can be run with `python3 -m unittest discover` + +More TBD. + ## Contributing We care a lot about making the world a safer place, and that's why we created `jake`. If you as well want to diff --git a/jake/__main__.py b/jake/__main__.py index 6402451..1df92fa 100644 --- a/jake/__main__.py +++ b/jake/__main__.py @@ -23,35 +23,48 @@ from jake.parse.parse import Parse from jake.audit.audit import Audit +from ._version import __version__ + def main(): parser = argparse.ArgumentParser() parser.add_argument('run', help='run jake', choices=['ddt']) - parser.add_argument('-V', '--version', help='show program version', action='store_true') - parser.add_argument('-E', '--env', help="conda environment to run", default='root') + parser.add_argument('-V', '--version', help='show program version and exit', action='store_true') parser.add_argument('-VV', '--verbose', help="set verbosity level to debug", action='store_true') + parser.add_argument('-C', '--clean', help="wipe out jake cache", action='store_true') args = parser.parse_args() log = logging.getLogger('jake') + if args.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.ERROR) + if args.version: + print(__version__) + _exit(0) + parse = Parse() ossindex = OssIndex() audit = Audit() + if args.clean: + ossindex.cleanCache() + if args.run == 'ddt': log.info('Calling OSS Index') purls = parse.getDependenciesFromStdin(sys.stdin) if purls is None: - log.error("No purls returned, likely culprit is no Conda installed") + log.error("No purls returned, ensure that conda list is returning a list of dependencies") _exit(EX_OSERR) - log.debug(purls) + log.debug("Total purls: %s", len(purls.get_coordinates())) response = ossindex.callOSSIndex(purls) - - code = audit.auditResults(response) + if response is not None: + code = audit.auditResults(response) + else: + log.error("Something went horribly wrong, please rerun with -VV to see what happened") + _exit(EX_OSERR) _exit(code) diff --git a/jake/_version.py b/jake/_version.py new file mode 100644 index 0000000..9ecee59 --- /dev/null +++ b/jake/_version.py @@ -0,0 +1 @@ +__version__="0.0.1" diff --git a/jake/audit/audit.py b/jake/audit/audit.py index 6cd9314..b6e1037 100644 --- a/jake/audit/audit.py +++ b/jake/audit/audit.py @@ -13,12 +13,16 @@ # limitations under the License. import logging +from typing import List + +from jake.types.coordinateresults import CoordinateResults + class Audit(object): def __init__(self): self._log = logging.getLogger('jake') - def auditResults(self, results): - self._log.debug(results) + def auditResults(self, results: List[CoordinateResults]): + self._log.debug("Results recieved, %s total results", len(results)) totalVulns = 0 pkgNum = 0 @@ -29,15 +33,15 @@ def auditResults(self, results): return totalVulns - def printResult(self, coordinate, number, length): - if len(coordinate['vulnerabilities']) == 0: - print("[{}/{}] - {} - no known vulnerabilities for this version".format(number, length, coordinate['coordinates'])) - return len(coordinate['vulnerabilities']) + def printResult(self, coordinate: CoordinateResults, number, length): + if len(coordinate.getVulnerabilities()) == 0: + print("[{}/{}] - {} - no known vulnerabilities for this version".format(number, length, coordinate.getCoordinates())) + return len(coordinate.getVulnerabilities()) else: - print("[{}/{}] - {} [VULNERABLE] {} known vulnerabilities for this version".format(number, length, coordinate['coordinates'], len(coordinate['vulnerabilities']))) - for vulnerability in coordinate['vulnerabilities']: + print("[{}/{}] - {} [VULNERABLE] {} known vulnerabilities for this version".format(number, length, coordinate.getCoordinates(), len(coordinate.getVulnerabilities()))) + for vulnerability in coordinate.getVulnerabilities(): self.printVulnerability(vulnerability) - return len(coordinate['vulnerabilities']) + return len(coordinate.getVulnerabilities()) def printVulnerability(self, vulnerability): print(vulnerability) diff --git a/jake/ossindex/ossindex.py b/jake/ossindex/ossindex.py index 779a22b..90057db 100644 --- a/jake/ossindex/ossindex.py +++ b/jake/ossindex/ossindex.py @@ -15,12 +15,29 @@ import logging import json +from typing import List +from datetime import datetime, timedelta +from dateutil.parser import parse +from tinydb import TinyDB, Query +from pathlib import Path +from jake.types.coordinates import Coordinates +from jake.types.results_decoder import ResultsDecoder +from jake.types.coordinateresults import CoordinateResults + class OssIndex(object): - def __init__(self, url='https://ossindex.sonatype.org/api/v3/component-report', headers={'Content-type': 'application/json', 'User-Agent': 'jake'}): + def __init__(self, url='https://ossindex.sonatype.org/api/v3/component-report', headers={'Content-type': 'application/vnd.ossindex.component-report-request.v1+json', 'User-Agent': 'jake'}, cache_location=''): self._url = url self._headers = headers self._log = logging.getLogger('jake') self._maxcoords = 128 + if cache_location == '': + home = str(Path.home()) + dir_oss = home + "/.ossindex/" + else: + dir_oss = cache_location + "/.ossindex/" + if not Path(dir_oss).exists(): + Path(dir_oss).mkdir(parents=True, exist_ok=True) + self._db = TinyDB(dir_oss + "jake.json") def get_url(self): return self._url @@ -47,18 +64,74 @@ def chunk(self, purls): chunks.append(divided) return chunks - def callOSSIndex(self, purls): - self._log.debug(purls) + def callOSSIndex(self, purls: Coordinates): + self._log.debug("Purls received, total purls before chunk: %s", len(purls.get_coordinates())) + + (purls, results) = self.getPurlsAndResultsFromCache(purls) + + self._log.debug("Purls checked against cache, total purls remaining to call OSS Index: %s", len(purls.get_coordinates())) chunk_purls = self.chunk(purls) - results = [] for purls in chunk_purls: data = {} data["coordinates"] = purls response = requests.post(self.get_url(), data=json.dumps(data), headers=self.get_headers()) if response.status_code == 200: - first_results = json.loads(response.text) + self._log.debug(response.headers) + first_results = json.loads(response.text, cls=ResultsDecoder) else: + self._log.debug("Response failed, status: %s", response.status_code) + self._log.debug("Failure reason if any: %s", response.reason) + self._log.debug("Failure text if any: %s", response.text) return None results.extend(first_results) + + (cached, num_cached) = self.maybeInsertIntoCache(results) + self._log.debug("Cached: " + str(cached) + " num_cached: " + str(num_cached)) return results + + def maybeInsertIntoCache(self, results: List[CoordinateResults]): + Coordinate = Query() + num_cached = 0 + cached = False + for coordinate in results: + mydatetime = datetime.now() + twelvelater = mydatetime + timedelta(hours=12) + result = self._db.search(Coordinate.purl == coordinate.getCoordinates()) + if len(result) is 0: + self._db.insert({'purl': coordinate.getCoordinates(), 'response': coordinate.toJSON(), 'ttl': twelvelater.isoformat()}) + self._log.debug("Coordinate inserted into cache: <%s>", coordinate.getCoordinates()) + num_cached += 1 + cached = True + else: + timetolive = parse(result[0]['ttl']) + if mydatetime > timetolive: + self._db.update({'response': coordinate.toJSON(), 'ttl': twelvelater.isoformat()}, doc_ids=[result[0].doc_id]) + self._log.debug("Coordinate: <%s> updated in cache because TTL expired", coordinate.getCoordinates()) + num_cached += 1 + cached = True + + return (cached, num_cached) + + def getPurlsAndResultsFromCache(self, purls: Coordinates): + valid = isinstance(purls, Coordinates) + if not valid: + return (None, None) + new_purls = Coordinates() + results = [] + Coordinate = Query() + for purl in purls.get_coordinates(): + mydatetime = datetime.now() + result = self._db.search(Coordinate.purl == purl) + if len(result) is 0 or parse(result[0]['ttl']) < mydatetime: + new_purls.add_coordinate(purl) + else: + results.append(json.loads(result[0]['response'], cls=ResultsDecoder)) + return (new_purls, results) + + def cleanCache(self): + self._db.purge() + return True + + def closeDB(self): + self._db.close() diff --git a/jake/parse/parse.py b/jake/parse/parse.py index 77666ce..e0aa0e6 100644 --- a/jake/parse/parse.py +++ b/jake/parse/parse.py @@ -14,7 +14,7 @@ import logging from shutil import which -from .coordinates import Coordinates +from jake.types.coordinates import Coordinates class Parse(object): def __init__(self): diff --git a/jake/test/__init__.py b/jake/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jake/condalistoutput.txt b/jake/test/condalistoutput.txt similarity index 100% rename from jake/condalistoutput.txt rename to jake/test/condalistoutput.txt diff --git a/jake/test/ossindexresponse.txt b/jake/test/ossindexresponse.txt new file mode 100644 index 0000000..80e3086 --- /dev/null +++ b/jake/test/ossindexresponse.txt @@ -0,0 +1 @@ +[{'coordinates': 'pkg:conda/astroid@2.3.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/astroid@2.3.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/blas@1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/blas@1.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ca-certificates@2019.8.28', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ca-certificates@2019.8.28', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/certifi@2019.9.11', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/certifi@2019.9.11', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/chardet@3.0.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/chardet@3.0.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/conda@4.3.16', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/conda@4.3.16', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/cytoolz@0.10.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/cytoolz@0.10.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/idna@2.8', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/idna@2.8', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/intel-openmp@2019.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/intel-openmp@2019.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/isort@4.3.21', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/isort@4.3.21', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/jake@0.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/jake@0.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/lazy-object-proxy@1.4.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/lazy-object-proxy@1.4.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libcxx@4.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libcxx@4.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libcxxabi@4.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libcxxabi@4.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libedit@3.1.20181209', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libedit@3.1.20181209', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libffi@3.2.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libffi@3.2.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libgfortran@3.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libgfortran@3.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mccabe@0.6.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mccabe@0.6.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl@2019.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl@2019.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl-service@2.3.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl-service@2.3.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl_fft@1.0.14', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl_fft@1.0.14', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl_random@1.1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl_random@1.1.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ncurses@6.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ncurses@6.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/numpy@1.17.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/numpy@1.17.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/numpy-base@1.17.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/numpy-base@1.17.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/openssl@1.1.1d', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/openssl@1.1.1d', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pickledb@0.9.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pickledb@0.9.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pip@19.2.3', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pip@19.2.3', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pycosat@0.6.3', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pycosat@0.6.3', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pylint@2.4.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pylint@2.4.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/python@3.7.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/python@3.7.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/readline@7.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/readline@7.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/requests@2.22.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/requests@2.22.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ruamel-yaml@0.16.5', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ruamel-yaml@0.16.5', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ruamel-yaml-clib@0.2.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ruamel-yaml-clib@0.2.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/setuptools@41.4.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/setuptools@41.4.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/six@1.12.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/six@1.12.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/sqlite@3.30.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/sqlite@3.30.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/tk@8.6.8', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/tk@8.6.8', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/toolz@0.10.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/toolz@0.10.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/typed-ast@1.4.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/typed-ast@1.4.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/urllib3@1.25.6', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/urllib3@1.25.6', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/wheel@0.33.6', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/wheel@0.33.6', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/wrapt@1.11.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/wrapt@1.11.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/xz@5.2.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/xz@5.2.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/zlib@1.2.11', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/zlib@1.2.11', 'vulnerabilities': []}] \ No newline at end of file diff --git a/jake/test/test_audit.py b/jake/test/test_audit.py new file mode 100644 index 0000000..81d609d --- /dev/null +++ b/jake/test/test_audit.py @@ -0,0 +1,30 @@ +# Copyright 2019 Sonatype Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import json + +from jake.audit.audit import Audit +from jake.types.results_decoder import ResultsDecoder + +class TestAudit(unittest.TestCase): + results = json.loads("[{'coordinates': 'pkg:conda/astroid@2.3.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/astroid@2.3.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/blas@1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/blas@1.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ca-certificates@2019.8.28', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ca-certificates@2019.8.28', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/certifi@2019.9.11', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/certifi@2019.9.11', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/chardet@3.0.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/chardet@3.0.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/conda@4.3.16', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/conda@4.3.16', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/cytoolz@0.10.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/cytoolz@0.10.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/idna@2.8', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/idna@2.8', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/intel-openmp@2019.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/intel-openmp@2019.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/isort@4.3.21', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/isort@4.3.21', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/lazy-object-proxy@1.4.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/lazy-object-proxy@1.4.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libcxx@4.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libcxx@4.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libcxxabi@4.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libcxxabi@4.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libedit@3.1.20181209', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libedit@3.1.20181209', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libffi@3.2.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libffi@3.2.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libgfortran@3.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libgfortran@3.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mccabe@0.6.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mccabe@0.6.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl@2019.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl@2019.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl-service@2.3.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl-service@2.3.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl_fft@1.0.14', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl_fft@1.0.14', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl_random@1.1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl_random@1.1.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ncurses@6.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ncurses@6.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/numpy@1.17.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/numpy@1.17.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/numpy-base@1.17.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/numpy-base@1.17.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/openssl@1.1.1d', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/openssl@1.1.1d', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pip@19.2.3', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pip@19.2.3', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pycosat@0.6.3', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pycosat@0.6.3', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pylint@2.4.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pylint@2.4.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/python@3.7.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/python@3.7.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/readline@7.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/readline@7.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/requests@2.22.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/requests@2.22.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ruamel.yaml@0.16.5', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ruamel.yaml@0.16.5', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ruamel.yaml.clib@0.2.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ruamel.yaml.clib@0.2.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/setuptools@41.4.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/setuptools@41.4.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/six@1.12.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/six@1.12.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/sqlite@3.30.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/sqlite@3.30.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/tk@8.6.8', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/tk@8.6.8', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/toolz@0.10.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/toolz@0.10.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/typed-ast@1.4.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/typed-ast@1.4.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/urllib3@1.25.6', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/urllib3@1.25.6', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/wheel@0.33.6', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/wheel@0.33.6', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/wrapt@1.11.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/wrapt@1.11.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/xz@5.2.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/xz@5.2.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/zlib@1.2.11', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/zlib@1.2.11', 'vulnerabilities': []}]".replace("'", '"'), cls=ResultsDecoder) + + def setUp(self): + self.func = Audit() + + def test_callauditResultsPrintsOutput(self): + self.assertEqual(self.func.auditResults(self.results), self.expectedResults()) + + def expectedResults(self): + return 0 diff --git a/jake/test/test_coordinateresults.py b/jake/test/test_coordinateresults.py new file mode 100644 index 0000000..c7e1b99 --- /dev/null +++ b/jake/test/test_coordinateresults.py @@ -0,0 +1,35 @@ +# Copyright 2019 Sonatype Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import json +import ast + +from jake.types.coordinateresults import CoordinateResults + +class TestResultsDecoder(unittest.TestCase): + def test_toJsonOnCoordinateResultsReturnsProperJson(self): + undertest = CoordinateResults() + undertest.setCoordinates("pkg:conda/thing@1.0.0") + undertest.setReference("http://www.wrestling.com") + undertest.setVulnerabilities([]) + + result = undertest.toJSON() + dictionary = ast.literal_eval(result) + + self.assertEqual(isinstance(result, str), True) + self.assertEqual(dictionary['coordinates'], "pkg:conda/thing@1.0.0") + self.assertEqual(dictionary['reference'], "http://www.wrestling.com") + self.assertEqual(dictionary['vulnerabilities'], []) + + diff --git a/jake/test/test_ossindex.py b/jake/test/test_ossindex.py new file mode 100644 index 0000000..e9a0432 --- /dev/null +++ b/jake/test/test_ossindex.py @@ -0,0 +1,152 @@ +# Copyright 2019 Sonatype Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import json +from unittest.mock import patch +from pathlib import Path +from tinydb import TinyDB, Query +from dateutil.parser import parse +from datetime import timedelta +from typing import List + +from jake.ossindex.ossindex import OssIndex +from jake.parse.parse import Parse +from jake.types.coordinates import Coordinates +from jake.types.coordinateresults import CoordinateResults +from jake.types.results_decoder import ResultsDecoder + +class TestOssIndex(unittest.TestCase): + def setUp(self): + self.func = OssIndex(url="http://blahblah", headers={"thing": "thing", "anotherthing": "anotherthing"}, cache_location="/tmp") + self.parse = Parse() + + def tearDown(self): + self.func.closeDB() + if Path('/tmp/.ossindex/jake.json').exists(): + Path('/tmp/.ossindex/jake.json').unlink() + + def test_getHeaders(self): + self.assertEqual(self.func.get_headers(), {"thing": "thing", "anotherthing": "anotherthing"}) + + def test_getUrl(self): + self.assertEqual(self.func.get_url(), "http://blahblah") + + def get_fakePurls(self): + fakePurls = Coordinates() + fakePurls.add_coordinate("pkg:conda/thing1") + fakePurls.add_coordinate("pkg:conda/thing2") + fakePurls.add_coordinate("pkg:conda/thing3") + return fakePurls + + def get_fakeActualPurls(self): + fakeActualPurls = Coordinates() + fakeActualPurls.add_coordinate("pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0") + return fakeActualPurls + + @patch('jake.ossindex.ossindex.requests.post') + def test_callGetDependenciesReturnsPurls(self, mock_post): + fn = Path(__file__).parent / "ossindexresponse.txt" + with open(fn, "r") as stdin: + mock_result = stdin.read().replace("'", '"') + mock_post.return_value.status_code = 200 + mock_post.return_value.text = mock_result + response = self.func.callOSSIndex(self.get_fakePurls()) + self.assertEqual(len(response), 46) + self.assertEqual(response[0].getCoordinates(), "pkg:conda/astroid@2.3.1") + + @patch('jake.ossindex.ossindex.requests.post') + def test_callOSSIndex_PostReturnsError(self, mock_post): + mock_post.return_value.status_code = 404 + mock_post.return_value.text = "yadda" + response = self.func.callOSSIndex(self.get_fakePurls()) + self.assertEqual(response, None) + + def test_chunk(self): + fn = Path(__file__).parent / "condalistoutput.txt" + with open(fn, "r") as stdin: + purls = self.parse.getDependenciesFromStdin(stdin) + actual_result = self.func.chunk(purls) + self.assertEqual(len(actual_result), 3) + self.assertEqual(len(actual_result[0]), 128) + self.assertEqual(actual_result[0][0], "pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0") + self.assertEqual(actual_result[1][0], "pkg:conda/mistune@0.8.4") + self.assertEqual(actual_result[2][0], "pkg:conda/yaml@0.1.7") + + def test_insertIntoCache(self): + fn = Path(__file__).parent / "ossindexresponse.txt" + with open(fn, "r") as stdin: + response = json.loads(stdin.read().replace("'", '"'), cls=ResultsDecoder) + (cached, num_cached) = self.func.maybeInsertIntoCache(response) + self.assertEqual(num_cached, 46) + self.assertEqual(cached, True) + + def test_insertIntoCacheDoesNotDuplicate(self): + fn = Path(__file__).parent / "ossindexresponse.txt" + with open(fn, "r") as stdin: + response = json.loads(stdin.read().replace("'", '"'), cls=ResultsDecoder) + self.func.maybeInsertIntoCache(response) + (cached, num_cached) = self.func.maybeInsertIntoCache(response) + self.assertEqual(num_cached, 0) + self.assertEqual(cached, False) + + def test_insertIntoCacheExpiredTTL(self): + db = TinyDB('/tmp/.ossindex/jake.json') + Coordinates = Query() + response = self.stringToCoordinatesResult("[{'coordinates': 'pkg:conda/astroid@2.3.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/astroid@2.3.1', 'vulnerabilities': []}]") + self.func.maybeInsertIntoCache(response) + resultExpired = db.search(Coordinates.purl == "pkg:conda/astroid@2.3.1") + timeUnwind = parse(resultExpired[0]['ttl']) - timedelta(hours=13) + db.update({'ttl': timeUnwind.isoformat()}, Coordinates.purl == "pkg:conda/astroid@2.3.1") + + nextResponse = self.stringToCoordinatesResult("[{'coordinates': 'pkg:conda/astroid@2.3.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/astroid@2.3.1', 'vulnerabilities': []}]") + (cached, num_cached) = self.func.maybeInsertIntoCache(nextResponse) + self.assertEqual(cached, True) + self.assertEqual(num_cached, 1) + db.close() + + def test_getPurlsFromCache(self): + self.func.maybeInsertIntoCache(self.stringToCoordinatesResult("[{'coordinates': 'pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/alabaster@0.7.12', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/alabaster@0.7.12', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/anaconda@2019.07', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/anaconda@2019.07', 'vulnerabilities': []}]")) + (new_purls, results) = self.func.getPurlsAndResultsFromCache(self.get_fakeActualPurls()) + self.assertEqual(isinstance(results, List), True) + self.assertEqual(isinstance(results[0], CoordinateResults), True) + self.assertEqual(results[0].getCoordinates(), "pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0") + self.assertEqual(results[0].getReference(), "https://ossindex.sonatype.org/component/pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0") + self.assertEqual(isinstance(results[0].getVulnerabilities(), List), True) + self.assertEqual(len(new_purls.get_coordinates()), 0) + self.assertEqual(isinstance(new_purls, Coordinates), True) + + def test_getPurlsFromCacheWithCacheMiss(self): + self.func.maybeInsertIntoCache(self.stringToCoordinatesResult("[{'coordinates': 'pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0', 'vulnerabilities': []}]")) + fake_purls = self.get_fakeActualPurls() + fake_purls.add_coordinate("pkg:conda/alabaster@0.7.12") + (new_purls, results) = self.func.getPurlsAndResultsFromCache(fake_purls) + self.assertEqual(len(new_purls.get_coordinates()), 1) + self.assertEqual(isinstance(new_purls, Coordinates), True) + self.assertEqual(isinstance(results, List), True) + self.assertEqual(isinstance(results[0], CoordinateResults), True) + self.assertEqual(isinstance(results[0].getVulnerabilities(), List), True) + self.assertEqual(results[0].getCoordinates(), "pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0") + self.assertEqual(results[0].getReference(), "https://ossindex.sonatype.org/component/pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0") + + def test_getPurlsFromCacheWithNonValidObject(self): + (new_purls, results) = self.func.getPurlsAndResultsFromCache("bad data") + self.assertEqual(new_purls, None) + self.assertEqual(results, None) + + def test_cleanWipesDB(self): + self.func.maybeInsertIntoCache(self.stringToCoordinatesResult("[{'coordinates': 'pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0', 'vulnerabilities': []}]")) + self.assertEqual(self.func.cleanCache(), True) + + def stringToCoordinatesResult(self, string): + return json.loads(string.replace("'", '"'), cls=ResultsDecoder) diff --git a/jake/test_parse.py b/jake/test/test_parse.py similarity index 98% rename from jake/test_parse.py rename to jake/test/test_parse.py index c4212ee..a0d74f7 100644 --- a/jake/test_parse.py +++ b/jake/test/test_parse.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import unittest -import sys import pathlib -import json from jake.parse.parse import Parse diff --git a/jake/test/test_results_decoder.py b/jake/test/test_results_decoder.py new file mode 100644 index 0000000..4a56635 --- /dev/null +++ b/jake/test/test_results_decoder.py @@ -0,0 +1,34 @@ +# Copyright 2019 Sonatype Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import pathlib +import json + +from typing import List + +from jake.types.results_decoder import ResultsDecoder +from jake.types.coordinateresults import CoordinateResults + +class TestResultsDecoder(unittest.TestCase): + def test_resultsDecoderCanTransformOssIndexResponseIntoCoordinateResultsList(self): + fn = pathlib.Path(__file__).parent / "ossindexresponse.txt" + with open(fn, "r") as stdin: + result = json.loads(stdin.read().replace("'", '"'), cls=ResultsDecoder) + self.assertEqual(len(result), 46) + self.assertEqual(isinstance(result, List), True) + self.assertEqual(isinstance(result[0], CoordinateResults), True) + self.assertEqual(result[0].getCoordinates(), "pkg:conda/astroid@2.3.1") + self.assertEqual(result[0].getReference(), "https://ossindex.sonatype.org/component/pkg:conda/astroid@2.3.1") + self.assertEqual(isinstance(result[0].getVulnerabilities(), List), True) + self.assertEqual(len(result[0].getVulnerabilities()), 0) diff --git a/jake/test_audit.py b/jake/test_audit.py deleted file mode 100644 index 75df21b..0000000 --- a/jake/test_audit.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2019 Sonatype Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest -import json -import ast - -from jake.audit.audit import Audit - -class TestAudit(unittest.TestCase): - jsonList = ast.literal_eval("[{'coordinates': 'pkg:conda/astroid@2.3.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/astroid@2.3.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/blas@1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/blas@1.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ca-certificates@2019.8.28', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ca-certificates@2019.8.28', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/certifi@2019.9.11', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/certifi@2019.9.11', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/chardet@3.0.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/chardet@3.0.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/conda@4.3.16', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/conda@4.3.16', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/cytoolz@0.10.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/cytoolz@0.10.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/idna@2.8', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/idna@2.8', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/intel-openmp@2019.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/intel-openmp@2019.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/isort@4.3.21', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/isort@4.3.21', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/lazy-object-proxy@1.4.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/lazy-object-proxy@1.4.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libcxx@4.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libcxx@4.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libcxxabi@4.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libcxxabi@4.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libedit@3.1.20181209', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libedit@3.1.20181209', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libffi@3.2.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libffi@3.2.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libgfortran@3.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libgfortran@3.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mccabe@0.6.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mccabe@0.6.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl@2019.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl@2019.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl-service@2.3.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl-service@2.3.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl_fft@1.0.14', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl_fft@1.0.14', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl_random@1.1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl_random@1.1.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ncurses@6.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ncurses@6.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/numpy@1.17.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/numpy@1.17.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/numpy-base@1.17.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/numpy-base@1.17.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/openssl@1.1.1d', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/openssl@1.1.1d', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pip@19.2.3', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pip@19.2.3', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pycosat@0.6.3', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pycosat@0.6.3', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pylint@2.4.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pylint@2.4.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/python@3.7.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/python@3.7.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/readline@7.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/readline@7.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/requests@2.22.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/requests@2.22.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ruamel.yaml@0.16.5', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ruamel.yaml@0.16.5', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ruamel.yaml.clib@0.2.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ruamel.yaml.clib@0.2.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/setuptools@41.4.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/setuptools@41.4.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/six@1.12.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/six@1.12.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/sqlite@3.30.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/sqlite@3.30.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/tk@8.6.8', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/tk@8.6.8', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/toolz@0.10.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/toolz@0.10.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/typed-ast@1.4.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/typed-ast@1.4.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/urllib3@1.25.6', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/urllib3@1.25.6', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/wheel@0.33.6', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/wheel@0.33.6', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/wrapt@1.11.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/wrapt@1.11.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/xz@5.2.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/xz@5.2.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/zlib@1.2.11', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/zlib@1.2.11', 'vulnerabilities': []}]") - jsonResult = json.dumps(jsonList) - - def setUp(self): - self.func = Audit() - - def test_callauditResultsPrintsOutput(self): - self.assertEqual(self.func.auditResults(json.loads(self.jsonResult)), self.expectedResults()) - - def expectedResults(self): - return 0 diff --git a/jake/test_ossindex.py b/jake/test_ossindex.py deleted file mode 100644 index 108df77..0000000 --- a/jake/test_ossindex.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2019 Sonatype Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import ast -import unittest -from unittest.mock import Mock, patch -import sys -import pathlib - -import json - -from jake.ossindex.ossindex import OssIndex -from jake.parse.parse import Parse -from jake.parse.coordinates import Coordinates - -class TestOssIndex(unittest.TestCase): - def setUp(self): - self.func = OssIndex(url="http://blahblah", headers={"thing": "thing", "anotherthing": "anotherthing"}) - - def test_getHeaders(self): - self.assertEqual(self.func.get_headers(), {"thing": "thing", "anotherthing": "anotherthing"}) - - def test_getUrl(self): - self.assertEqual(self.func.get_url(), "http://blahblah") - - def get_fakePurls(self): - fakePurls = Coordinates() - fakePurls.add_coordinate("pkg:conda/thing1") - fakePurls.add_coordinate("pkg:conda/thing2") - fakePurls.add_coordinate("pkg:conda/thing3") - return fakePurls - - @patch('jake.ossindex.ossindex.requests.post') - def test_callGetDependenciesReturnsPurls(self, mock_post): - mock_result = '[{"coordinates": "pkg:conda/thing1"}, {"coordinates": "pkg:conda/thing2"}, {"coordinates": "pkg:conda/thing3"}]' - - mock_post.return_value.status_code = 200 - mock_post.return_value.text = mock_result - response = self.func.callOSSIndex(self.get_fakePurls()) - - self.assertEqual(len(response), 3) - self.assertEqual(response[0]["coordinates"], "pkg:conda/thing1") - - def test_chunk(self): - fn = pathlib.Path(__file__).parent / "condalistoutput.txt" - sys.stdin = open(fn, "r") - parse = Parse() - purls = parse.getDependenciesFromStdin(sys.stdin) - actual_result = self.func.chunk(purls) - self.assertEqual(len(actual_result), 3) - self.assertEqual(len(actual_result[0]), 128) - self.assertEqual(actual_result[0][0], "pkg:conda/_ipyw_jlab_nb_ext_conf@0.1.0") - self.assertEqual(actual_result[1][0], "pkg:conda/mistune@0.8.4") - self.assertEqual(actual_result[2][0], "pkg:conda/yaml@0.1.7") - \ No newline at end of file diff --git a/jake/types/__init__.py b/jake/types/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jake/types/coordinateresults.py b/jake/types/coordinateresults.py new file mode 100644 index 0000000..580c602 --- /dev/null +++ b/jake/types/coordinateresults.py @@ -0,0 +1,47 @@ +# Copyright 2019 Sonatype Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +class CoordinateResults(object): + def __init__(self): + self._coordinates = "" + self._reference = "" + self._vulnerabilities = [] + + def setCoordinates(self, coordinate): + self._coordinates = coordinate + + def getCoordinates(self): + return self._coordinates + + def setReference(self, reference): + self._reference = reference + + def getReference(self): + return self._reference + + def setVulnerabilities(self, vulnerabilities): + self._vulnerabilities = vulnerabilities + + def getVulnerabilities(self): + return self._vulnerabilities + + def toJSON(self): + return json.dumps(self, default=lambda o: CoordinateJsonResult(o.getCoordinates(), o.getReference(), o.getVulnerabilities()).__dict__) + +class CoordinateJsonResult(object): + def __init__(self, coordinate, reference, vulnerabilities): + self.coordinates = coordinate + self.reference = reference + self.vulnerabilities = vulnerabilities diff --git a/jake/parse/coordinates.py b/jake/types/coordinates.py similarity index 100% rename from jake/parse/coordinates.py rename to jake/types/coordinates.py diff --git a/jake/types/results_decoder.py b/jake/types/results_decoder.py new file mode 100644 index 0000000..39e79a6 --- /dev/null +++ b/jake/types/results_decoder.py @@ -0,0 +1,28 @@ +# Copyright 2019 Sonatype Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +from jake.types.coordinateresults import CoordinateResults + +class ResultsDecoder(json.JSONDecoder): + def __init__(self): + json.JSONDecoder.__init__(self, object_hook=self.dict_to_object) + + def dict_to_object(self, dictionary): + item = CoordinateResults() + item.setCoordinates(dictionary["coordinates"]) + item.setReference(dictionary["reference"]) + item.setVulnerabilities(dictionary["vulnerabilities"]) + + return item diff --git a/requirements.txt b/requirements.txt index 2b71d46..6c75c76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,12 +7,15 @@ idna==2.8 isort==4.3.21 lazy-object-proxy==1.4.2 mccabe==0.6.1 +pickleDB==0.9.2 pycosat==0.6.3 pylint==2.4.2 +python-dateutil==2.8.0 requests==2.22.0 ruamel.yaml==0.16.5 ruamel.yaml.clib==0.2.0 six==1.12.0 +tinydb==3.15.1 toolz==0.10.0 typed-ast==1.4.0 urllib3==1.25.6 diff --git a/setup.py b/setup.py index 4bc430b..4954e9a 100644 --- a/setup.py +++ b/setup.py @@ -11,11 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from jake._version import __version__ from setuptools import setup setup( name = 'jake', - version = '0.0.1', + version = __version__, packages = ['jake'], entry_points = { 'console_scripts': [