Implemented caching in jake (#7)

* Implemented caching in jake
sonatype-nexus-community · Oct 30, 2019 · a5a31cc · a5a31cc
1 parent 7cdf0f5
commit a5a31cc
Show file tree

Hide file tree

Showing 23 changed files with 482 additions and 123 deletions.
diff --git a/.gitignore b/.gitignore
@@ -106,6 +106,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+jake.json
 
 # Spyder project settings
 .spyderproject

diff --git a/README.md b/README.md
@@ -11,27 +11,61 @@
 ### Usage
 
 ```
- ~ > jake
+$ jake --help
+usage: jake [-h] [-V] [-VV] [-C] {ddt}
+
+positional arguments:
+  {ddt}           run jake
+
+optional arguments:
+  -h, --help      show this help message and exit
+  -V, --version   show program version
+  -VV, --verbose  set verbosity level to debug
+  -C, --clean     wipe out jake cache
 ```
 
+Typical usage of `jake` is to run it like so: `conda list | jake ddt`, which will feed your Conda dependencies in your current Conda environment to `jake`, which will then reach out and check OSS Index to see if they are vulnerable!
+
 ### Options
 
+You may also run `jake` with `-VV` for a slew of debug data, in case you are running in to an odd situation, or you want to help out on development!
+
+You can also run `jake -C` to clean out your local cache if desired. We cache results from OSS Index for 12 hours to prevent you from potentially getting rate limited (as your dependencies likely won't change super often). 
+
 ## Why Jake?
 
 Jake The Snake was scared of Snakes. The finishing move was DDT. He finishes the Snake with DDT.
 
 ## Installation
 
+### Download from PyPI
+
+TBD
+
 ### Build from source
 
-### Download as Conda package
+* Clone the repo
+* Install Python 3.7 or higher
+* Ensure pip is installed (it should be)
+* Run `python3 venv .venv` (or whatever virtual environment you prefer)
+* Run `source .venv/bin/activate`
+* Run `pip install -r requirements`
+* Run `pip install -e .`
+
+Once you've done this, you should have `jake` available to test with fairly globally, pointed at the local source you've cloned.
 
 ## Development
 
-`jake` is written using Python.
+`jake` is written using Python 3.7
 
 This project also uses `pip` for dependencies, so you will need to download make sure you have `pip`.
 
+Follow instructions in Build from source.
+
+Tests can be run with `python3 -m unittest discover`
+
+More TBD.
+
 ## Contributing
 
 We care a lot about making the world a safer place, and that's why we created `jake`. If you as well want to

diff --git a/jake/__main__.py b/jake/__main__.py
@@ -23,35 +23,48 @@
 from jake.parse.parse import Parse
 from jake.audit.audit import Audit
 
+from ._version import __version__
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('run', help='run jake', choices=['ddt'])
-    parser.add_argument('-V', '--version', help='show program version', action='store_true')
-    parser.add_argument('-E', '--env', help="conda environment to run", default='root')
+    parser.add_argument('-V', '--version', help='show program version and exit', action='store_true')
     parser.add_argument('-VV', '--verbose', help="set verbosity level to debug", action='store_true')
+    parser.add_argument('-C', '--clean', help="wipe out jake cache", action='store_true')
     args = parser.parse_args()
     log = logging.getLogger('jake')
+
     if args.verbose:
         log.setLevel(logging.DEBUG)
     else:
         log.setLevel(logging.ERROR)
 
+    if args.version:
+        print(__version__)
+        _exit(0)
+
     parse = Parse()
     ossindex = OssIndex()
     audit = Audit()
 
+    if args.clean:
+        ossindex.cleanCache()
+
     if args.run == 'ddt':
         log.info('Calling OSS Index')
         purls = parse.getDependenciesFromStdin(sys.stdin)
         if purls is None:
-            log.error("No purls returned, likely culprit is no Conda installed")
+            log.error("No purls returned, ensure that conda list is returning a list of dependencies")
             _exit(EX_OSERR)
 
-        log.debug(purls)
+        log.debug("Total purls: %s", len(purls.get_coordinates()))
 
         response = ossindex.callOSSIndex(purls)
-
-        code = audit.auditResults(response)
+        if response is not None:
+            code = audit.auditResults(response)
+        else:
+            log.error("Something went horribly wrong, please rerun with -VV to see what happened")
+            _exit(EX_OSERR)
 
         _exit(code)
 

diff --git a/jake/_version.py b/jake/_version.py
@@ -0,0 +1 @@
+__version__="0.0.1"
diff --git a/jake/audit/audit.py b/jake/audit/audit.py
@@ -13,12 +13,16 @@
 # limitations under the License.
 import logging
 
+from typing import List
+
+from jake.types.coordinateresults import CoordinateResults
+
 class Audit(object):
     def __init__(self):
         self._log = logging.getLogger('jake')
 
-    def auditResults(self, results):
-        self._log.debug(results)
+    def auditResults(self, results: List[CoordinateResults]):
+        self._log.debug("Results recieved, %s total results", len(results))
 
         totalVulns = 0
         pkgNum = 0
@@ -29,15 +33,15 @@ def auditResults(self, results):
 
         return totalVulns
 
-    def printResult(self, coordinate, number, length):
-        if len(coordinate['vulnerabilities']) == 0:
-            print("[{}/{}] - {} - no known vulnerabilities for this version".format(number, length, coordinate['coordinates']))
-            return len(coordinate['vulnerabilities'])
+    def printResult(self, coordinate: CoordinateResults, number, length):
+        if len(coordinate.getVulnerabilities()) == 0:
+            print("[{}/{}] - {} - no known vulnerabilities for this version".format(number, length, coordinate.getCoordinates()))
+            return len(coordinate.getVulnerabilities())
         else:
-            print("[{}/{}] - {} [VULNERABLE] {} known vulnerabilities for this version".format(number, length, coordinate['coordinates'], len(coordinate['vulnerabilities'])))
-            for vulnerability in coordinate['vulnerabilities']:
+            print("[{}/{}] - {} [VULNERABLE] {} known vulnerabilities for this version".format(number, length, coordinate.getCoordinates(), len(coordinate.getVulnerabilities())))
+            for vulnerability in coordinate.getVulnerabilities():
                 self.printVulnerability(vulnerability)
-            return len(coordinate['vulnerabilities'])
+            return len(coordinate.getVulnerabilities())
 
     def printVulnerability(self, vulnerability):
         print(vulnerability)
diff --git a/jake/ossindex/ossindex.py b/jake/ossindex/ossindex.py
@@ -15,12 +15,29 @@
 import logging
 import json
 
+from typing import List
+from datetime import datetime, timedelta
+from dateutil.parser import parse
+from tinydb import TinyDB, Query
+from pathlib import Path
+from jake.types.coordinates import Coordinates
+from jake.types.results_decoder import ResultsDecoder
+from jake.types.coordinateresults import CoordinateResults
+
 class OssIndex(object):
-    def __init__(self, url='https://ossindex.sonatype.org/api/v3/component-report', headers={'Content-type': 'application/json', 'User-Agent': 'jake'}):
+    def __init__(self, url='https://ossindex.sonatype.org/api/v3/component-report', headers={'Content-type': 'application/vnd.ossindex.component-report-request.v1+json', 'User-Agent': 'jake'}, cache_location=''):
         self._url = url
         self._headers = headers
         self._log = logging.getLogger('jake')
         self._maxcoords = 128
+        if cache_location == '':
+            home = str(Path.home())
+            dir_oss = home + "/.ossindex/"
+        else:
+            dir_oss = cache_location + "/.ossindex/"
+        if not Path(dir_oss).exists():
+            Path(dir_oss).mkdir(parents=True, exist_ok=True)
+        self._db = TinyDB(dir_oss + "jake.json")
 
     def get_url(self):
         return self._url
@@ -47,18 +64,74 @@ def chunk(self, purls):
             chunks.append(divided)
         return chunks
 
-    def callOSSIndex(self, purls):
-        self._log.debug(purls)
+    def callOSSIndex(self, purls: Coordinates):
+        self._log.debug("Purls received, total purls before chunk: %s", len(purls.get_coordinates()))
+
+        (purls, results) = self.getPurlsAndResultsFromCache(purls)
+
+        self._log.debug("Purls checked against cache, total purls remaining to call OSS Index: %s", len(purls.get_coordinates()))
 
         chunk_purls = self.chunk(purls)
-        results = []
         for purls in chunk_purls:
             data = {}
             data["coordinates"] = purls
             response = requests.post(self.get_url(), data=json.dumps(data), headers=self.get_headers())
             if response.status_code == 200:
-                first_results = json.loads(response.text)
+                self._log.debug(response.headers)
+                first_results = json.loads(response.text, cls=ResultsDecoder)
             else:
+                self._log.debug("Response failed, status: %s", response.status_code)
+                self._log.debug("Failure reason if any: %s", response.reason)
+                self._log.debug("Failure text if any: %s", response.text)
                 return None
             results.extend(first_results)
+
+        (cached, num_cached) = self.maybeInsertIntoCache(results)
+        self._log.debug("Cached: " + str(cached) + " num_cached: " + str(num_cached))
         return results
+
+    def maybeInsertIntoCache(self, results: List[CoordinateResults]):
+        Coordinate = Query()
+        num_cached = 0
+        cached = False
+        for coordinate in results:
+            mydatetime = datetime.now()
+            twelvelater = mydatetime + timedelta(hours=12)
+            result = self._db.search(Coordinate.purl == coordinate.getCoordinates())
+            if len(result) is 0:
+                self._db.insert({'purl': coordinate.getCoordinates(), 'response': coordinate.toJSON(), 'ttl': twelvelater.isoformat()})
+                self._log.debug("Coordinate inserted into cache: <%s>", coordinate.getCoordinates())
+                num_cached += 1
+                cached = True
+            else:
+                timetolive = parse(result[0]['ttl'])
+                if mydatetime > timetolive:
+                    self._db.update({'response': coordinate.toJSON(), 'ttl': twelvelater.isoformat()}, doc_ids=[result[0].doc_id])
+                    self._log.debug("Coordinate: <%s> updated in cache because TTL expired", coordinate.getCoordinates())
+                    num_cached += 1
+                    cached = True
+
+        return (cached, num_cached)
+
+    def getPurlsAndResultsFromCache(self, purls: Coordinates):
+        valid = isinstance(purls, Coordinates)
+        if not valid:
+            return (None, None)
+        new_purls = Coordinates()
+        results = []
+        Coordinate = Query()
+        for purl in purls.get_coordinates():
+            mydatetime = datetime.now()
+            result = self._db.search(Coordinate.purl == purl)
+            if len(result) is 0 or parse(result[0]['ttl']) < mydatetime:
+                new_purls.add_coordinate(purl)
+            else:
+                results.append(json.loads(result[0]['response'], cls=ResultsDecoder))
+        return (new_purls, results)
+
+    def cleanCache(self):
+        self._db.purge()
+        return True
+
+    def closeDB(self):
+        self._db.close()
diff --git a/jake/parse/parse.py b/jake/parse/parse.py
@@ -14,7 +14,7 @@
 import logging
 from shutil import which
 
-from .coordinates import Coordinates
+from jake.types.coordinates import Coordinates
 
 class Parse(object):
     def __init__(self):

diff --git a/jake/test/__init__.py b/jake/test/__init__.py
diff --git a/jake/condalistoutput.txt → jake/test/condalistoutput.txt b/jake/condalistoutput.txt → jake/test/condalistoutput.txt
diff --git a/jake/test/ossindexresponse.txt b/jake/test/ossindexresponse.txt
@@ -0,0 +1 @@
+[{'coordinates': 'pkg:conda/astroid@2.3.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/astroid@2.3.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/blas@1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/blas@1.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ca-certificates@2019.8.28', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ca-certificates@2019.8.28', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/certifi@2019.9.11', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/certifi@2019.9.11', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/chardet@3.0.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/chardet@3.0.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/conda@4.3.16', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/conda@4.3.16', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/cytoolz@0.10.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/cytoolz@0.10.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/idna@2.8', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/idna@2.8', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/intel-openmp@2019.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/intel-openmp@2019.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/isort@4.3.21', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/isort@4.3.21', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/jake@0.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/jake@0.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/lazy-object-proxy@1.4.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/lazy-object-proxy@1.4.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libcxx@4.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libcxx@4.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libcxxabi@4.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libcxxabi@4.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libedit@3.1.20181209', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libedit@3.1.20181209', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libffi@3.2.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libffi@3.2.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/libgfortran@3.0.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/libgfortran@3.0.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mccabe@0.6.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mccabe@0.6.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl@2019.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl@2019.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl-service@2.3.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl-service@2.3.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl_fft@1.0.14', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl_fft@1.0.14', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/mkl_random@1.1.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/mkl_random@1.1.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ncurses@6.1', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ncurses@6.1', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/numpy@1.17.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/numpy@1.17.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/numpy-base@1.17.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/numpy-base@1.17.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/openssl@1.1.1d', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/openssl@1.1.1d', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pickledb@0.9.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pickledb@0.9.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pip@19.2.3', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pip@19.2.3', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pycosat@0.6.3', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pycosat@0.6.3', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/pylint@2.4.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/pylint@2.4.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/python@3.7.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/python@3.7.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/readline@7.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/readline@7.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/requests@2.22.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/requests@2.22.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ruamel-yaml@0.16.5', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ruamel-yaml@0.16.5', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/ruamel-yaml-clib@0.2.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/ruamel-yaml-clib@0.2.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/setuptools@41.4.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/setuptools@41.4.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/six@1.12.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/six@1.12.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/sqlite@3.30.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/sqlite@3.30.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/tk@8.6.8', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/tk@8.6.8', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/toolz@0.10.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/toolz@0.10.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/typed-ast@1.4.0', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/typed-ast@1.4.0', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/urllib3@1.25.6', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/urllib3@1.25.6', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/wheel@0.33.6', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/wheel@0.33.6', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/wrapt@1.11.2', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/wrapt@1.11.2', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/xz@5.2.4', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/xz@5.2.4', 'vulnerabilities': []}, {'coordinates': 'pkg:conda/zlib@1.2.11', 'reference': 'https://ossindex.sonatype.org/component/pkg:conda/zlib@1.2.11', 'vulnerabilities': []}]